From 30f89e7cc4641256a6052fb03ee8bdb6cd16c0c6 Mon Sep 17 00:00:00 2001 From: Jonathan Molinatto Date: Mon, 29 Sep 2025 11:06:33 -0400 Subject: [PATCH 1/8] remove the keyword field from the pipeline. it should not be used for byte counts --- ...t-proxy-bcreportermainv1.log-expected.json | 10 --------- ...st-proxy-bcreportersslv1.log-expected.json | 10 --------- .../test-proxy-main.log-expected.json | 22 ------------------- .../elasticsearch/ingest_pipeline/default.yml | 11 ++++++++++ .../data_stream/log/fields/base-fields.yml | 6 +++++ packages/proxysg/docs/README.md | 2 ++ 6 files changed, 19 insertions(+), 42 deletions(-) diff --git a/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportermainv1.log-expected.json b/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportermainv1.log-expected.json index 3f483ed860d..5a20e20b2b0 100644 --- a/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportermainv1.log-expected.json +++ b/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportermainv1.log-expected.json @@ -31,7 +31,6 @@ "ip": "10.0.1.2" }, "client_to_server": { - "bytes": "111", "categories": "Web Infrastructure", "host": "www.msftconnecttest.com", "method": "GET", @@ -54,7 +53,6 @@ "supplier_country": "None" }, "server_to_client": { - "bytes": "1033", "filter_result": "DENIED", "status": "302" }, @@ -122,7 +120,6 @@ "ip": "10.0.1.2" }, "client_to_server": { - "bytes": "582", "categories": "Chat (IM)/SMS;Social Networking", "host": "tr6.snapchat.com", "method": "OPTIONS", @@ -139,7 +136,6 @@ "supplier_country": "None" }, "server_to_client": { - "bytes": "1297", "filter_result": "DENIED", "status": "401" }, @@ -211,7 +207,6 @@ "ip": "10.0.1.2" }, "client_to_server": { - "bytes": "291", "categories": "Web Infrastructure", "host": "ctldl.windowsupdate.com", "method": "GET", @@ -229,7 +224,6 @@ "supplier_country": "None" }, "server_to_client": { - "bytes": "1297", "filter_result": "DENIED", "status": "401" }, @@ -297,7 +291,6 @@ "ip": "10.0.1.2" }, "client_to_server": { - "bytes": "111", "categories": "Web Infrastructure", "host": "www.msftconnecttest.com", "method": "GET", @@ -314,7 +307,6 @@ "supplier_country": "None" }, "server_to_client": { - "bytes": "1033", "filter_result": "DENIED", "status": "302" }, @@ -381,7 +373,6 @@ "ip": "10.0.1.2" }, "client_to_server": { - "bytes": "510", "categories": "Web Ads/Analytics", "host": "dt.adsafeprotected.com", "method": "OPTIONS", @@ -399,7 +390,6 @@ "supplier_country": "None" }, "server_to_client": { - "bytes": "1316", "filter_result": "DENIED", "status": "401" }, diff --git a/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportersslv1.log-expected.json b/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportersslv1.log-expected.json index e4b442e1055..a75626e4b41 100644 --- a/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportersslv1.log-expected.json +++ b/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportersslv1.log-expected.json @@ -31,7 +31,6 @@ "ip": "10.0.1.1" }, "client_to_server": { - "bytes": "11343", "categories": "Web Ads/Analytics", "host": "server.googleapis.com", "method": "POST", @@ -53,7 +52,6 @@ "supplier_country": "None" }, "server_to_client": { - "bytes": "1039", "filter_result": "DENIED", "status": "307" }, @@ -121,7 +119,6 @@ "ip": "10.0.1.1" }, "client_to_server": { - "bytes": "720", "categories": "Web Ads/Analytics", "host": "px.moatads.com", "method": "GET", @@ -145,7 +142,6 @@ "supplier_country": "None" }, "server_to_client": { - "bytes": "1231", "filter_result": "DENIED", "status": "307" }, @@ -210,7 +206,6 @@ "ip": "10.0.1.1" }, "client_to_server": { - "bytes": "7791", "categories": "Web Ads/Analytics", "host": "t.txt.com", "method": "POST", @@ -232,7 +227,6 @@ "supplier_country": "None" }, "server_to_client": { - "bytes": "1167", "filter_result": "DENIED", "status": "307" }, @@ -300,7 +294,6 @@ "ip": "10.0.1.1" }, "client_to_server": { - "bytes": "3174", "categories": "Sports/Recreation", "host": "secure.espn.com", "method": "unknown", @@ -324,7 +317,6 @@ "supplier_name": "secure.espn.com" }, "server_to_client": { - "bytes": "42796", "filter_result": "OBSERVED", "status": "0" }, @@ -375,7 +367,6 @@ "ip": "10.0.1.1" }, "client_to_server": { - "bytes": "764", "categories": "Technology/Internet", "host": "edge.microsoft.com", "method": "GET", @@ -397,7 +388,6 @@ "supplier_country": "None" }, "server_to_client": { - "bytes": "1323", "filter_result": "DENIED", "status": "307" }, diff --git a/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-main.log-expected.json b/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-main.log-expected.json index e54c9df989e..fdbcf472a11 100644 --- a/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-main.log-expected.json +++ b/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-main.log-expected.json @@ -35,7 +35,6 @@ "ip": "10.82.255.36" }, "client_to_server": { - "bytes": "969", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "pixel.tapad.com", "method": "GET", @@ -53,7 +52,6 @@ "supplier_name": "pixel.tapad.com" }, "server_to_client": { - "bytes": "1242", "filter_result": "OBSERVED", "status": "302" }, @@ -135,7 +133,6 @@ "ip": "10.82.255.36" }, "client_to_server": { - "bytes": "1360", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "p.rfihub.com", "method": "GET", @@ -152,7 +149,6 @@ "supplier_name": "p.rfihub.com" }, "server_to_client": { - "bytes": "1158", "filter_result": "OBSERVED", "status": "302" }, @@ -220,7 +216,6 @@ "ip": "10.82.255.36" }, "client_to_server": { - "bytes": "356", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "token.rubiconproject.com", "method": "CONNECT", @@ -234,7 +229,6 @@ "ip": "81.2.69.144" }, "server_to_client": { - "bytes": "39", "filter_result": "OBSERVED", "status": "200" }, @@ -301,7 +295,6 @@ "ip": "10.82.255.36" }, "client_to_server": { - "bytes": "5784", "categories": "Web Ads/Analytics", "host": "rtb-csync.smartadserver.com", "method": "unknown", @@ -316,7 +309,6 @@ "supplier_name": "rtb-csync.smartadserver.com" }, "server_to_client": { - "bytes": "4897", "filter_result": "OBSERVED", "status": "0" }, @@ -383,7 +375,6 @@ "ip": "10.82.255.36" }, "client_to_server": { - "bytes": "334", "categories": "Technology/Internet", "host": "a.vidoomy.com", "method": "CONNECT", @@ -397,7 +388,6 @@ "ip": "81.2.69.144" }, "server_to_client": { - "bytes": "39", "filter_result": "OBSERVED", "status": "200" }, @@ -464,7 +454,6 @@ "ip": "10.82.255.36" }, "client_to_server": { - "bytes": "348", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "pixel.quantserve.com", "method": "CONNECT", @@ -478,7 +467,6 @@ "ip": "81.2.69.144" }, "server_to_client": { - "bytes": "39", "filter_result": "OBSERVED", "status": "200" }, @@ -546,7 +534,6 @@ "ip": "10.82.255.36" }, "client_to_server": { - "bytes": "809", "categories": "Web Ads/Analytics", "host": "x.bidswitch.net", "method": "GET", @@ -563,7 +550,6 @@ "supplier_name": "x.bidswitch.net" }, "server_to_client": { - "bytes": "298", "filter_result": "OBSERVED", "status": "302" }, @@ -632,7 +618,6 @@ "ip": "10.82.255.36" }, "client_to_server": { - "bytes": "893", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "token.rubiconproject.com", "method": "GET", @@ -652,7 +637,6 @@ "supplier_name": "token.rubiconproject.com" }, "server_to_client": { - "bytes": "947", "filter_result": "OBSERVED", "status": "200" }, @@ -721,7 +705,6 @@ "ip": "10.82.255.36" }, "client_to_server": { - "bytes": "546", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "rules.quantcount.com", "method": "GET", @@ -740,7 +723,6 @@ "supplier_name": "rules.quantcount.com" }, "server_to_client": { - "bytes": "4569", "filter_result": "OBSERVED", "status": "200" }, @@ -808,7 +790,6 @@ "ip": "10.82.255.36" }, "client_to_server": { - "bytes": "702", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "aax-eu.amazon-adsystem.com", "method": "GET", @@ -828,7 +809,6 @@ "supplier_name": "aax-eu.amazon-adsystem.com" }, "server_to_client": { - "bytes": "855", "filter_result": "OBSERVED", "status": "200" }, @@ -897,7 +877,6 @@ "ip": "10.82.255.36" }, "client_to_server": { - "bytes": "775", "categories": "Web Ads/Analytics", "host": "x.bidswitch.net", "method": "GET", @@ -915,7 +894,6 @@ "supplier_name": "x.bidswitch.net" }, "server_to_client": { - "bytes": "541", "filter_result": "OBSERVED", "status": "302" }, diff --git a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 9cef9613646..289b8cac491 100644 --- a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -103,6 +103,17 @@ processors: type: long ignore_missing: true + + - remove: + tag: proxysg_remove_original_server_to_client_bytes + field: proxysg.server_to_client.bytes + ignore_missing: true + + - remove: + tag: proxysg_remove_original_client_to_server_bytes + field: proxysg.client_to_server.bytes + ignore_missing: true + # ECS mappings - set: field: client.ip diff --git a/packages/proxysg/data_stream/log/fields/base-fields.yml b/packages/proxysg/data_stream/log/fields/base-fields.yml index f23dc472310..a6998f9b7ad 100644 --- a/packages/proxysg/data_stream/log/fields/base-fields.yml +++ b/packages/proxysg/data_stream/log/fields/base-fields.yml @@ -13,3 +13,9 @@ - name: log.source.address type: keyword description: Source address for the log. +- name: server.bytes + type: long + description: Count of bytes sent by the server. +- name: client.bytes + type: long + description: Count of bytes sent by the client. diff --git a/packages/proxysg/docs/README.md b/packages/proxysg/docs/README.md index df3d73fac9a..36f799b9c29 100644 --- a/packages/proxysg/docs/README.md +++ b/packages/proxysg/docs/README.md @@ -186,6 +186,7 @@ An example event for `log` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| client.bytes | Count of bytes sent by the client. | long | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | @@ -299,4 +300,5 @@ An example event for `log` looks as following: | proxysg.x_sc_connection_issuer_keyring | | keyword | | proxysg.x_sc_connection_issuer_keyring_alias | | keyword | | proxysg.x_virus_id | | keyword | +| server.bytes | Count of bytes sent by the server. | long | From 14a9d6f828c815f6dad4f3ad3c28d0413936773a Mon Sep 17 00:00:00 2001 From: Jonathan Molinatto Date: Wed, 29 Oct 2025 13:56:51 -0400 Subject: [PATCH 2/8] get in line --- ...t-proxy-bcreportermainv1.log-expected.json | 10 +++++++++ ...st-proxy-bcreportersslv1.log-expected.json | 10 +++++++++ .../test-proxy-main.log-expected.json | 22 +++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 11 ---------- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportermainv1.log-expected.json b/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportermainv1.log-expected.json index 5a20e20b2b0..3f483ed860d 100644 --- a/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportermainv1.log-expected.json +++ b/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportermainv1.log-expected.json @@ -31,6 +31,7 @@ "ip": "10.0.1.2" }, "client_to_server": { + "bytes": "111", "categories": "Web Infrastructure", "host": "www.msftconnecttest.com", "method": "GET", @@ -53,6 +54,7 @@ "supplier_country": "None" }, "server_to_client": { + "bytes": "1033", "filter_result": "DENIED", "status": "302" }, @@ -120,6 +122,7 @@ "ip": "10.0.1.2" }, "client_to_server": { + "bytes": "582", "categories": "Chat (IM)/SMS;Social Networking", "host": "tr6.snapchat.com", "method": "OPTIONS", @@ -136,6 +139,7 @@ "supplier_country": "None" }, "server_to_client": { + "bytes": "1297", "filter_result": "DENIED", "status": "401" }, @@ -207,6 +211,7 @@ "ip": "10.0.1.2" }, "client_to_server": { + "bytes": "291", "categories": "Web Infrastructure", "host": "ctldl.windowsupdate.com", "method": "GET", @@ -224,6 +229,7 @@ "supplier_country": "None" }, "server_to_client": { + "bytes": "1297", "filter_result": "DENIED", "status": "401" }, @@ -291,6 +297,7 @@ "ip": "10.0.1.2" }, "client_to_server": { + "bytes": "111", "categories": "Web Infrastructure", "host": "www.msftconnecttest.com", "method": "GET", @@ -307,6 +314,7 @@ "supplier_country": "None" }, "server_to_client": { + "bytes": "1033", "filter_result": "DENIED", "status": "302" }, @@ -373,6 +381,7 @@ "ip": "10.0.1.2" }, "client_to_server": { + "bytes": "510", "categories": "Web Ads/Analytics", "host": "dt.adsafeprotected.com", "method": "OPTIONS", @@ -390,6 +399,7 @@ "supplier_country": "None" }, "server_to_client": { + "bytes": "1316", "filter_result": "DENIED", "status": "401" }, diff --git a/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportersslv1.log-expected.json b/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportersslv1.log-expected.json index a75626e4b41..e4b442e1055 100644 --- a/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportersslv1.log-expected.json +++ b/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-bcreportersslv1.log-expected.json @@ -31,6 +31,7 @@ "ip": "10.0.1.1" }, "client_to_server": { + "bytes": "11343", "categories": "Web Ads/Analytics", "host": "server.googleapis.com", "method": "POST", @@ -52,6 +53,7 @@ "supplier_country": "None" }, "server_to_client": { + "bytes": "1039", "filter_result": "DENIED", "status": "307" }, @@ -119,6 +121,7 @@ "ip": "10.0.1.1" }, "client_to_server": { + "bytes": "720", "categories": "Web Ads/Analytics", "host": "px.moatads.com", "method": "GET", @@ -142,6 +145,7 @@ "supplier_country": "None" }, "server_to_client": { + "bytes": "1231", "filter_result": "DENIED", "status": "307" }, @@ -206,6 +210,7 @@ "ip": "10.0.1.1" }, "client_to_server": { + "bytes": "7791", "categories": "Web Ads/Analytics", "host": "t.txt.com", "method": "POST", @@ -227,6 +232,7 @@ "supplier_country": "None" }, "server_to_client": { + "bytes": "1167", "filter_result": "DENIED", "status": "307" }, @@ -294,6 +300,7 @@ "ip": "10.0.1.1" }, "client_to_server": { + "bytes": "3174", "categories": "Sports/Recreation", "host": "secure.espn.com", "method": "unknown", @@ -317,6 +324,7 @@ "supplier_name": "secure.espn.com" }, "server_to_client": { + "bytes": "42796", "filter_result": "OBSERVED", "status": "0" }, @@ -367,6 +375,7 @@ "ip": "10.0.1.1" }, "client_to_server": { + "bytes": "764", "categories": "Technology/Internet", "host": "edge.microsoft.com", "method": "GET", @@ -388,6 +397,7 @@ "supplier_country": "None" }, "server_to_client": { + "bytes": "1323", "filter_result": "DENIED", "status": "307" }, diff --git a/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-main.log-expected.json b/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-main.log-expected.json index fdbcf472a11..e54c9df989e 100644 --- a/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-main.log-expected.json +++ b/packages/proxysg/data_stream/log/_dev/test/pipeline/test-proxy-main.log-expected.json @@ -35,6 +35,7 @@ "ip": "10.82.255.36" }, "client_to_server": { + "bytes": "969", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "pixel.tapad.com", "method": "GET", @@ -52,6 +53,7 @@ "supplier_name": "pixel.tapad.com" }, "server_to_client": { + "bytes": "1242", "filter_result": "OBSERVED", "status": "302" }, @@ -133,6 +135,7 @@ "ip": "10.82.255.36" }, "client_to_server": { + "bytes": "1360", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "p.rfihub.com", "method": "GET", @@ -149,6 +152,7 @@ "supplier_name": "p.rfihub.com" }, "server_to_client": { + "bytes": "1158", "filter_result": "OBSERVED", "status": "302" }, @@ -216,6 +220,7 @@ "ip": "10.82.255.36" }, "client_to_server": { + "bytes": "356", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "token.rubiconproject.com", "method": "CONNECT", @@ -229,6 +234,7 @@ "ip": "81.2.69.144" }, "server_to_client": { + "bytes": "39", "filter_result": "OBSERVED", "status": "200" }, @@ -295,6 +301,7 @@ "ip": "10.82.255.36" }, "client_to_server": { + "bytes": "5784", "categories": "Web Ads/Analytics", "host": "rtb-csync.smartadserver.com", "method": "unknown", @@ -309,6 +316,7 @@ "supplier_name": "rtb-csync.smartadserver.com" }, "server_to_client": { + "bytes": "4897", "filter_result": "OBSERVED", "status": "0" }, @@ -375,6 +383,7 @@ "ip": "10.82.255.36" }, "client_to_server": { + "bytes": "334", "categories": "Technology/Internet", "host": "a.vidoomy.com", "method": "CONNECT", @@ -388,6 +397,7 @@ "ip": "81.2.69.144" }, "server_to_client": { + "bytes": "39", "filter_result": "OBSERVED", "status": "200" }, @@ -454,6 +464,7 @@ "ip": "10.82.255.36" }, "client_to_server": { + "bytes": "348", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "pixel.quantserve.com", "method": "CONNECT", @@ -467,6 +478,7 @@ "ip": "81.2.69.144" }, "server_to_client": { + "bytes": "39", "filter_result": "OBSERVED", "status": "200" }, @@ -534,6 +546,7 @@ "ip": "10.82.255.36" }, "client_to_server": { + "bytes": "809", "categories": "Web Ads/Analytics", "host": "x.bidswitch.net", "method": "GET", @@ -550,6 +563,7 @@ "supplier_name": "x.bidswitch.net" }, "server_to_client": { + "bytes": "298", "filter_result": "OBSERVED", "status": "302" }, @@ -618,6 +632,7 @@ "ip": "10.82.255.36" }, "client_to_server": { + "bytes": "893", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "token.rubiconproject.com", "method": "GET", @@ -637,6 +652,7 @@ "supplier_name": "token.rubiconproject.com" }, "server_to_client": { + "bytes": "947", "filter_result": "OBSERVED", "status": "200" }, @@ -705,6 +721,7 @@ "ip": "10.82.255.36" }, "client_to_server": { + "bytes": "546", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "rules.quantcount.com", "method": "GET", @@ -723,6 +740,7 @@ "supplier_name": "rules.quantcount.com" }, "server_to_client": { + "bytes": "4569", "filter_result": "OBSERVED", "status": "200" }, @@ -790,6 +808,7 @@ "ip": "10.82.255.36" }, "client_to_server": { + "bytes": "702", "categories": "FastwebRes_CallCntr;Web Ads/Analytics", "host": "aax-eu.amazon-adsystem.com", "method": "GET", @@ -809,6 +828,7 @@ "supplier_name": "aax-eu.amazon-adsystem.com" }, "server_to_client": { + "bytes": "855", "filter_result": "OBSERVED", "status": "200" }, @@ -877,6 +897,7 @@ "ip": "10.82.255.36" }, "client_to_server": { + "bytes": "775", "categories": "Web Ads/Analytics", "host": "x.bidswitch.net", "method": "GET", @@ -894,6 +915,7 @@ "supplier_name": "x.bidswitch.net" }, "server_to_client": { + "bytes": "541", "filter_result": "OBSERVED", "status": "302" }, diff --git a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/default.yml index cff62788a56..cd0e00a08c6 100644 --- a/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proxysg/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -107,17 +107,6 @@ processors: type: long ignore_missing: true - - - remove: - tag: proxysg_remove_original_server_to_client_bytes - field: proxysg.server_to_client.bytes - ignore_missing: true - - - remove: - tag: proxysg_remove_original_client_to_server_bytes - field: proxysg.client_to_server.bytes - ignore_missing: true - # ECS mappings - set: tag: set_client_ip_2339ed7a From a9310340cbd588dbe9bfe5e4bae8e652ad9c8e27 Mon Sep 17 00:00:00 2001 From: Jonathan Molinatto Date: Thu, 6 Nov 2025 12:51:57 -0500 Subject: [PATCH 3/8] created the knowledge base --- packages/cisco_ftd/changelog.yml | 5 + .../docs/knowledge_base/system_info.md | 214 ++++++++++++++++++ packages/cisco_ftd/manifest.yml | 2 +- 3 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 packages/cisco_ftd/docs/knowledge_base/system_info.md diff --git a/packages/cisco_ftd/changelog.yml b/packages/cisco_ftd/changelog.yml index 1e0e68252fb..9a1c95d6863 100644 --- a/packages/cisco_ftd/changelog.yml +++ b/packages/cisco_ftd/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.12.0" + changes: + - description: Update documentation to the new template. + type: enhancement + link: https://github.com/elastic/integrations/pull/1 - version: "3.11.0" changes: - description: Add support for Security Group Tag (SGT) and Endpoint Group (EPG) fields in connection events. diff --git a/packages/cisco_ftd/docs/knowledge_base/system_info.md b/packages/cisco_ftd/docs/knowledge_base/system_info.md new file mode 100644 index 00000000000..50111867b6c --- /dev/null +++ b/packages/cisco_ftd/docs/knowledge_base/system_info.md @@ -0,0 +1,214 @@ +# Service Info + +## Common use cases +- Network security monitoring and threat detection +- Firewall log analysis and compliance reporting +- Malware detection and file transfer monitoring +- VPN connection tracking and analysis +- Access control rule monitoring +- SSL/TLS inspection and policy enforcement +- URL filtering and web application identification +- DNS query monitoring +- Network flow analysis and connection tracking + +## Data types collected +- Security events (malware detection, file transfers, threat intelligence) +- Access control events (rule matches, connection allows/blocks) +- VPN events (connection establishment, termination, user authentication) +- SSL/TLS inspection events +- DNS query and response events +- Network flow information (source/destination IPs, ports, protocols) +- File transfer events (uploads, downloads, file analysis results) +- User authentication and authorization events (AAA) +- System events (failover, updates, configuration changes) + +## Compatibility +- Compatible with Cisco Firepower Threat Defense (FTD) devices +- Supports syslog message collection via TCP, UDP, or logfile input +- Tested with various Cisco Firepower device models and FTD software versions +- Requires Elastic Stack version ^8.11.0 || ^9.0.0 + +## Scaling and Performance +- Supports high-volume syslog ingestion via TCP and UDP inputs +- Can handle multiple concurrent connections when using TCP input +- UDP input provides low-latency log collection suitable for high-throughput environments +- Logfile input allows reading from local log files for batch processing or archival data +- Performance depends on network bandwidth, Elastic Agent resources, and Elasticsearch cluster capacity +- For high-volume deployments, consider using multiple Elastic Agents with load balancing + +# Set Up Instructions + +## Vendor prerequisites +- Cisco Firepower Threat Defense (FTD) device configured and operational +- Network connectivity between FTD device and Elastic Agent host +- Syslog logging enabled on the FTD device +- Appropriate firewall rules to allow syslog traffic from FTD to Elastic Agent (if applicable) +- Access to FTD management interface for configuration changes + +## Elastic prerequisites +- Elastic Stack version ^8.11.0 || ^9.0.0 +- Elastic Agent installed and configured +- Sufficient network bandwidth and system resources for log ingestion +- Appropriate Elasticsearch cluster capacity for expected log volume + +## Vendor set up steps + +### TCP Input Configuration +1. Log into the FTD management interface (Firepower Management Center or FDM) +2. Navigate to the device-specific configuration page +3. Search for or navigate to "FTD Logging" or "Configure Logging on FTD" page +4. Configure syslog server settings: + - Set the syslog server IP address to the Elastic Agent host IP + - Set the syslog port (default: 9003 for TCP) + - Select TCP as the transport protocol + - Enable the appropriate log types (security events, connection events, etc.) +5. Save and deploy the configuration to the FTD device +6. Verify syslog connectivity by checking for test messages + +### UDP Input Configuration +1. Log into the FTD management interface +2. Navigate to the device-specific configuration page +3. Search for or navigate to "FTD Logging" or "Configure Logging on FTD" page +4. Configure syslog server settings: + - Set the syslog server IP address to the Elastic Agent host IP + - Set the syslog port (default: 9003 for UDP) + - Select UDP as the transport protocol + - Enable the appropriate log types (security events, connection events, etc.) +5. Save and deploy the configuration to the FTD device +6. Verify syslog connectivity by checking for test messages + +### Logfile Input Configuration +1. Ensure FTD device is configured to write logs to a file system accessible by Elastic Agent +2. Identify the log file path(s) on the system (e.g., `/var/log/cisco-ftd.log`) +3. Ensure Elastic Agent has read permissions for the log file(s) +4. Configure log rotation if needed to prevent disk space issues + +Note: Cisco provides a range of Firepower devices, which may have different configuration steps. We recommend users navigate to the device specific configuration page, and search for/go to the "FTD Logging" or "Configure Logging on FTD" page for the specific device. + +## Kibana set up steps +1. Log into Kibana +2. Navigate to **Integrations** > **Browse integrations** +3. Search for "Cisco FTD" and select the integration +4. Click **Add Cisco FTD** +5. Configure the integration: + - **Name**: Provide a name for the integration instance + - **Data stream**: Select "Cisco FTD logs" + - **Input type**: Choose TCP, UDP, or Log file based on your configuration + - **TCP/UDP Configuration**: + - Set the host and port to match your Elastic Agent configuration + - Configure timezone offset if needed (default: UTC) + - Enable "Preserve original event" if you want to keep raw log messages + - Configure internal/external zones if needed for network direction detection + - **Logfile Configuration**: + - Specify the log file path(s) + - Configure timezone offset if needed + - Set internal/external zones if applicable +6. Click **Save and continue** +7. Add the integration to an agent policy or create a new agent policy +8. Deploy the agent policy to your Elastic Agents +9. Verify the agent is receiving data by checking the agent status in Kibana + +# Validation Steps +1. **Verify Agent Status**: + - In Kibana, navigate to **Management** > **Fleet** > **Agents** + - Confirm the Elastic Agent shows as "Healthy" and has the Cisco FTD integration assigned + - Check the agent logs for any connection errors + +2. **Trigger Test Events**: + - Generate test network traffic through the FTD device (e.g., web browsing, file download) + - Or trigger a security event by accessing a known malicious URL or downloading a test file + - Verify the FTD device is sending syslog messages (check FTD logs or management interface) + +3. **Verify Data Ingestion**: + - In Kibana, navigate to **Discover** + - Select the `logs-cisco_ftd.log-*` data stream + - Verify events are appearing with recent timestamps + - Check that events contain expected fields such as `cisco.ftd.*`, `source.ip`, `destination.ip`, etc. + +4. **Validate Event Fields**: + - Open a sample event and verify: + - `@timestamp` is correctly parsed + - `cisco.ftd.message_id` is present + - Network fields (`source.ip`, `destination.ip`, `source.port`, `destination.port`) are populated + - Security event fields are present for security-related events + - `event.original` contains the raw syslog message (if preserve_original_event is enabled) + +5. **Check for Parsing Errors**: + - Filter for `event.outcome: failure` or check for `error.message` fields + - Review any events with parsing issues + - Verify timezone configuration if timestamps appear incorrect + +# Troubleshooting + +## Common Configuration Issues + +**Issue**: No data appearing in Kibana Discover +- **Solution**: + - Verify Elastic Agent is running and healthy + - Check network connectivity between FTD device and Elastic Agent + - Verify syslog server configuration on FTD device matches Elastic Agent host/port + - Check firewall rules allow syslog traffic + - Review Elastic Agent logs for connection errors + - Verify the integration is properly assigned to the agent policy + +**Issue**: Service failed to start +- **Solution**: + - Check Elastic Agent logs for specific error messages + - Verify port is not already in use by another service + - Ensure Elastic Agent has necessary permissions (especially for logfile input) + - Check system resources (CPU, memory, disk space) + +**Issue**: Incorrect timezone in events +- **Solution**: + - Configure the `tz_offset` parameter in the integration settings + - Use IANA timezone format (e.g., "America/New_York") or offset format (e.g., "+0500") + - Verify FTD device timezone settings match your configuration + +**Issue**: Network direction not correctly identified +- **Solution**: + - Configure internal and external zones in the integration settings + - Ensure zone names match exactly with FTD device zone configuration + - Verify `private_is_internal` setting matches your network topology + - Check that `cisco.ftd.ingress_zone` and `cisco.ftd.egress_zone` fields are present in events + +## Ingestion Errors + +**Issue**: Events showing parsing errors or missing fields +- **Solution**: + - Check `event.original` field to see the raw syslog message + - Verify FTD device is sending logs in expected syslog format + - Review Elastic Agent logs for parsing error details + - Ensure FTD device software version is compatible with the integration + - Check if custom message formats require pipeline modifications + +**Issue**: `cisco.ftd.security` field contains flattened data but aggregations fail +- **Solution**: + - Starting from version 2.21.0, known security fields are moved to `cisco.ftd.security_event` for better aggregation support + - Use `cisco.ftd.security_event.*` fields for aggregations instead of `cisco.ftd.security.*` + - To add more fields to `cisco.ftd.security_event`, create a custom ingest pipeline: + 1. Navigate to **Stack Management** > **Ingest Pipelines** + 2. Create pipeline named `logs-cisco_ftd.log@custom` + 3. Add Rename processors to move fields from `cisco.ftd.security.*` to `cisco.ftd.security_event.*` + 4. Optionally add Convert processors to set correct data types + +**Issue**: Missing or incorrect field mappings +- **Solution**: + - Review the exported fields documentation in the integration README + - Verify FTD message IDs are supported by checking sample events + - Some fields may be vendor-specific and require custom mapping + - Check integration version changelog for recent field additions + +## API Authentication Errors +- **Not applicable**: This integration uses syslog protocol and does not require API authentication + +## Vendor Resources +- [Cisco Firepower Threat Defense Documentation](https://www.cisco.com/c/en/us/support/security/firepower-threat-defense/products-installation-and-configuration-guides-list.html) +- [Cisco FTD Logging Configuration Guide](https://www.cisco.com/c/en/us/support/security/firepower-threat-defense/products-installation-and-configuration-guides-list.html) +- [Cisco Firepower Management Center Documentation](https://www.cisco.com/c/en/us/support/security/firepower-management-center/products-installation-and-configuration-guides-list.html) + +# Documentation sites +- [Cisco Firepower Threat Defense Product Page](https://www.cisco.com/c/en/us/products/security/firepower-threat-defense/index.html) +- [Cisco Firepower Documentation](https://www.cisco.com/c/en/us/support/security/firepower-threat-defense/tsd-products-support-series-home.html) +- [Cisco FTD Syslog Configuration](https://www.cisco.com/c/en/us/support/security/firepower-threat-defense/products-installation-and-configuration-guides-list.html) +- [Elastic Integrations Documentation](https://www.elastic.co/guide/en/integrations/index.html) +- [Elastic Agent Documentation](https://www.elastic.co/guide/en/fleet/current/index.html) diff --git a/packages/cisco_ftd/manifest.yml b/packages/cisco_ftd/manifest.yml index 8a309a32d33..fbd0de9b679 100644 --- a/packages/cisco_ftd/manifest.yml +++ b/packages/cisco_ftd/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: cisco_ftd title: Cisco FTD -version: "3.11.0" +version: "3.12.0" description: Collect logs from Cisco FTD with Elastic Agent. type: integration categories: From 4a07d79b1786b0fe72988c6e38bc799068c2b545 Mon Sep 17 00:00:00 2001 From: Jonathan Molinatto Date: Thu, 6 Nov 2025 13:00:17 -0500 Subject: [PATCH 4/8] generated new documentation --- packages/cisco_ftd/_dev/build/docs/README.md | 100 ++++++++++++++++++- packages/cisco_ftd/docs/README.md | 100 ++++++++++++++++++- 2 files changed, 196 insertions(+), 4 deletions(-) diff --git a/packages/cisco_ftd/_dev/build/docs/README.md b/packages/cisco_ftd/_dev/build/docs/README.md index 86d44f65773..076fcecd8a4 100644 --- a/packages/cisco_ftd/_dev/build/docs/README.md +++ b/packages/cisco_ftd/_dev/build/docs/README.md @@ -1,6 +1,6 @@ # Cisco FTD Integration -This integration is for [Cisco](https://www.cisco.com/c/en/us/support/security/index.html) Firepower Threat Defence (FTD) device's logs. The package processes syslog messages from Cisco Firepower devices +This integration is for [Cisco](https://www.cisco.com/c/en/us/support/security/index.html) Firepower Threat Defence (FTD) device's logs. The package processes syslog messages from Cisco Firepower devices. It includes the following datasets for receiving logs over syslog or read from a file: @@ -10,9 +10,27 @@ It includes the following datasets for receiving logs over syslog or read from a Cisco provides a range of Firepower devices, which may have different configuration steps. We recommend users navigate to the device specific configuration page, and search for/go to the "FTD Logging" or "Configure Logging on FTD" page for the specific device. +### Input Types + +The integration supports three input types: + +1. **TCP Input**: Collects logs via TCP syslog. Configure the FTD device to send syslog messages to the Elastic Agent host on the specified TCP port (default: 9003). + +2. **UDP Input**: Collects logs via UDP syslog. Configure the FTD device to send syslog messages to the Elastic Agent host on the specified UDP port (default: 9003). UDP provides lower latency but less reliability than TCP. + +3. **Logfile Input**: Reads logs from local log files. Useful for batch processing or when syslog forwarding is not available. Specify the path to the log file(s) on the system. + +### Configuration Parameters + +- **Host and Port**: Configure the listening host (default: localhost) and port (default: 9003) for TCP/UDP inputs. +- **Timezone**: IANA timezone or offset (e.g., `+0200`) for interpreting syslog timestamps without timezone information (default: UTC). +- **Preserve Original Event**: When enabled, stores the raw syslog message in the `event.original` field. +- **Internal/External Zones**: Configure zone names to help determine network direction. Private CIDR ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) can be used as fallback. +- **Consider Private Networks as Internal**: When enabled, treats private CIDR ranges as internal networks for direction detection (default: true). + ## Handling security fields -Due to unknown amount of sub-fields present under the field `cisco.ftd.security`, it is mapped as [`flattened` datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html). This limited certain operations, such as aggregations, to be performed on sub-fields of `cisco.ftd.security`. See [flattened dataype limitations](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html#supported-operations) for more details. +Due to unknown amount of sub-fields present under the field `cisco.ftd.security`, it is mapped as [`flattened` datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html). This limited certain operations, such as aggregations, to be performed on sub-fields of `cisco.ftd.security`. See [flattened datatype limitations](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html#supported-operations) for more details. After analyzing more example logs, starting Cisco FTD integration version `2.21.0`, a new field `cisco.ftd.security_event` is added with a known set of fields moved over from `cisco.ftd.security`. With this, users can now perform aggregations on sub-fields of `cisco.ftd.security_event`. In addition to already moved fields, if users desire to add more fields onto `cisco.ftd.security_event` from `cisco.ftd.security`, they can make use of [`@custom` ingest pipeline](https://www.elastic.co/guide/en/elasticsearch/reference/current/ingest.html#pipelines-for-fleet-elastic-agent) that is automatically applied on every document at the end of the existing default pipeline. @@ -36,3 +54,81 @@ The `log` dataset collects the Cisco Firepower Threat Defense (FTD) logs. {{event "log"}} {{fields "log"}} + +## Use Cases + +- **Network Security Monitoring**: Monitor firewall events, access control rule matches, and security policy violations +- **Threat Detection**: Detect malware, botnets, and other security threats through file analysis and threat intelligence +- **Compliance Reporting**: Track network access, user authentication, and security events for compliance requirements +- **VPN Monitoring**: Monitor VPN connections, user authentication, and session management +- **SSL/TLS Inspection**: Track SSL/TLS inspection events and policy enforcement +- **URL Filtering**: Monitor web application usage, URL categories, and web filtering policies +- **DNS Monitoring**: Track DNS queries and responses for security analysis +- **Network Flow Analysis**: Analyze network connections, traffic patterns, and bandwidth usage + +## Event Types + +The integration processes various Cisco FTD event types including: + +- **Security Events**: Malware detection, file transfers, threat intelligence matches +- **Access Control Events**: Rule matches, connection allows/blocks, policy decisions +- **VPN Events**: Connection establishment, termination, user authentication (AAA) +- **SSL/TLS Events**: SSL inspection, certificate validation, policy enforcement +- **DNS Events**: DNS queries, responses, and filtering +- **System Events**: Failover, updates, configuration changes +- **File Events**: File uploads/downloads, file analysis results, sandbox status + +## Field Mappings + +The integration maps Cisco FTD syslog messages to Elastic Common Schema (ECS) fields: + +- Network fields: `source.ip`, `destination.ip`, `source.port`, `destination.port`, `network.protocol`, `network.transport` +- Event fields: `event.action`, `event.category`, `event.type`, `event.severity`, `event.code` +- File fields: `file.name`, `file.hash.sha256`, `file.size`, `file.type` +- URL fields: `url.original`, `url.domain`, `url.path`, `url.scheme` +- User fields: `user.name`, `source.user.name`, `destination.user.name` +- Observer fields: `observer.hostname`, `observer.product`, `observer.vendor`, `observer.type` + +Cisco-specific fields are prefixed with `cisco.ftd.*` and include: + +- `cisco.ftd.message_id`: The Cisco FTD message identifier +- `cisco.ftd.rule_name`: Access Control List rule name +- `cisco.ftd.security_event.*`: Structured security event fields +- `cisco.ftd.security.*`: Flattened security fields (for unknown/variable fields) +- `cisco.ftd.threat_category`: Threat category (virus, botnet, trojan, etc.) +- `cisco.ftd.threat_level`: Threat level (very-low, low, moderate, high, very-high) + +## Troubleshooting + +### No Data Appearing + +- Verify Elastic Agent is running and healthy +- Check network connectivity between FTD device and Elastic Agent +- Verify syslog server configuration on FTD device matches Elastic Agent host/port +- Check firewall rules allow syslog traffic +- Review Elastic Agent logs for connection errors + +### Parsing Errors + +- Check `event.original` field to see the raw syslog message +- Verify FTD device is sending logs in expected syslog format +- Review Elastic Agent logs for parsing error details +- Ensure FTD device software version is compatible with the integration + +### Incorrect Timestamps + +- Configure the `tz_offset` parameter in the integration settings +- Use IANA timezone format (e.g., "America/New_York") or offset format (e.g., "+0500") +- Verify FTD device timezone settings match your configuration + +### Network Direction Issues + +- Configure internal and external zones in the integration settings +- Ensure zone names match exactly with FTD device zone configuration +- Verify `private_is_internal` setting matches your network topology + +## Additional Resources + +- [Cisco Firepower Threat Defense Documentation](https://www.cisco.com/c/en/us/support/security/firepower-threat-defense/products-installation-and-configuration-guides-list.html) +- [Elastic Integrations Documentation](https://www.elastic.co/guide/en/integrations/index.html) +- [Elastic Agent Documentation](https://www.elastic.co/guide/en/fleet/current/index.html) diff --git a/packages/cisco_ftd/docs/README.md b/packages/cisco_ftd/docs/README.md index 8880037cd63..138b98bb871 100644 --- a/packages/cisco_ftd/docs/README.md +++ b/packages/cisco_ftd/docs/README.md @@ -1,6 +1,6 @@ # Cisco FTD Integration -This integration is for [Cisco](https://www.cisco.com/c/en/us/support/security/index.html) Firepower Threat Defence (FTD) device's logs. The package processes syslog messages from Cisco Firepower devices +This integration is for [Cisco](https://www.cisco.com/c/en/us/support/security/index.html) Firepower Threat Defence (FTD) device's logs. The package processes syslog messages from Cisco Firepower devices. It includes the following datasets for receiving logs over syslog or read from a file: @@ -10,9 +10,27 @@ It includes the following datasets for receiving logs over syslog or read from a Cisco provides a range of Firepower devices, which may have different configuration steps. We recommend users navigate to the device specific configuration page, and search for/go to the "FTD Logging" or "Configure Logging on FTD" page for the specific device. +### Input Types + +The integration supports three input types: + +1. **TCP Input**: Collects logs via TCP syslog. Configure the FTD device to send syslog messages to the Elastic Agent host on the specified TCP port (default: 9003). + +2. **UDP Input**: Collects logs via UDP syslog. Configure the FTD device to send syslog messages to the Elastic Agent host on the specified UDP port (default: 9003). UDP provides lower latency but less reliability than TCP. + +3. **Logfile Input**: Reads logs from local log files. Useful for batch processing or when syslog forwarding is not available. Specify the path to the log file(s) on the system. + +### Configuration Parameters + +- **Host and Port**: Configure the listening host (default: localhost) and port (default: 9003) for TCP/UDP inputs. +- **Timezone**: IANA timezone or offset (e.g., `+0200`) for interpreting syslog timestamps without timezone information (default: UTC). +- **Preserve Original Event**: When enabled, stores the raw syslog message in the `event.original` field. +- **Internal/External Zones**: Configure zone names to help determine network direction. Private CIDR ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) can be used as fallback. +- **Consider Private Networks as Internal**: When enabled, treats private CIDR ranges as internal networks for direction detection (default: true). + ## Handling security fields -Due to unknown amount of sub-fields present under the field `cisco.ftd.security`, it is mapped as [`flattened` datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html). This limited certain operations, such as aggregations, to be performed on sub-fields of `cisco.ftd.security`. See [flattened dataype limitations](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html#supported-operations) for more details. +Due to unknown amount of sub-fields present under the field `cisco.ftd.security`, it is mapped as [`flattened` datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html). This limited certain operations, such as aggregations, to be performed on sub-fields of `cisco.ftd.security`. See [flattened datatype limitations](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html#supported-operations) for more details. After analyzing more example logs, starting Cisco FTD integration version `2.21.0`, a new field `cisco.ftd.security_event` is added with a known set of fields moved over from `cisco.ftd.security`. With this, users can now perform aggregations on sub-fields of `cisco.ftd.security_event`. In addition to already moved fields, if users desire to add more fields onto `cisco.ftd.security_event` from `cisco.ftd.security`, they can make use of [`@custom` ingest pipeline](https://www.elastic.co/guide/en/elasticsearch/reference/current/ingest.html#pipelines-for-fleet-elastic-agent) that is automatically applied on every document at the end of the existing default pipeline. @@ -531,3 +549,81 @@ An example event for `log` looks as following: | user_agent.original | Unparsed user_agent string. | keyword | | user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | + +## Use Cases + +- **Network Security Monitoring**: Monitor firewall events, access control rule matches, and security policy violations +- **Threat Detection**: Detect malware, botnets, and other security threats through file analysis and threat intelligence +- **Compliance Reporting**: Track network access, user authentication, and security events for compliance requirements +- **VPN Monitoring**: Monitor VPN connections, user authentication, and session management +- **SSL/TLS Inspection**: Track SSL/TLS inspection events and policy enforcement +- **URL Filtering**: Monitor web application usage, URL categories, and web filtering policies +- **DNS Monitoring**: Track DNS queries and responses for security analysis +- **Network Flow Analysis**: Analyze network connections, traffic patterns, and bandwidth usage + +## Event Types + +The integration processes various Cisco FTD event types including: + +- **Security Events**: Malware detection, file transfers, threat intelligence matches +- **Access Control Events**: Rule matches, connection allows/blocks, policy decisions +- **VPN Events**: Connection establishment, termination, user authentication (AAA) +- **SSL/TLS Events**: SSL inspection, certificate validation, policy enforcement +- **DNS Events**: DNS queries, responses, and filtering +- **System Events**: Failover, updates, configuration changes +- **File Events**: File uploads/downloads, file analysis results, sandbox status + +## Field Mappings + +The integration maps Cisco FTD syslog messages to Elastic Common Schema (ECS) fields: + +- Network fields: `source.ip`, `destination.ip`, `source.port`, `destination.port`, `network.protocol`, `network.transport` +- Event fields: `event.action`, `event.category`, `event.type`, `event.severity`, `event.code` +- File fields: `file.name`, `file.hash.sha256`, `file.size`, `file.type` +- URL fields: `url.original`, `url.domain`, `url.path`, `url.scheme` +- User fields: `user.name`, `source.user.name`, `destination.user.name` +- Observer fields: `observer.hostname`, `observer.product`, `observer.vendor`, `observer.type` + +Cisco-specific fields are prefixed with `cisco.ftd.*` and include: + +- `cisco.ftd.message_id`: The Cisco FTD message identifier +- `cisco.ftd.rule_name`: Access Control List rule name +- `cisco.ftd.security_event.*`: Structured security event fields +- `cisco.ftd.security.*`: Flattened security fields (for unknown/variable fields) +- `cisco.ftd.threat_category`: Threat category (virus, botnet, trojan, etc.) +- `cisco.ftd.threat_level`: Threat level (very-low, low, moderate, high, very-high) + +## Troubleshooting + +### No Data Appearing + +- Verify Elastic Agent is running and healthy +- Check network connectivity between FTD device and Elastic Agent +- Verify syslog server configuration on FTD device matches Elastic Agent host/port +- Check firewall rules allow syslog traffic +- Review Elastic Agent logs for connection errors + +### Parsing Errors + +- Check `event.original` field to see the raw syslog message +- Verify FTD device is sending logs in expected syslog format +- Review Elastic Agent logs for parsing error details +- Ensure FTD device software version is compatible with the integration + +### Incorrect Timestamps + +- Configure the `tz_offset` parameter in the integration settings +- Use IANA timezone format (e.g., "America/New_York") or offset format (e.g., "+0500") +- Verify FTD device timezone settings match your configuration + +### Network Direction Issues + +- Configure internal and external zones in the integration settings +- Ensure zone names match exactly with FTD device zone configuration +- Verify `private_is_internal` setting matches your network topology + +## Additional Resources + +- [Cisco Firepower Threat Defense Documentation](https://www.cisco.com/c/en/us/support/security/firepower-threat-defense/products-installation-and-configuration-guides-list.html) +- [Elastic Integrations Documentation](https://www.elastic.co/guide/en/integrations/index.html) +- [Elastic Agent Documentation](https://www.elastic.co/guide/en/fleet/current/index.html) From 9772d943634ef262ca204ae292abdfc1d94da438 Mon Sep 17 00:00:00 2001 From: Jonathan Molinatto Date: Fri, 7 Nov 2025 09:50:22 -0500 Subject: [PATCH 5/8] generated by ep update documentation --- packages/cisco_ftd/_dev/build/docs/README.md | 160 ++++++++----------- 1 file changed, 66 insertions(+), 94 deletions(-) diff --git a/packages/cisco_ftd/_dev/build/docs/README.md b/packages/cisco_ftd/_dev/build/docs/README.md index 076fcecd8a4..d4687cc7bbe 100644 --- a/packages/cisco_ftd/_dev/build/docs/README.md +++ b/packages/cisco_ftd/_dev/build/docs/README.md @@ -1,134 +1,106 @@ -# Cisco FTD Integration +# Cisco FTD Integration for Elastic -This integration is for [Cisco](https://www.cisco.com/c/en/us/support/security/index.html) Firepower Threat Defence (FTD) device's logs. The package processes syslog messages from Cisco Firepower devices. +## Overview -It includes the following datasets for receiving logs over syslog or read from a file: +The Cisco Firepower Threat Defense (FTD) integration for Elastic collects logs from Cisco FTD devices, enabling comprehensive monitoring, threat detection, and security analysis within the Elastic Stack. This integration parses syslog messages from Cisco FTD, providing real-time visibility into network traffic, security events, and system activity. By centralizing these logs, organizations can enhance their security posture, streamline incident response, and gain deep insights into their network's operations. -- `log` dataset: supports Cisco Firepower Threat Defense (FTD) logs. +### Compatibility -## Configuration +This integration is compatible with Cisco FTD devices that support syslog export. It requires Elastic Stack version 8.11.0 or higher. -Cisco provides a range of Firepower devices, which may have different configuration steps. We recommend users navigate to the device specific configuration page, and search for/go to the "FTD Logging" or "Configure Logging on FTD" page for the specific device. +### How it works -### Input Types +The integration works by receiving syslog data sent from a Cisco FTD device. Elastic Agent can be configured to listen for these logs on a specific TCP or UDP port, or to read them directly from a log file. Once received, the agent processes and parses the logs before sending them to Elasticsearch. -The integration supports three input types: +## What data does this integration collect? -1. **TCP Input**: Collects logs via TCP syslog. Configure the FTD device to send syslog messages to the Elastic Agent host on the specified TCP port (default: 9003). +The Cisco FTD integration collects logs containing detailed information about: +* **Connection Events**: Firewall traffic, network address translation (NAT), and connection summaries. +* **Security Events**: Intrusion detection and prevention (IPS/IDS) alerts, file and malware protection events, and security intelligence data. +* **System Events**: Device health, system status, and configuration changes. -2. **UDP Input**: Collects logs via UDP syslog. Configure the FTD device to send syslog messages to the Elastic Agent host on the specified UDP port (default: 9003). UDP provides lower latency but less reliability than TCP. +### Supported use cases -3. **Logfile Input**: Reads logs from local log files. Useful for batch processing or when syslog forwarding is not available. Specify the path to the log file(s) on the system. +- **Real-time Threat Detection**: Use Elastic SIEM to identify and respond to threats like malware, intrusions, and policy violations. +- **Network Traffic Analysis**: Visualize and analyze network traffic patterns to identify anomalies, troubleshoot connectivity issues, and optimize performance. +- **Security Auditing and Compliance**: Maintain a searchable archive of all firewall activity to support compliance requirements and forensic investigations. +- **Operational Monitoring**: Track the health and status of your FTD devices to ensure they are functioning correctly. -### Configuration Parameters +## What do I need to use this integration? -- **Host and Port**: Configure the listening host (default: localhost) and port (default: 9003) for TCP/UDP inputs. -- **Timezone**: IANA timezone or offset (e.g., `+0200`) for interpreting syslog timestamps without timezone information (default: UTC). -- **Preserve Original Event**: When enabled, stores the raw syslog message in the `event.original` field. -- **Internal/External Zones**: Configure zone names to help determine network direction. Private CIDR ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) can be used as fallback. -- **Consider Private Networks as Internal**: When enabled, treats private CIDR ranges as internal networks for direction detection (default: true). +Elastic Agent must be installed on a host that is reachable by your Cisco FTD device over the network. For more details, check the Elastic Agent [installation instructions](docs-content://reference/fleet/install-elastic-agents.md). You can install only one Elastic Agent per host. -## Handling security fields +Elastic Agent is required to stream data from the syslog or log file receiver and ship the data to Elastic, where the events will then be processed via the integration's ingest pipelines. -Due to unknown amount of sub-fields present under the field `cisco.ftd.security`, it is mapped as [`flattened` datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html). This limited certain operations, such as aggregations, to be performed on sub-fields of `cisco.ftd.security`. See [flattened datatype limitations](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html#supported-operations) for more details. +## How do I deploy this integration? -After analyzing more example logs, starting Cisco FTD integration version `2.21.0`, a new field `cisco.ftd.security_event` is added with a known set of fields moved over from `cisco.ftd.security`. With this, users can now perform aggregations on sub-fields of `cisco.ftd.security_event`. In addition to already moved fields, if users desire to add more fields onto `cisco.ftd.security_event` from `cisco.ftd.security`, they can make use of [`@custom` ingest pipeline](https://www.elastic.co/guide/en/elasticsearch/reference/current/ingest.html#pipelines-for-fleet-elastic-agent) that is automatically applied on every document at the end of the existing default pipeline. +### Onboard / configure -To create and [add processors](https://www.elastic.co/guide/en/elasticsearch/reference/current/processors.html) to this `@custom` pipeline for Cisco FTD, users must follow below steps: -1. In Kibana, navigate to `Stack Management -> Ingest Pipelines`. -2. Click `Create Pipeline -> New Pipeline`. -3. Add `Name` as `logs-cisco_ftd.log@custom` and an optional `Description`. -4. Add processors to rename appropriate fields from `cisco.ftd.security` to `cisco.ftd.security_event`. - - Under `Processors`, click `Add a processor`. - - Say, you want to move field `threat_name` from `cisco.ftd.security` into `cisco.ftd.security_event`, then add a `Rename` processor with `Field` as `cisco.ftd.security.threat_name` and `Target field` as `cisco.ftd.security_event.threat_name`. - - Optionally add `Convert` processor to convert the datatype of the renamed field under `cisco.ftd.security_event`. +#### 1. Configure Cisco FTD to send Syslog Data -Now that the fields are available under `cisco.ftd.security_event`, users can perform aggregations of sub-fields under `cisco.ftd.security_event` as desired. +You must configure your Cisco FTD device to forward syslog messages to the Elastic Agent. The specific steps may vary depending on whether you are using Firepower Device Manager (FDM) or Firepower Management Center (FMC). -## Logs +1. **Define the Elastic Agent as a Syslog Server**: + * In your FDM or FMC interface, navigate to the syslog configuration section (e.g., **Objects > Syslog Servers** or **Device > System Settings > Logging**). + * Add a new syslog server, providing the IP address and port of the machine where the Elastic Agent is running. + * Ensure the protocol (TCP or UDP) matches the input you configure in the integration. -### FTD +2. **Configure Logging Rules**: + * Create or edit a logging rule to send specific event classes to the newly configured syslog server. + * It is recommended to send all relevant message IDs to ensure comprehensive data collection. -The `log` dataset collects the Cisco Firepower Threat Defense (FTD) logs. +3. **Deploy Changes**: + * Save and deploy your configuration changes to the FTD device. -{{event "log"}} +For detailed, step-by-step instructions, refer to the official Cisco documentation, such as [Configure Logging on FTD](https://www.cisco.com/c/en/us/td/docs/security/firepower/70/fdm/fptd-fdm-config-guide-700/fptd-fdm-logging.html). -{{fields "log"}} +#### 2. Add the Cisco FTD Integration in Elastic -## Use Cases +1. In Kibana, navigate to **Management > Integrations**. +2. In the search bar, enter **Cisco FTD**. +3. Click the integration to see more details and then click **Add integration**. +4. Configure the integration settings. You must select the input method that matches your Cisco FTD configuration (TCP, UDP, or log file). + * **For TCP/UDP**: Specify the `host` and `port` where the Elastic Agent should listen for syslog messages. This must match the destination you configured on your FTD device. + * **For Log File**: Provide the file `paths` that the agent should monitor. +5. Click **Save and continue** to add the integration policy to an Elastic Agent. -- **Network Security Monitoring**: Monitor firewall events, access control rule matches, and security policy violations -- **Threat Detection**: Detect malware, botnets, and other security threats through file analysis and threat intelligence -- **Compliance Reporting**: Track network access, user authentication, and security events for compliance requirements -- **VPN Monitoring**: Monitor VPN connections, user authentication, and session management -- **SSL/TLS Inspection**: Track SSL/TLS inspection events and policy enforcement -- **URL Filtering**: Monitor web application usage, URL categories, and web filtering policies -- **DNS Monitoring**: Track DNS queries and responses for security analysis -- **Network Flow Analysis**: Analyze network connections, traffic patterns, and bandwidth usage +### Validation -## Event Types +To validate that the integration is working, navigate to the **Discover** tab in Kibana. Filter for the `cisco_ftd.log` dataset (`data_stream.dataset : "cisco_ftd.log"`) and verify that logs from your FTD device are being ingested. You can also check the pre-built dashboards for this integration by searching for "Cisco FTD" in the **Dashboards** section. -The integration processes various Cisco FTD event types including: - -- **Security Events**: Malware detection, file transfers, threat intelligence matches -- **Access Control Events**: Rule matches, connection allows/blocks, policy decisions -- **VPN Events**: Connection establishment, termination, user authentication (AAA) -- **SSL/TLS Events**: SSL inspection, certificate validation, policy enforcement -- **DNS Events**: DNS queries, responses, and filtering -- **System Events**: Failover, updates, configuration changes -- **File Events**: File uploads/downloads, file analysis results, sandbox status - -## Field Mappings - -The integration maps Cisco FTD syslog messages to Elastic Common Schema (ECS) fields: +## Troubleshooting -- Network fields: `source.ip`, `destination.ip`, `source.port`, `destination.port`, `network.protocol`, `network.transport` -- Event fields: `event.action`, `event.category`, `event.type`, `event.severity`, `event.code` -- File fields: `file.name`, `file.hash.sha256`, `file.size`, `file.type` -- URL fields: `url.original`, `url.domain`, `url.path`, `url.scheme` -- User fields: `user.name`, `source.user.name`, `destination.user.name` -- Observer fields: `observer.hostname`, `observer.product`, `observer.vendor`, `observer.type` +For help with Elastic ingest tools, check [Common problems](https://www.elastic.co/docs/troubleshoot/ingest/fleet/common-problems). -Cisco-specific fields are prefixed with `cisco.ftd.*` and include: +### Handling `security` fields -- `cisco.ftd.message_id`: The Cisco FTD message identifier -- `cisco.ftd.rule_name`: Access Control List rule name -- `cisco.ftd.security_event.*`: Structured security event fields -- `cisco.ftd.security.*`: Flattened security fields (for unknown/variable fields) -- `cisco.ftd.threat_category`: Threat category (virus, botnet, trojan, etc.) -- `cisco.ftd.threat_level`: Threat level (very-low, low, moderate, high, very-high) +A field named `cisco.ftd.security` contains a variable number of sub-fields, which is mapped as a [`flattened` datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html). This mapping limits certain operations, such as aggregations, on its sub-fields. -## Troubleshooting +To enable aggregations on common security-related fields, the integration automatically moves a known set of fields from `cisco.ftd.security` to a new field, `cisco.ftd.security_event`. If you need to perform aggregations on additional fields within `cisco.ftd.security`, you can create a custom ingest pipeline to move them. -### No Data Appearing +To create this custom pipeline: +1. In Kibana, navigate to **Stack Management > Ingest Pipelines**. +2. Click **Create Pipeline > New Pipeline**. +3. Set the `Name` to `logs-cisco_ftd.log@custom`. +4. Add a **Rename** processor: + * Set `Field` to the source field, e.g., `cisco.ftd.security.threat_name`. + * Set `Target field` to the destination, e.g., `cisco.ftd.security_event.threat_name`. +5. Add more processors as needed and save the pipeline. This `@custom` pipeline will be automatically applied to all incoming Cisco FTD logs. -- Verify Elastic Agent is running and healthy -- Check network connectivity between FTD device and Elastic Agent -- Verify syslog server configuration on FTD device matches Elastic Agent host/port -- Check firewall rules allow syslog traffic -- Review Elastic Agent logs for connection errors +## Reference -### Parsing Errors +### log -- Check `event.original` field to see the raw syslog message -- Verify FTD device is sending logs in expected syslog format -- Review Elastic Agent logs for parsing error details -- Ensure FTD device software version is compatible with the integration +The `log` data stream collects logs from Cisco Firepower Threat Defense (FTD) devices. -### Incorrect Timestamps +#### log fields -- Configure the `tz_offset` parameter in the integration settings -- Use IANA timezone format (e.g., "America/New_York") or offset format (e.g., "+0500") -- Verify FTD device timezone settings match your configuration +{{ fields "log" }} -### Network Direction Issues +#### log sample event -- Configure internal and external zones in the integration settings -- Ensure zone names match exactly with FTD device zone configuration -- Verify `private_is_internal` setting matches your network topology +{{ event "log" }} -## Additional Resources -- [Cisco Firepower Threat Defense Documentation](https://www.cisco.com/c/en/us/support/security/firepower-threat-defense/products-installation-and-configuration-guides-list.html) -- [Elastic Integrations Documentation](https://www.elastic.co/guide/en/integrations/index.html) -- [Elastic Agent Documentation](https://www.elastic.co/guide/en/fleet/current/index.html) +### Inputs used +{{ inputDocs }} From 8c557c69565259f3c6597a0288a718308d4f50d9 Mon Sep 17 00:00:00 2001 From: Jonathan Molinatto Date: Fri, 7 Nov 2025 10:00:02 -0500 Subject: [PATCH 6/8] update the version in the manifest --- packages/cisco_ftd/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cisco_ftd/manifest.yml b/packages/cisco_ftd/manifest.yml index fbd0de9b679..a1c265aa879 100644 --- a/packages/cisco_ftd/manifest.yml +++ b/packages/cisco_ftd/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: cisco_ftd title: Cisco FTD -version: "3.12.0" +version: "3.13.0" description: Collect logs from Cisco FTD with Elastic Agent. type: integration categories: From a3bba24bfbee6fca3acf9801f47aa1bd8412c28d Mon Sep 17 00:00:00 2001 From: Jonathan Molinatto Date: Fri, 7 Nov 2025 10:43:31 -0500 Subject: [PATCH 7/8] rebuild the package to generate the publish docs --- packages/cisco_ftd/docs/README.md | 547 ++++++++++++++++-------------- 1 file changed, 298 insertions(+), 249 deletions(-) diff --git a/packages/cisco_ftd/docs/README.md b/packages/cisco_ftd/docs/README.md index 138b98bb871..e04fbc11afa 100644 --- a/packages/cisco_ftd/docs/README.md +++ b/packages/cisco_ftd/docs/README.md @@ -1,217 +1,99 @@ -# Cisco FTD Integration +# Cisco FTD Integration for Elastic -This integration is for [Cisco](https://www.cisco.com/c/en/us/support/security/index.html) Firepower Threat Defence (FTD) device's logs. The package processes syslog messages from Cisco Firepower devices. +## Overview -It includes the following datasets for receiving logs over syslog or read from a file: +The Cisco Firepower Threat Defense (FTD) integration for Elastic collects logs from Cisco FTD devices, enabling comprehensive monitoring, threat detection, and security analysis within the Elastic Stack. This integration parses syslog messages from Cisco FTD, providing real-time visibility into network traffic, security events, and system activity. By centralizing these logs, organizations can enhance their security posture, streamline incident response, and gain deep insights into their network's operations. -- `log` dataset: supports Cisco Firepower Threat Defense (FTD) logs. +### Compatibility -## Configuration +This integration is compatible with Cisco FTD devices that support syslog export. It requires Elastic Stack version 8.11.0 or higher. -Cisco provides a range of Firepower devices, which may have different configuration steps. We recommend users navigate to the device specific configuration page, and search for/go to the "FTD Logging" or "Configure Logging on FTD" page for the specific device. +### How it works -### Input Types +The integration works by receiving syslog data sent from a Cisco FTD device. Elastic Agent can be configured to listen for these logs on a specific TCP or UDP port, or to read them directly from a log file. Once received, the agent processes and parses the logs before sending them to Elasticsearch. -The integration supports three input types: +## What data does this integration collect? -1. **TCP Input**: Collects logs via TCP syslog. Configure the FTD device to send syslog messages to the Elastic Agent host on the specified TCP port (default: 9003). +The Cisco FTD integration collects logs containing detailed information about: +* **Connection Events**: Firewall traffic, network address translation (NAT), and connection summaries. +* **Security Events**: Intrusion detection and prevention (IPS/IDS) alerts, file and malware protection events, and security intelligence data. +* **System Events**: Device health, system status, and configuration changes. -2. **UDP Input**: Collects logs via UDP syslog. Configure the FTD device to send syslog messages to the Elastic Agent host on the specified UDP port (default: 9003). UDP provides lower latency but less reliability than TCP. +### Supported use cases -3. **Logfile Input**: Reads logs from local log files. Useful for batch processing or when syslog forwarding is not available. Specify the path to the log file(s) on the system. +- **Real-time Threat Detection**: Use Elastic SIEM to identify and respond to threats like malware, intrusions, and policy violations. +- **Network Traffic Analysis**: Visualize and analyze network traffic patterns to identify anomalies, troubleshoot connectivity issues, and optimize performance. +- **Security Auditing and Compliance**: Maintain a searchable archive of all firewall activity to support compliance requirements and forensic investigations. +- **Operational Monitoring**: Track the health and status of your FTD devices to ensure they are functioning correctly. -### Configuration Parameters +## What do I need to use this integration? -- **Host and Port**: Configure the listening host (default: localhost) and port (default: 9003) for TCP/UDP inputs. -- **Timezone**: IANA timezone or offset (e.g., `+0200`) for interpreting syslog timestamps without timezone information (default: UTC). -- **Preserve Original Event**: When enabled, stores the raw syslog message in the `event.original` field. -- **Internal/External Zones**: Configure zone names to help determine network direction. Private CIDR ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) can be used as fallback. -- **Consider Private Networks as Internal**: When enabled, treats private CIDR ranges as internal networks for direction detection (default: true). +Elastic Agent must be installed on a host that is reachable by your Cisco FTD device over the network. For more details, check the Elastic Agent [installation instructions](docs-content://reference/fleet/install-elastic-agents.md). You can install only one Elastic Agent per host. -## Handling security fields +Elastic Agent is required to stream data from the syslog or log file receiver and ship the data to Elastic, where the events will then be processed via the integration's ingest pipelines. -Due to unknown amount of sub-fields present under the field `cisco.ftd.security`, it is mapped as [`flattened` datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html). This limited certain operations, such as aggregations, to be performed on sub-fields of `cisco.ftd.security`. See [flattened datatype limitations](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html#supported-operations) for more details. +## How do I deploy this integration? -After analyzing more example logs, starting Cisco FTD integration version `2.21.0`, a new field `cisco.ftd.security_event` is added with a known set of fields moved over from `cisco.ftd.security`. With this, users can now perform aggregations on sub-fields of `cisco.ftd.security_event`. In addition to already moved fields, if users desire to add more fields onto `cisco.ftd.security_event` from `cisco.ftd.security`, they can make use of [`@custom` ingest pipeline](https://www.elastic.co/guide/en/elasticsearch/reference/current/ingest.html#pipelines-for-fleet-elastic-agent) that is automatically applied on every document at the end of the existing default pipeline. +### Onboard / configure -To create and [add processors](https://www.elastic.co/guide/en/elasticsearch/reference/current/processors.html) to this `@custom` pipeline for Cisco FTD, users must follow below steps: -1. In Kibana, navigate to `Stack Management -> Ingest Pipelines`. -2. Click `Create Pipeline -> New Pipeline`. -3. Add `Name` as `logs-cisco_ftd.log@custom` and an optional `Description`. -4. Add processors to rename appropriate fields from `cisco.ftd.security` to `cisco.ftd.security_event`. - - Under `Processors`, click `Add a processor`. - - Say, you want to move field `threat_name` from `cisco.ftd.security` into `cisco.ftd.security_event`, then add a `Rename` processor with `Field` as `cisco.ftd.security.threat_name` and `Target field` as `cisco.ftd.security_event.threat_name`. - - Optionally add `Convert` processor to convert the datatype of the renamed field under `cisco.ftd.security_event`. +#### 1. Configure Cisco FTD to send Syslog Data -Now that the fields are available under `cisco.ftd.security_event`, users can perform aggregations of sub-fields under `cisco.ftd.security_event` as desired. +You must configure your Cisco FTD device to forward syslog messages to the Elastic Agent. The specific steps may vary depending on whether you are using Firepower Device Manager (FDM) or Firepower Management Center (FMC). -## Logs +1. **Define the Elastic Agent as a Syslog Server**: + * In your FDM or FMC interface, navigate to the syslog configuration section (e.g., **Objects > Syslog Servers** or **Device > System Settings > Logging**). + * Add a new syslog server, providing the IP address and port of the machine where the Elastic Agent is running. + * Ensure the protocol (TCP or UDP) matches the input you configure in the integration. -### FTD +2. **Configure Logging Rules**: + * Create or edit a logging rule to send specific event classes to the newly configured syslog server. + * It is recommended to send all relevant message IDs to ensure comprehensive data collection. -The `log` dataset collects the Cisco Firepower Threat Defense (FTD) logs. +3. **Deploy Changes**: + * Save and deploy your configuration changes to the FTD device. -An example event for `log` looks as following: +For detailed, step-by-step instructions, refer to the official Cisco documentation, such as [Configure Logging on FTD](https://www.cisco.com/c/en/us/td/docs/security/firepower/70/fdm/fptd-fdm-config-guide-700/fptd-fdm-logging.html). -```json -{ - "@timestamp": "2019-08-16T09:39:02.000Z", - "agent": { - "ephemeral_id": "477973c4-b380-4791-ad68-919bc71782eb", - "id": "dc57df32-adfa-4d1e-9386-d4519fc2d1e9", - "name": "elastic-agent-79310", - "type": "filebeat", - "version": "8.18.0" - }, - "cisco": { - "ftd": { - "rule_name": "malware-and-file-policy", - "security": { - "file_storage_status": "Not Stored (Disposition Was Pending)", - "threat_name": "Win.Ransomware.Eicar::95.sbx.tg" - }, - "security_event": { - "application_protocol": "HTTP", - "client": "cURL", - "dst_ip": "81.2.69.144", - "dst_port": 80, - "file_action": "Malware Cloud Lookup", - "file_direction": "Download", - "file_name": "eicar_com.zip", - "file_policy": "malware-and-file-policy", - "file_sandbox_status": "File Size Is Too Small", - "file_sha256": "2546dcffc5ad854d4ddc64fbf056871cd5a00f2471cb7a5bfd4ac23b6e9eedad", - "file_size": 184, - "file_type": "ZIP", - "first_packet_second": "2019-08-16T09:39:02Z", - "protocol": "tcp", - "sha_disposition": "Unavailable", - "spero_disposition": "Spero detection not performed on file", - "src_ip": "10.0.1.20", - "src_port": 46004, - "uri": "http://www.eicar.org/download/eicar_com.zip", - "user": "No Authentication Required" - }, - "threat_category": "Win.Ransomware.Eicar::95.sbx.tg" - } - }, - "data_stream": { - "dataset": "cisco_ftd.log", - "namespace": "84072", - "type": "logs" - }, - "destination": { - "address": "81.2.69.144", - "geo": { - "city_name": "London", - "continent_name": "Europe", - "country_iso_code": "GB", - "country_name": "United Kingdom", - "location": { - "lat": 51.5142, - "lon": -0.0931 - }, - "region_iso_code": "GB-ENG", - "region_name": "England" - }, - "ip": "81.2.69.144", - "port": 80 - }, - "ecs": { - "version": "8.17.0" - }, - "elastic_agent": { - "id": "dc57df32-adfa-4d1e-9386-d4519fc2d1e9", - "snapshot": false, - "version": "8.18.0" - }, - "event": { - "action": "malware-detected", - "agent_id_status": "verified", - "category": [ - "malware", - "file" - ], - "code": "430005", - "dataset": "cisco_ftd.log", - "ingested": "2025-05-20T10:32:02Z", - "kind": "event", - "original": "2019-08-16T09:39:03Z firepower %FTD-1-430005: SrcIP: 10.0.1.20, DstIP: 81.2.69.144, SrcPort: 46004, DstPort: 80, Protocol: tcp, FileDirection: Download, FileAction: Malware Cloud Lookup, FileSHA256: 2546dcffc5ad854d4ddc64fbf056871cd5a00f2471cb7a5bfd4ac23b6e9eedad, SHA_Disposition: Unavailable, SperoDisposition: Spero detection not performed on file, ThreatName: Win.Ransomware.Eicar::95.sbx.tg, FileName: eicar_com.zip, FileType: ZIP, FileSize: 184, ApplicationProtocol: HTTP, Client: cURL, User: No Authentication Required, FirstPacketSecond: 2019-08-16T09:39:02Z, FilePolicy: malware-and-file-policy, FileStorageStatus: Not Stored (Disposition Was Pending), FileSandboxStatus: File Size Is Too Small, URI: http://www.eicar.org/download/eicar_com.zip", - "severity": 1, - "start": "2019-08-16T09:39:02Z", - "timezone": "UTC", - "type": [ - "info" - ] - }, - "file": { - "hash": { - "sha256": "2546dcffc5ad854d4ddc64fbf056871cd5a00f2471cb7a5bfd4ac23b6e9eedad" - }, - "name": "eicar_com.zip", - "size": 184 - }, - "host": { - "hostname": "firepower" - }, - "input": { - "type": "udp" - }, - "log": { - "level": "alert", - "source": { - "address": "192.168.249.3:51934" - } - }, - "network": { - "application": "curl", - "community_id": "1:jk2uwniJ2oCG0t73HeZ9w8gtA8E=", - "direction": "outbound", - "iana_number": "6", - "protocol": "http", - "transport": "tcp" - }, - "observer": { - "hostname": "firepower", - "product": "ftd", - "type": "idps", - "vendor": "Cisco" - }, - "related": { - "hash": [ - "2546dcffc5ad854d4ddc64fbf056871cd5a00f2471cb7a5bfd4ac23b6e9eedad" - ], - "hosts": [ - "firepower" - ], - "ip": [ - "10.0.1.20", - "81.2.69.144" - ] - }, - "rule": { - "ruleset": "malware-and-file-policy" - }, - "source": { - "address": "10.0.1.20", - "ip": "10.0.1.20", - "port": 46004 - }, - "tags": [ - "preserve_original_event", - "private_is_internal", - "cisco-ftd", - "forwarded" - ], - "url": { - "domain": "www.eicar.org", - "extension": "zip", - "original": "http://www.eicar.org/download/eicar_com.zip", - "path": "/download/eicar_com.zip", - "scheme": "http" - } -} -``` +#### 2. Add the Cisco FTD Integration in Elastic + +1. In Kibana, navigate to **Management > Integrations**. +2. In the search bar, enter **Cisco FTD**. +3. Click the integration to see more details and then click **Add integration**. +4. Configure the integration settings. You must select the input method that matches your Cisco FTD configuration (TCP, UDP, or log file). + * **For TCP/UDP**: Specify the `host` and `port` where the Elastic Agent should listen for syslog messages. This must match the destination you configured on your FTD device. + * **For Log File**: Provide the file `paths` that the agent should monitor. +5. Click **Save and continue** to add the integration policy to an Elastic Agent. + +### Validation + +To validate that the integration is working, navigate to the **Discover** tab in Kibana. Filter for the `cisco_ftd.log` dataset (`data_stream.dataset : "cisco_ftd.log"`) and verify that logs from your FTD device are being ingested. You can also check the pre-built dashboards for this integration by searching for "Cisco FTD" in the **Dashboards** section. + +## Troubleshooting + +For help with Elastic ingest tools, check [Common problems](https://www.elastic.co/docs/troubleshoot/ingest/fleet/common-problems). + +### Handling `security` fields + +A field named `cisco.ftd.security` contains a variable number of sub-fields, which is mapped as a [`flattened` datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html). This mapping limits certain operations, such as aggregations, on its sub-fields. + +To enable aggregations on common security-related fields, the integration automatically moves a known set of fields from `cisco.ftd.security` to a new field, `cisco.ftd.security_event`. If you need to perform aggregations on additional fields within `cisco.ftd.security`, you can create a custom ingest pipeline to move them. + +To create this custom pipeline: +1. In Kibana, navigate to **Stack Management > Ingest Pipelines**. +2. Click **Create Pipeline > New Pipeline**. +3. Set the `Name` to `logs-cisco_ftd.log@custom`. +4. Add a **Rename** processor: + * Set `Field` to the source field, e.g., `cisco.ftd.security.threat_name`. + * Set `Target field` to the destination, e.g., `cisco.ftd.security_event.threat_name`. +5. Add more processors as needed and save the pipeline. This `@custom` pipeline will be automatically applied to all incoming Cisco FTD logs. + +## Reference + +### log + +The `log` data stream collects logs from Cisco Firepower Threat Defense (FTD) devices. + +#### log fields **Exported fields** @@ -550,80 +432,247 @@ An example event for `log` looks as following: | user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -## Use Cases +#### log sample event + +An example event for `log` looks as following: + +```json +{ + "@timestamp": "2019-08-16T09:39:02.000Z", + "agent": { + "ephemeral_id": "477973c4-b380-4791-ad68-919bc71782eb", + "id": "dc57df32-adfa-4d1e-9386-d4519fc2d1e9", + "name": "elastic-agent-79310", + "type": "filebeat", + "version": "8.18.0" + }, + "cisco": { + "ftd": { + "rule_name": "malware-and-file-policy", + "security": { + "file_storage_status": "Not Stored (Disposition Was Pending)", + "threat_name": "Win.Ransomware.Eicar::95.sbx.tg" + }, + "security_event": { + "application_protocol": "HTTP", + "client": "cURL", + "dst_ip": "81.2.69.144", + "dst_port": 80, + "file_action": "Malware Cloud Lookup", + "file_direction": "Download", + "file_name": "eicar_com.zip", + "file_policy": "malware-and-file-policy", + "file_sandbox_status": "File Size Is Too Small", + "file_sha256": "2546dcffc5ad854d4ddc64fbf056871cd5a00f2471cb7a5bfd4ac23b6e9eedad", + "file_size": 184, + "file_type": "ZIP", + "first_packet_second": "2019-08-16T09:39:02Z", + "protocol": "tcp", + "sha_disposition": "Unavailable", + "spero_disposition": "Spero detection not performed on file", + "src_ip": "10.0.1.20", + "src_port": 46004, + "uri": "http://www.eicar.org/download/eicar_com.zip", + "user": "No Authentication Required" + }, + "threat_category": "Win.Ransomware.Eicar::95.sbx.tg" + } + }, + "data_stream": { + "dataset": "cisco_ftd.log", + "namespace": "84072", + "type": "logs" + }, + "destination": { + "address": "81.2.69.144", + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144", + "port": 80 + }, + "ecs": { + "version": "8.17.0" + }, + "elastic_agent": { + "id": "dc57df32-adfa-4d1e-9386-d4519fc2d1e9", + "snapshot": false, + "version": "8.18.0" + }, + "event": { + "action": "malware-detected", + "agent_id_status": "verified", + "category": [ + "malware", + "file" + ], + "code": "430005", + "dataset": "cisco_ftd.log", + "ingested": "2025-05-20T10:32:02Z", + "kind": "event", + "original": "2019-08-16T09:39:03Z firepower %FTD-1-430005: SrcIP: 10.0.1.20, DstIP: 81.2.69.144, SrcPort: 46004, DstPort: 80, Protocol: tcp, FileDirection: Download, FileAction: Malware Cloud Lookup, FileSHA256: 2546dcffc5ad854d4ddc64fbf056871cd5a00f2471cb7a5bfd4ac23b6e9eedad, SHA_Disposition: Unavailable, SperoDisposition: Spero detection not performed on file, ThreatName: Win.Ransomware.Eicar::95.sbx.tg, FileName: eicar_com.zip, FileType: ZIP, FileSize: 184, ApplicationProtocol: HTTP, Client: cURL, User: No Authentication Required, FirstPacketSecond: 2019-08-16T09:39:02Z, FilePolicy: malware-and-file-policy, FileStorageStatus: Not Stored (Disposition Was Pending), FileSandboxStatus: File Size Is Too Small, URI: http://www.eicar.org/download/eicar_com.zip", + "severity": 1, + "start": "2019-08-16T09:39:02Z", + "timezone": "UTC", + "type": [ + "info" + ] + }, + "file": { + "hash": { + "sha256": "2546dcffc5ad854d4ddc64fbf056871cd5a00f2471cb7a5bfd4ac23b6e9eedad" + }, + "name": "eicar_com.zip", + "size": 184 + }, + "host": { + "hostname": "firepower" + }, + "input": { + "type": "udp" + }, + "log": { + "level": "alert", + "source": { + "address": "192.168.249.3:51934" + } + }, + "network": { + "application": "curl", + "community_id": "1:jk2uwniJ2oCG0t73HeZ9w8gtA8E=", + "direction": "outbound", + "iana_number": "6", + "protocol": "http", + "transport": "tcp" + }, + "observer": { + "hostname": "firepower", + "product": "ftd", + "type": "idps", + "vendor": "Cisco" + }, + "related": { + "hash": [ + "2546dcffc5ad854d4ddc64fbf056871cd5a00f2471cb7a5bfd4ac23b6e9eedad" + ], + "hosts": [ + "firepower" + ], + "ip": [ + "10.0.1.20", + "81.2.69.144" + ] + }, + "rule": { + "ruleset": "malware-and-file-policy" + }, + "source": { + "address": "10.0.1.20", + "ip": "10.0.1.20", + "port": 46004 + }, + "tags": [ + "preserve_original_event", + "private_is_internal", + "cisco-ftd", + "forwarded" + ], + "url": { + "domain": "www.eicar.org", + "extension": "zip", + "original": "http://www.eicar.org/download/eicar_com.zip", + "path": "/download/eicar_com.zip", + "scheme": "http" + } +} +``` + + +### Inputs used +These inputs can be used with this integration: +
+logfile -- **Network Security Monitoring**: Monitor firewall events, access control rule matches, and security policy violations -- **Threat Detection**: Detect malware, botnets, and other security threats through file analysis and threat intelligence -- **Compliance Reporting**: Track network access, user authentication, and security events for compliance requirements -- **VPN Monitoring**: Monitor VPN connections, user authentication, and session management -- **SSL/TLS Inspection**: Track SSL/TLS inspection events and policy enforcement -- **URL Filtering**: Monitor web application usage, URL categories, and web filtering policies -- **DNS Monitoring**: Track DNS queries and responses for security analysis -- **Network Flow Analysis**: Analyze network connections, traffic patterns, and bandwidth usage +## Setup +For more details about the logfile input settings, check the [Filebeat documentation](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-log). -## Event Types +### Collecting logs from logfile -The integration processes various Cisco FTD event types including: +To collect logs via logfile, select **Collect logs via the logfile input** and configure the following parameter: -- **Security Events**: Malware detection, file transfers, threat intelligence matches -- **Access Control Events**: Rule matches, connection allows/blocks, policy decisions -- **VPN Events**: Connection establishment, termination, user authentication (AAA) -- **SSL/TLS Events**: SSL inspection, certificate validation, policy enforcement -- **DNS Events**: DNS queries, responses, and filtering -- **System Events**: Failover, updates, configuration changes -- **File Events**: File uploads/downloads, file analysis results, sandbox status +- Paths: List of glob-based paths to crawl and fetch log files from. Supports glob patterns like + `/var/log/*.log` or `/var/log/*/*.log` for subfolder matching. Each file found starts a + separate harvester. +
+
+tcp -## Field Mappings +## Setup -The integration maps Cisco FTD syslog messages to Elastic Common Schema (ECS) fields: +For more details about the TCP input settings, check the [Filebeat documentation](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-tcp). -- Network fields: `source.ip`, `destination.ip`, `source.port`, `destination.port`, `network.protocol`, `network.transport` -- Event fields: `event.action`, `event.category`, `event.type`, `event.severity`, `event.code` -- File fields: `file.name`, `file.hash.sha256`, `file.size`, `file.type` -- URL fields: `url.original`, `url.domain`, `url.path`, `url.scheme` -- User fields: `user.name`, `source.user.name`, `destination.user.name` -- Observer fields: `observer.hostname`, `observer.product`, `observer.vendor`, `observer.type` +### Collecting logs from TCP -Cisco-specific fields are prefixed with `cisco.ftd.*` and include: +To collect logs via TCP, select **Collect logs via TCP** and configure the following parameters: -- `cisco.ftd.message_id`: The Cisco FTD message identifier -- `cisco.ftd.rule_name`: Access Control List rule name -- `cisco.ftd.security_event.*`: Structured security event fields -- `cisco.ftd.security.*`: Flattened security fields (for unknown/variable fields) -- `cisco.ftd.threat_category`: Threat category (virus, botnet, trojan, etc.) -- `cisco.ftd.threat_level`: Threat level (very-low, low, moderate, high, very-high) +**Required Settings:** +- Host +- Port -## Troubleshooting +**Common Optional Settings:** +- Max Message Size - Maximum size of incoming messages +- Max Connections - Maximum number of concurrent connections +- Timeout - How long to wait for data before closing idle connections +- Line Delimiter - Character(s) that separate log messages -### No Data Appearing +## SSL/TLS Configuration -- Verify Elastic Agent is running and healthy -- Check network connectivity between FTD device and Elastic Agent -- Verify syslog server configuration on FTD device matches Elastic Agent host/port -- Check firewall rules allow syslog traffic -- Review Elastic Agent logs for connection errors +To enable encrypted connections, configure the following SSL settings: -### Parsing Errors +**SSL Settings:** +- Enable SSL*- Toggle to enable SSL/TLS encryption +- Certificate - Path to the SSL certificate file (`.crt` or `.pem`) +- Certificate Key - Path to the private key file (`.key`) +- Certificate Authorities - Path to CA certificate file for client certificate validation (optional) +- Client Authentication - Require client certificates (`none`, `optional`, or `required`) +- Supported Protocols - TLS versions to support (e.g., `TLSv1.2`, `TLSv1.3`) + +**Example SSL Configuration:** +```yaml +ssl.enabled: true +ssl.certificate: "/path/to/server.crt" +ssl.key: "/path/to/server.key" +ssl.certificate_authorities: ["/path/to/ca.crt"] +ssl.client_authentication: "optional" +``` +
+
+udp -- Check `event.original` field to see the raw syslog message -- Verify FTD device is sending logs in expected syslog format -- Review Elastic Agent logs for parsing error details -- Ensure FTD device software version is compatible with the integration +## Setup -### Incorrect Timestamps +For more details about the UDP input settings, check the [Filebeat documentation](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-udp). -- Configure the `tz_offset` parameter in the integration settings -- Use IANA timezone format (e.g., "America/New_York") or offset format (e.g., "+0500") -- Verify FTD device timezone settings match your configuration +### Collecting logs from UDP -### Network Direction Issues +To collect logs via UDP, select **Collect logs via UDP** and configure the following parameters: -- Configure internal and external zones in the integration settings -- Ensure zone names match exactly with FTD device zone configuration -- Verify `private_is_internal` setting matches your network topology +**Required Settings:** +- Host +- Port -## Additional Resources +**Common Optional Settings:** +- Max Message Size - Maximum size of UDP packets to accept (default: 10KB, max: 64KB) +- Read Buffer - UDP socket read buffer size for handling bursts of messages +- Read Timeout - How long to wait for incoming packets before checking for shutdown +
-- [Cisco Firepower Threat Defense Documentation](https://www.cisco.com/c/en/us/support/security/firepower-threat-defense/products-installation-and-configuration-guides-list.html) -- [Elastic Integrations Documentation](https://www.elastic.co/guide/en/integrations/index.html) -- [Elastic Agent Documentation](https://www.elastic.co/guide/en/fleet/current/index.html) From fd4d86ef9e5ed0d454a9a91643fff43f3f3d621a Mon Sep 17 00:00:00 2001 From: Jonathan Molinatto Date: Mon, 10 Nov 2025 11:30:53 -0500 Subject: [PATCH 8/8] update links --- packages/cisco_ftd/_dev/build/docs/README.md | 2 +- packages/cisco_ftd/docs/README.md | 2 +- .../cisco_ftd/docs/knowledge_base/system_info.md | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/cisco_ftd/_dev/build/docs/README.md b/packages/cisco_ftd/_dev/build/docs/README.md index d4687cc7bbe..4b0d0d045ca 100644 --- a/packages/cisco_ftd/_dev/build/docs/README.md +++ b/packages/cisco_ftd/_dev/build/docs/README.md @@ -52,7 +52,7 @@ You must configure your Cisco FTD device to forward syslog messages to the Elast 3. **Deploy Changes**: * Save and deploy your configuration changes to the FTD device. -For detailed, step-by-step instructions, refer to the official Cisco documentation, such as [Configure Logging on FTD](https://www.cisco.com/c/en/us/td/docs/security/firepower/70/fdm/fptd-fdm-config-guide-700/fptd-fdm-logging.html). +For detailed, step-by-step instructions, refer to the official Cisco documentation, such as [Configure Logging on FTD via FMC](https://www.cisco.com/c/en/us/support/docs/security/firepower-ngfw/200479-Configure-Logging-on-FTD-via-FMC.html). #### 2. Add the Cisco FTD Integration in Elastic diff --git a/packages/cisco_ftd/docs/README.md b/packages/cisco_ftd/docs/README.md index e04fbc11afa..f24de7d55a4 100644 --- a/packages/cisco_ftd/docs/README.md +++ b/packages/cisco_ftd/docs/README.md @@ -52,7 +52,7 @@ You must configure your Cisco FTD device to forward syslog messages to the Elast 3. **Deploy Changes**: * Save and deploy your configuration changes to the FTD device. -For detailed, step-by-step instructions, refer to the official Cisco documentation, such as [Configure Logging on FTD](https://www.cisco.com/c/en/us/td/docs/security/firepower/70/fdm/fptd-fdm-config-guide-700/fptd-fdm-logging.html). +For detailed, step-by-step instructions, refer to the official Cisco documentation, such as [Configure Logging on FTD via FMC](https://www.cisco.com/c/en/us/support/docs/security/firepower-ngfw/200479-Configure-Logging-on-FTD-via-FMC.html). #### 2. Add the Cisco FTD Integration in Elastic diff --git a/packages/cisco_ftd/docs/knowledge_base/system_info.md b/packages/cisco_ftd/docs/knowledge_base/system_info.md index 50111867b6c..8c13b63371e 100644 --- a/packages/cisco_ftd/docs/knowledge_base/system_info.md +++ b/packages/cisco_ftd/docs/knowledge_base/system_info.md @@ -202,13 +202,13 @@ Note: Cisco provides a range of Firepower devices, which may have different conf - **Not applicable**: This integration uses syslog protocol and does not require API authentication ## Vendor Resources -- [Cisco Firepower Threat Defense Documentation](https://www.cisco.com/c/en/us/support/security/firepower-threat-defense/products-installation-and-configuration-guides-list.html) -- [Cisco FTD Logging Configuration Guide](https://www.cisco.com/c/en/us/support/security/firepower-threat-defense/products-installation-and-configuration-guides-list.html) -- [Cisco Firepower Management Center Documentation](https://www.cisco.com/c/en/us/support/security/firepower-management-center/products-installation-and-configuration-guides-list.html) +- [Cisco Secure Firewall Management Center Examples](https://www.cisco.com/c/en/us/support/security/defense-center/products-configuration-examples-list.html) +- [Configure Logging on FTD via FMC](https://www.cisco.com/c/en/us/support/docs/security/firepower-ngfw/200479-Configure-Logging-on-FTD-via-FMC.html) +- [Configure FMC to Send Audit Logs to a Syslog Server](https://www.cisco.com/c/en/us/support/docs/security/secure-firewall-management-center/221019-configure-fmc-to-send-audit-logs-to-a-sy.html) # Documentation sites -- [Cisco Firepower Threat Defense Product Page](https://www.cisco.com/c/en/us/products/security/firepower-threat-defense/index.html) -- [Cisco Firepower Documentation](https://www.cisco.com/c/en/us/support/security/firepower-threat-defense/tsd-products-support-series-home.html) -- [Cisco FTD Syslog Configuration](https://www.cisco.com/c/en/us/support/security/firepower-threat-defense/products-installation-and-configuration-guides-list.html) +- [Cisco Secure Firewall Management Center Examples](https://www.cisco.com/c/en/us/support/security/defense-center/products-configuration-examples-list.html) +- [Cisco Firepower Threat Defense Product Page](https://www.cisco.com/c/en/us/support/security/firepower-ngfw/series.html) +- [Cisco FTD Software Release and Sustaining Bulletin](https://www.cisco.com/c/en/us/products/collateral/security/firewalls/bulletin-c25-743178.html) - [Elastic Integrations Documentation](https://www.elastic.co/guide/en/integrations/index.html) - [Elastic Agent Documentation](https://www.elastic.co/guide/en/fleet/current/index.html)