From bfb883f835354a1794cac60d5121ffeb9c71f4a8 Mon Sep 17 00:00:00 2001 From: "sanjay.ms" Date: Tue, 14 Apr 2026 20:58:43 +0530 Subject: [PATCH] gigamon: add CEF/UDP input to ami data stream --- packages/gigamon/changelog.yml | 5 + .../test/pipeline/test-ami.json-expected.json | 4 +- .../ami/_dev/test/pipeline/test-cef.json | 46 ++ .../test/pipeline/test-cef.json-expected.json | 103 +++ .../data_stream/ami/agent/stream/udp.yml.hbs | 43 + .../ingest_pipeline/cef-pipeline.yml | 735 ++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 112 ++- .../gigamon/data_stream/ami/fields/fields.yml | 16 + packages/gigamon/data_stream/ami/manifest.yml | 73 ++ packages/gigamon/docs/README.md | 12 +- packages/gigamon/manifest.yml | 14 +- 11 files changed, 1119 insertions(+), 44 deletions(-) create mode 100644 packages/gigamon/data_stream/ami/_dev/test/pipeline/test-cef.json create mode 100644 packages/gigamon/data_stream/ami/_dev/test/pipeline/test-cef.json-expected.json create mode 100644 packages/gigamon/data_stream/ami/agent/stream/udp.yml.hbs create mode 100644 packages/gigamon/data_stream/ami/elasticsearch/ingest_pipeline/cef-pipeline.yml diff --git a/packages/gigamon/changelog.yml b/packages/gigamon/changelog.yml index 1355384c379..a5664ea68c6 100644 --- a/packages/gigamon/changelog.yml +++ b/packages/gigamon/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.3.0" + changes: + - description: Add CEF/UDP input support to Gigamon AMI data stream. + type: enhancement + link: https://github.com/elastic/integrations/pull/18402 - version: "2.2.0" changes: - description: Updated the AI Insights dashboard filter to detect all AI engines. diff --git a/packages/gigamon/data_stream/ami/_dev/test/pipeline/test-ami.json-expected.json b/packages/gigamon/data_stream/ami/_dev/test/pipeline/test-ami.json-expected.json index e4f334e8b32..09354c41521 100644 --- a/packages/gigamon/data_stream/ami/_dev/test/pipeline/test-ami.json-expected.json +++ b/packages/gigamon/data_stream/ami/_dev/test/pipeline/test-ami.json-expected.json @@ -339,7 +339,7 @@ "category": [ "network" ], - "duration": 3.4123791E9, + "duration": 3.412379E9, "kind": "event", "type": [ "info" @@ -4130,7 +4130,7 @@ "category": [ "network" ], - "duration": 3.5493742E9, + "duration": 3.549374E9, "kind": "event", "type": [ "info" diff --git a/packages/gigamon/data_stream/ami/_dev/test/pipeline/test-cef.json b/packages/gigamon/data_stream/ami/_dev/test/pipeline/test-cef.json new file mode 100644 index 00000000000..61545757493 --- /dev/null +++ b/packages/gigamon/data_stream/ami/_dev/test/pipeline/test-cef.json @@ -0,0 +1,46 @@ +{ + "events": [ + { + "cef": { + "version": "0", + "name": "metadata generation", + "severity": "6", + "extensions": { + "GigamonApplicationID": "67", + "GigamonApplicationName": "elasticsearch", + "app_tags": "Enterprise", + "sourceAddress": "10.114.82.184", + "destinationAddress": "10.114.82.48", + "sourcePort": "33100", + "destinationPort": "9200", + "transportProtocol": "6", + "sourceMacAddress": "00:50:56:9d:49:ec", + "destinationMacAddress": "00:50:56:9d:7f:9b", + "GigamonMdataIpVer": "4", + "GigamonInitiatorOctets": "11810", + "GigamonResponderOctets": "575", + "GigamonInitiatorPackets": "13", + "GigamonResponderPackets": "5", + "GigamonMdataFlowStartMsec": "2026:04:27 17:41:51.651", + "GigamonMdataFlowEndMsec": "2026:04:27 17:41:54.659", + "GigamonMdataIntfName": "0", + "GigamonMdataEgressIntfID": "0", + "GigamonMdataSysUpTimeFirst": "899345728", + "GigamonMdataSysUpTimeLast": "899348736", + "GigamonMdataFlowEndReason": "3", + "GigamonMdataSeqNum": "1160592", + "GigamonMdataTcpRttMin": "0.000285", + "GigamonMdataTcpRttMax": "0.000285", + "GigamonMdataTcpRttAppMin": "0.007603", + "GigamonMdataTcpRttAppMax": "0.015634", + "GigamonMdata_http_server": "10.114.82.48", + "GigamonMdata_http_uri_path": "/_bulk", + "GigamonMdata_http_host": "10.114.82.48:9200", + "GigamonMdata_http_method": "POST", + "GigamonMdata_http_version": "1.1", + "GigamonMdata_http_code": "200" + } + } + } + ] +} \ No newline at end of file diff --git a/packages/gigamon/data_stream/ami/_dev/test/pipeline/test-cef.json-expected.json b/packages/gigamon/data_stream/ami/_dev/test/pipeline/test-cef.json-expected.json new file mode 100644 index 00000000000..d6099454f20 --- /dev/null +++ b/packages/gigamon/data_stream/ami/_dev/test/pipeline/test-cef.json-expected.json @@ -0,0 +1,103 @@ +{ + "expected": [ + { + "destination": { + "bytes": 575, + "ip": "10.114.82.48", + "mac": "00-50-56-9D-7F-9B", + "packets": 5, + "port": 9200 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "end": "2026-04-27T17:41:54.659Z", + "kind": "event", + "start": "2026-04-27T17:41:51.651Z", + "type": [ + "info" + ] + }, + "gigamon": { + "ami": { + "app_id": "67", + "app_name": "elasticsearch", + "app_tags": "Enterprise", + "dst_bytes": 575, + "dst_ip": "10.114.82.48", + "dst_mac": "00:50:56:9d:7f:9b", + "dst_packets": 5, + "dst_port": 9200, + "egress_intf_id": "0", + "end_reason": "3", + "end_reason_value": "End of Flow", + "end_time": "2026-04-27T17:41:54.659Z", + "http_code": 200, + "http_host": "10.114.82.48:9200", + "http_method": "POST", + "http_server": "10.114.82.48", + "http_uri_path": "/_bulk", + "http_version": "1.1", + "intf_name": "0", + "ip_version": "4", + "protocol": "6", + "seq_num": 1160592, + "src_bytes": 11810, + "src_ip": "10.114.82.184", + "src_mac": "00:50:56:9d:49:ec", + "src_packets": 13, + "src_port": 33100, + "start_time": "2026-04-27T17:41:51.651Z", + "sys_up_time_first": 899345728, + "sys_up_time_last": 899348736, + "tcp_rtt_app_max": 0.015634, + "tcp_rtt_app_min": 0.007603, + "tcp_rtt_max": 2.85E-4, + "tcp_rtt_min": 2.85E-4 + } + }, + "host": { + "name": "10.114.82.48:9200" + }, + "http": { + "request": { + "method": "POST" + }, + "response": { + "status_code": 200 + }, + "version": "1.1" + }, + "network": { + "protocol": "elasticsearch" + }, + "server": { + "domain": "10.114.82.48" + }, + "service": { + "id": "67" + }, + "source": { + "bytes": 11810, + "ip": "10.114.82.184", + "mac": "00-50-56-9D-49-EC", + "packets": 13, + "port": 33100 + }, + "tags": [ + "preserve_original_event", + "_geoip_database_unavailable_GeoLite2-City.mmdb", + "_geoip_database_unavailable_GeoLite2-ASN.mmdb", + "_geoip_database_unavailable_GeoLite2-City.mmdb", + "_geoip_database_unavailable_GeoLite2-ASN.mmdb" + ], + "url": { + "path": "/_bulk" + } + } + ] +} \ No newline at end of file diff --git a/packages/gigamon/data_stream/ami/agent/stream/udp.yml.hbs b/packages/gigamon/data_stream/ami/agent/stream/udp.yml.hbs new file mode 100644 index 00000000000..7b8d3317157 --- /dev/null +++ b/packages/gigamon/data_stream/ami/agent/stream/udp.yml.hbs @@ -0,0 +1,43 @@ +host: "{{listen_address}}:{{listen_port}}" +{{#if udp_options.length}} +{{udp_options}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +processors: +{{#if preprocessors}} +- copy_fields: + fields: + - from: "message" + to: "@metadata.event_original" +{{preprocessors}} +{{/if}} +- rename: + fields: + - {from: "message", to: "event.original"} +- decode_cef: + field: event.original +{{#if decode_cef_timezone}} + timezone: "{{ decode_cef_timezone }}" +{{/if}} +{{#if ignore_empty_values }} + ignore_empty_values: true +{{/if}} +{{#if preprocessors}} +- convert: + mode: rename + fields: + - from: "@metadata.event_original" + to: "event.original" +{{/if}} +{{#if processors}} +{{processors}} +{{/if}} diff --git a/packages/gigamon/data_stream/ami/elasticsearch/ingest_pipeline/cef-pipeline.yml b/packages/gigamon/data_stream/ami/elasticsearch/ingest_pipeline/cef-pipeline.yml new file mode 100644 index 00000000000..e2d13eb0466 --- /dev/null +++ b/packages/gigamon/data_stream/ami/elasticsearch/ingest_pipeline/cef-pipeline.yml @@ -0,0 +1,735 @@ +--- +description: Pipeline for Gigamon CEF Ami logs.. CEF decoding happens in the Agent. This performs additional enrichment and vendor specific transformations. +processors: + - rename: + field: cef.extensions + target_field: gigamon.ami + if: ctx.cef.extensions != null + tag: rename_cef_extension + - rename: + field: cef.device + target_field: gigamon.ami.device + if: ctx.cef.device != null + tag: rename_cef_device + + # rename + + - rename: + field: gigamon.ami.GigamonMdata_smb_version + ignore_missing: true + target_field: gigamon.ami.smb_version + + - rename: + field: gigamon.ami.GigamonMdataSeqNum + ignore_missing: true + target_field: gigamon.ami.seq_num + + - rename: + field: gigamon.ami.GigamonApplicationID + target_field: gigamon.ami.app_id + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonApplicationName + target_field: gigamon.ami.app_name + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataAppTags + target_field: gigamon.ami.app_tags + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataIpVer + target_field: gigamon.ami.ip_version + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataFlowStartMsec + target_field: gigamon.ami.start_time + ignore_missing: true + # convert: + # '0': Network + # '1': Endpoint + # '2': Access + # '3': Threat + # '4': Mobile + + - rename: + field: gigamon.ami.GigamonMdataFlowEndMsec + target_field: gigamon.ami.end_time + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataIntfName + target_field: gigamon.ami.intf_name + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataEgressIntfID + target_field: gigamon.ami.egress_intf_id + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataSysUpTimeFirst + target_field: gigamon.ami.sys_up_time_first + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataSysUpTimeLast + target_field: gigamon.ami.sys_up_time_last + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataFlowEndReason + target_field: gigamon.ami.end_reason + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_smb_command_string + target_field: gigamon.ami.smb_command_string + ignore_missing: true + + - rename: + field: gigamon.ami.dMac + target_field: gigamon.ami.dst_mac + ignore_missing: true + + - rename: + field: gigamon.ami.dst + target_field: gigamon.ami.dst_ip + ignore_missing: true + + - rename: + field: gigamon.ami.dpt + target_field: gigamon.ami.dst_port + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonResponderOctets + target_field: gigamon.ami.dst_bytes + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonResponderPackets + target_field: gigamon.ami.dst_packets + ignore_missing: true + + - rename: + field: gigamon.ami.sMac + target_field: gigamon.ami.src_mac + ignore_missing: true + + - rename: + field: gigamon.ami.src + target_field: gigamon.ami.src_ip + ignore_missing: true + + - rename: + field: gigamon.ami.spt + target_field: gigamon.ami.src_port + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonInitiatorOctets + target_field: gigamon.ami.src_bytes + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonInitiatorPackets + target_field: gigamon.ami.src_packets + ignore_missing: true + + - rename: + if: ctx.gigamon?.ami?.dst_port == null + field: gigamon.ami.destinationPort + target_field: gigamon.ami.dst_port + ignore_missing: true + + - rename: + if: ctx.gigamon?.ami?.src_port == null + field: gigamon.ami.sourcePort + target_field: gigamon.ami.src_port + ignore_missing: true + + - rename: + if: ctx.gigamon?.ami?.dst_ip == null + field: gigamon.ami.destinationAddress + target_field: gigamon.ami.dst_ip + ignore_missing: true + + - rename: + if: ctx.gigamon?.ami?.src_ip == null + field: gigamon.ami.sourceAddress + target_field: gigamon.ami.src_ip + ignore_missing: true + + - rename: + if: ctx.gigamon?.ami?.src_mac == null + field: gigamon.ami.sourceMacAddress + target_field: gigamon.ami.src_mac + ignore_missing: true + + - rename: + if: ctx.gigamon?.ami?.dst_mac == null + field: gigamon.ami.destinationMacAddress + target_field: gigamon.ami.dst_mac + ignore_missing: true + + - rename: + field: gigamon.ami.transportProtocol + target_field: gigamon.ami.protocol + ignore_missing: true + + # dns fields + - rename: + field: gigamon.ami.GigamonMdata_dns_qdcount + target_field: gigamon.ami.dns_qdcount + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_transaction_id + target_field: gigamon.ami.dns_transaction_id + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_name + target_field: gigamon.ami.dns_name + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_host + target_field: gigamon.ami.dns_host + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_host_addr + target_field: gigamon.ami.dns_host_addr + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_host_type + target_field: gigamon.ami.dns_host_type + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_ttl + target_field: gigamon.ami.dns_ttl + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_flags + target_field: gigamon.ami.dns_flags + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_opcode + target_field: gigamon.ami.dns_opcode + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_class + target_field: gigamon.ami.dns_class + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_host_class + target_field: gigamon.ami.dns_host_class + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_host_raw + target_field: gigamon.ami.dns_host_raw + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_query + target_field: gigamon.ami.dns_query + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_query_type + target_field: gigamon.ami.dns_query_type + ignore_missing: true + + - rename: + field: gigamon.ami.deviceInboundInterface + target_field: gigamon.ami.device_inbound_interface + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_ancount + target_field: gigamon.ami.dns_ancount + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_arcount + target_field: gigamon.ami.dns_arcount + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_reply_code + target_field: gigamon.ami.dns_reply_code + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_response_time + target_field: gigamon.ami.dns_response_time + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_reverse_addr + target_field: gigamon.ami.dns_reverse_addr + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_dns_tunneling + target_field: gigamon.ami.dns_tunneling + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ip_wrong_crc + target_field: gigamon.ami.ip_wrong_crc + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_krb5_login + target_field: gigamon.ami.krb5_login + ignore_missing: true + + # http_fields + - rename: + field: gigamon.ami.GigamonMdata_http_server + target_field: gigamon.ami.http_server + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_uri + target_field: gigamon.ami.http_uri + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_uri_full + target_field: gigamon.ami.http_uri_full + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_mime_type + target_field: gigamon.ami.http_mime_type + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_server_agent + target_field: gigamon.ami.http_server_agent + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_rtt + target_field: gigamon.ami.http_rtt + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_code + target_field: gigamon.ami.http_code + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_content_len + target_field: gigamon.ami.http_content_len + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_uri_path + target_field: gigamon.ami.http_uri_path + ignore_missing: true + # convert: + # '*v1*': V1 + + - rename: + field: gigamon.ami.GigamonMdata_http_request_size + target_field: gigamon.ami.http_request_size + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_host + target_field: gigamon.ami.http_host + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_uri_decoded + target_field: gigamon.ami.http_uri_decoded + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_uri_raw + target_field: gigamon.ami.http_uri_raw + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_content_type + target_field: gigamon.ami.http_content_type + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_method + target_field: gigamon.ami.http_method + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_version + target_field: gigamon.ami.http_version + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_user_agent + target_field: gigamon.ami.http_user_agent + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_response_ts + target_field: gigamon.ami.http_response_ts + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_content_encoding + target_field: gigamon.ami.http_content_encoding + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_http_referer + target_field: gigamon.ami.http_referer + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataFlowStartSec + target_field: gigamon.ami.flow_start_sec + ignore_missing: true + + # tcp_fields + - rename: + field: gigamon.ami.GigamonMdataTcpFlags + target_field: gigamon.ami.tcp_flags + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_tcp_rtt_app + target_field: gigamon.ami.tcp_rtt_app + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_tcp_loss_count + target_field: gigamon.ami.tcp_loss_count + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_tcp_rtt + target_field: gigamon.ami.tcp_rtt + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_tcp_retransmission_bytes + target_field: gigamon.ami.tcp_retransmission_bytes + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_tcp_wrong_crc + target_field: gigamon.ami.tcp_wrong_crc + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_tcp_flag_reset + target_field: gigamon.ami.tcp_flag_reset + ignore_missing: true + + # ssl_fields + - rename: + field: gigamon.ami.GigamonMdata_ssl_certif_md5 + target_field: gigamon.ami.ssl_certif_md5 + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_common_name + target_field: gigamon.ami.ssl_common_name + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_validity_not_before + target_field: gigamon.ami.ssl_validity_not_before + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_validity_not_after + target_field: gigamon.ami.ssl_validity_not_after + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_serial_number + target_field: gigamon.ami.ssl_serial_number + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_handshake_type + target_field: gigamon.ami.ssl_handshake_type + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_organization_name + target_field: gigamon.ami.ssl_organization_name + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_request_size + target_field: gigamon.ami.ssl_request_size + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_cipher_suite_id + target_field: gigamon.ami.ssl_cipher_suite_id + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_cipher_suite_list + target_field: gigamon.ami.ssl_cipher_suite_list + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certif_sha1 + target_field: gigamon.ami.ssl_certif_sha1 + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_content_type + target_field: gigamon.ami.ssl_content_type + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_protocol_version + target_field: gigamon.ami.ssl_protocol_version + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_client_hello_extension_type + target_field: gigamon.ami.ssl_client_hello_extension_type + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_server_hello_extension_type + target_field: gigamon.ami.ssl_server_hello_extension_type + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_dn_subject + target_field: gigamon.ami.ssl_certificate_dn_subject + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_subject_cn + target_field: gigamon.ami.ssl_certificate_subject_cn + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_subject_l + target_field: gigamon.ami.ssl_certificate_subject_l + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_subject_st + target_field: gigamon.ami.ssl_certificate_subject_st + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_subject_o + target_field: gigamon.ami.ssl_certificate_subject_o + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_subject_ou + target_field: gigamon.ami.ssl_certificate_subject_ou + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_subject_c + target_field: gigamon.ami.ssl_certificate_subject_c + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_dn_issuer + target_field: gigamon.ami.ssl_certificate_dn_issuer + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_issuer_cn + target_field: gigamon.ami.ssl_certificate_issuer_cn + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_issuer_l + target_field: gigamon.ami.ssl_certificate_issuer_l + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_issuer_st + target_field: gigamon.ami.ssl_certificate_issuer_st + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_issuer_o + target_field: gigamon.ami.ssl_certificate_issuer_o + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_issuer_ou + target_field: gigamon.ami.ssl_certificate_issuer_ou + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_issuer_c + target_field: gigamon.ami.ssl_certificate_issuer_c + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_client_hello_extension_len + target_field: gigamon.ami.ssl_client_hello_extension_len + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_server_hello_extension_len + target_field: gigamon.ami.ssl_server_hello_extension_len + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_nb_compression_methods + target_field: gigamon.ami.ssl_nb_compression_methods + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_compression_method + target_field: gigamon.ami.ssl_compression_method + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_ext_sig_algorithms_len + target_field: gigamon.ami.ssl_ext_sig_algorithms_len + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_ext_sig_algorithm_scheme + target_field: gigamon.ami.ssl_ext_sig_algorithm_scheme + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_ext_sig_algorithm_hash + target_field: gigamon.ami.ssl_ext_sig_algorithm_hash + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_ext_sig_algorithm_sig + target_field: gigamon.ami.ssl_ext_sig_algorithm_sig + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_subject_key_algo_oid + target_field: gigamon.ami.ssl_certificate_subject_key_algo_oid + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_certificate_subject_key_size + target_field: gigamon.ami.ssl_certificate_subject_key_size + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_cert_extension_oid + target_field: gigamon.ami.ssl_cert_extension_oid + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_cert_ext_authority_key_id + target_field: gigamon.ami.ssl_cert_ext_authority_key_id + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_cert_ext_subject_key_id + target_field: gigamon.ami.ssl_cert_ext_subject_key_id + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_fingerprint_ja3 + target_field: gigamon.ami.ssl_fingerprint_ja3 + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_index + target_field: gigamon.ami.ssl_index + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_issuer + target_field: gigamon.ami.ssl_issuer + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_session_id + target_field: gigamon.ami.ssl_session_id + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_declassify_override + target_field: gigamon.ami.ssl_declassify_override + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdata_ssl_signalization_override + target_field: gigamon.ami.ssl_signalization_override + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataTcpRttMin + target_field: gigamon.ami.tcp_rtt_min + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataTcpRttMax + target_field: gigamon.ami.tcp_rtt_max + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataTcpRttMean + target_field: gigamon.ami.tcp_rtt_mean + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataTcpRttAppMin + target_field: gigamon.ami.tcp_rtt_app_min + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataTcpRttAppMax + target_field: gigamon.ami.tcp_rtt_app_max + ignore_missing: true + + - rename: + field: gigamon.ami.GigamonMdataTcpRttAppMean + target_field: gigamon.ami.tcp_rtt_app_mean + ignore_missing: true + + # Cleanup + - remove: + field: + - cef + - _tmp + ignore_missing: true + ignore_failure: true +on_failure: + - set: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + - set: + field: event.kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false diff --git a/packages/gigamon/data_stream/ami/elasticsearch/ingest_pipeline/default.yml b/packages/gigamon/data_stream/ami/elasticsearch/ingest_pipeline/default.yml index 6a4b1b4184c..d34445bd3d5 100644 --- a/packages/gigamon/data_stream/ami/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gigamon/data_stream/ami/elasticsearch/ingest_pipeline/default.yml @@ -9,6 +9,10 @@ processors: target_field: gigamon.ami if: ctx.json != null tag: rename_json + - pipeline: + if: ctx.cef != null + name: '{{ IngestPipeline "cef-pipeline" }}' + tag: cef_pipeline - set: field: event.kind value: event @@ -466,6 +470,78 @@ processors: - append: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: gigamon.ami.tcp_rtt_min + if: ctx.gigamon?.ami?.tcp_rtt_min != null + tag: convert_tcp_rtt_min + type: double + on_failure: + - remove: + field: gigamon.ami.tcp_rtt_min + ignore_missing: true + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: gigamon.ami.tcp_rtt_max + if: ctx.gigamon?.ami?.tcp_rtt_max != null + tag: convert_tcp_rtt_max + type: double + on_failure: + - remove: + field: gigamon.ami.tcp_rtt_max + ignore_missing: true + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: gigamon.ami.tcp_rtt_mean + if: ctx.gigamon?.ami?.tcp_rtt_mean != null + tag: convert_tcp_rtt_mean + type: double + on_failure: + - remove: + field: gigamon.ami.tcp_rtt_mean + ignore_missing: true + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: gigamon.ami.tcp_rtt_app_min + if: ctx.gigamon?.ami?.tcp_rtt_app_min != null + tag: convert_tcp_rtt_app_min + type: double + on_failure: + - remove: + field: gigamon.ami.tcp_rtt_app_min + ignore_missing: true + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: gigamon.ami.tcp_rtt_app_max + if: ctx.gigamon?.ami?.tcp_rtt_app_max != null + tag: convert_tcp_rtt_app_max + type: double + on_failure: + - remove: + field: gigamon.ami.tcp_rtt_app_max + ignore_missing: true + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: gigamon.ami.tcp_rtt_app_mean + if: ctx.gigamon?.ami?.tcp_rtt_app_mean != null + tag: convert_tcp_rtt_app_mean + type: double + on_failure: + - remove: + field: gigamon.ami.tcp_rtt_app_mean + ignore_missing: true + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: lang: painless description: Gigamon AMI lookup mappings @@ -895,18 +971,6 @@ processors: ctx.gigamon.ami.dns_reply_code_value = params['dns_reply_code'][ctx.gigamon.ami.dns_reply_code]; } - - convert: - field: gigamon.ami.dns_response_time - if: ctx.gigamon?.ami?.dns_response_time != null - tag: convert_dns_response_time - type: float - on_failure: - - remove: - field: gigamon.ami.dns_response_time - ignore_missing: true - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: lang: painless description: Convert dns_response_time in seconds to nanoseconds for populating event.duration @@ -960,18 +1024,6 @@ processors: value: "{{{gigamon.ami.dns_host_addr}}}" if: ctx.gigamon?.ami?.dns_host_addr != null allow_duplicates: false - - convert: - field: gigamon.ami.dns_ttl - if: ctx.gigamon?.ami?.dns_ttl != null - tag: convert_dns_ttl - type: long - on_failure: - - remove: - field: gigamon.ami.dns_ttl - ignore_missing: true - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: dns.answers.ttl tag: set_dns_answers.ttl @@ -1041,18 +1093,6 @@ processors: tag: set_http_version copy_from: gigamon.ami.http_version if: ctx.gigamon?.ami?.http_version != null - - convert: - field: gigamon.ami.http_code - if: ctx.gigamon?.ami?.http_code != null - tag: convert_http_code - type: long - on_failure: - - remove: - field: gigamon.ami.http_code - ignore_missing: true - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: http.response.status_code tag: set_http_response_status_code diff --git a/packages/gigamon/data_stream/ami/fields/fields.yml b/packages/gigamon/data_stream/ami/fields/fields.yml index bce490c41f2..bb0a17d04c6 100644 --- a/packages/gigamon/data_stream/ami/fields/fields.yml +++ b/packages/gigamon/data_stream/ami/fields/fields.yml @@ -10,6 +10,8 @@ type: keyword - name: app_name type: keyword + - name: app_tags + type: keyword - name: ts type: date - name: vendor @@ -169,6 +171,8 @@ type: keyword - name: http_user_agent type: keyword + - name: http_response_ts + type: double # tcp_ fields - name: tcp_flags type: keyword @@ -306,3 +310,15 @@ type: keyword - name: smb_path type: keyword + - name: tcp_rtt_min + type: double + - name: tcp_rtt_max + type: double + - name: tcp_rtt_mean + type: double + - name: tcp_rtt_app_min + type: double + - name: tcp_rtt_app_max + type: double + - name: tcp_rtt_app_mean + type: double diff --git a/packages/gigamon/data_stream/ami/manifest.yml b/packages/gigamon/data_stream/ami/manifest.yml index a97387cda19..b7c1ac66f8e 100644 --- a/packages/gigamon/data_stream/ami/manifest.yml +++ b/packages/gigamon/data_stream/ami/manifest.yml @@ -61,3 +61,76 @@ streams: show_user: false description: >- Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - title: Gigamon CEF logs + description: Collect Gigamon CEF logs using udp input + input: udp + template_path: udp.yml.hbs + vars: + - name: listen_port + title: Listen Port + type: integer + description: The UDP port to listen for traffic. + required: true + show_user: true + multi: false + default: 9560 + - name: decode_cef_timezone + title: CEF Timezone + type: text + description: IANA time zone or time offset (e.g. `+0200`) to use when interpreting timestamps without a time zone in the CEF message. + required: false + show_user: false + multi: false + - name: tags + title: Tags + type: text + description: A list of tags to include in events. Including `forwarded` indicates that the events did not originate on this host and causes `host.name` to not be added to events. + required: true + show_user: false + multi: true + default: + - cef + - forwarded + - name: preserve_original_event + type: bool + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original. + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: udp_options + title: Custom UDP Options + type: yaml + description: Specify custom configuration options for the UDP input. + required: false + show_user: false + multi: false + default: | + #read_buffer: 100MiB + #max_message_size: 50KiB + #timeout: 300s + - name: preprocessors + title: Pre-Processors + type: yaml + description: > + Pre-processors are run before the CEF message is decoded. They can be used to correct CEF formatting inconsistencies that may exist from some sources. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + required: false + show_user: false + multi: false + - name: ignore_empty_values + title: Ignore Empty Values + type: bool + description: Ignore CEF fields that are empty. The alternative behavior is to treat an empty field as an error. + required: true + show_user: true + multi: false + default: false diff --git a/packages/gigamon/docs/README.md b/packages/gigamon/docs/README.md index 4c43e2b63d6..c5e7010df9d 100644 --- a/packages/gigamon/docs/README.md +++ b/packages/gigamon/docs/README.md @@ -212,6 +212,7 @@ An example event for `ami` looks as following: | event.module | Event module | constant_keyword | | gigamon.ami.app_id | | keyword | | gigamon.ami.app_name | | keyword | +| gigamon.ami.app_tags | | keyword | | gigamon.ami.device_inbound_interface | | keyword | | gigamon.ami.dns_ancount | | long | | gigamon.ami.dns_arcount | | long | @@ -260,6 +261,7 @@ An example event for `ami` looks as following: | gigamon.ami.http_mime_type | | keyword | | gigamon.ami.http_referer | | keyword | | gigamon.ami.http_request_size | | long | +| gigamon.ami.http_response_ts | | double | | gigamon.ami.http_rtt | | double | | gigamon.ami.http_server | | keyword | | gigamon.ami.http_server_agent | | keyword | @@ -355,6 +357,12 @@ An example event for `ami` looks as following: | gigamon.ami.tcp_retransmission_bytes | | long | | gigamon.ami.tcp_rtt | | double | | gigamon.ami.tcp_rtt_app | | double | +| gigamon.ami.tcp_rtt_app_max | | double | +| gigamon.ami.tcp_rtt_app_mean | | double | +| gigamon.ami.tcp_rtt_app_min | | double | +| gigamon.ami.tcp_rtt_max | | double | +| gigamon.ami.tcp_rtt_mean | | double | +| gigamon.ami.tcp_rtt_min | | double | | gigamon.ami.tcp_wrong_crc | | keyword | | gigamon.ami.ts | | date | | gigamon.ami.vendor | | keyword | @@ -365,6 +373,4 @@ An example event for `ami` looks as following: | ml_is_dga.malicious_probability | | float | | ml_is_dga.record_score | | keyword | | ml_is_dga.risk_score | | integer | -| tags | User defined tags. | keyword | - - +| tags | User defined tags. | keyword | \ No newline at end of file diff --git a/packages/gigamon/manifest.yml b/packages/gigamon/manifest.yml index a440b45a2a9..9adf5907f76 100644 --- a/packages/gigamon/manifest.yml +++ b/packages/gigamon/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.3 name: gigamon title: Gigamon -version: "2.2.0" +version: "2.3.0" description: Collect logs from Gigamon with Elastic Agent. type: integration categories: @@ -79,8 +79,8 @@ icons: type: image/svg+xml policy_templates: - name: gigamon - title: Gigamon AMI json - description: Collect json data from Gigamon AMI + title: Gigamon AMI + description: Collect logs from Gigamon AMI via HTTP Endpoint (JSON) or UDP (CEF) inputs: - type: http_endpoint title: Collect json data from Gigamon AMI via HTTP Endpoint @@ -90,6 +90,14 @@ policy_templates: type: text title: Listen Address description: The bind address to listen for http endpoint connections. Set to '0.0.0.0' to bind to all available interfaces. + - title: 'Collect CEF logs from Gigamon AMI (input: udp)' + description: 'Collect CEF logs from Gigamon AMI (input: udp)' + type: udp + vars: + - name: listen_address + type: text + title: Listen Address + description: The bind address to listen for UDP connections. Set to '0.0.0.0' to bind to all available interfaces. owner: github: elastic/integration-experience type: partner