From db0a75e739dd0e2f3a446c757719b96b0c7bd916 Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Sat, 9 May 2026 19:07:56 +0200 Subject: [PATCH 1/2] [juniper_srx] extract IP from RPD_IFD_NOTIFICATION events Add a tag-specific grok in the system pipeline to extract the interface address from RPD_IFD_NOTIFICATION messages (pattern: "address #N ") into host.ip and related.ip. Declare juniper.srx.system.if_trace and juniper.srx.interface_index fields surfaced by the message k-v extractor. Also avoid overwriting an already-populated message field when renaming juniper.srx.message. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/juniper_srx/changelog.yml | 5 ++ .../log/_dev/test/pipeline/test-system.log | 1 + .../pipeline/test-system.log-expected.json | 60 +++++++++++++++++++ .../elasticsearch/ingest_pipeline/system.yml | 24 +++++++- .../data_stream/log/fields/fields.yml | 4 ++ packages/juniper_srx/docs/README.md | 2 + packages/juniper_srx/manifest.yml | 2 +- 7 files changed, 96 insertions(+), 2 deletions(-) diff --git a/packages/juniper_srx/changelog.yml b/packages/juniper_srx/changelog.yml index c0e7aa5d5fe..72cdf69dee2 100644 --- a/packages/juniper_srx/changelog.yml +++ b/packages/juniper_srx/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.28.0" + changes: + - description: Extract interface address from RPD_IFD_NOTIFICATION events into host.ip and related.ip, and avoid overwriting an already-populated message field. + type: enhancement + link: https://github.com/elastic/integrations/pull/18927 - version: "1.27.0" changes: - description: Improve integration documentation. diff --git a/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log b/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log index 2a3dadbf708..8565c250355 100644 --- a/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log +++ b/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log @@ -20,3 +20,4 @@ <166>1 2023-05-08T10:54:24.756+10:00 AB1234-A-AB-AB01C-ABC - - - - AB1234-A-AB-AB01C-ABC nh_fabric_fill_jnhinfo: Storing nh_id as 0x2dd and jnh as 0x58e302 <167>1 2023-05-08T10:54:24.704+10:00 AB1234-A-AB-AB01C-ABC - - - - AB1234-A-AB-AB01C-ABC Copying remote chassis chassis 1, IP: 81.2.69.192 <166>1 2023-05-08T10:54:24.756+10:00 AB1234-A-AB-AB01C-ABC - - - - AB1234-A-AB-AB01C-ABC nh_fabric_fill_jnhinfo: ABCDE: Test default message 123456 +<30>1 2026-05-07T12:35:17.448+02:00 AB1234-A-AB-AB01C-ABC rpd 20893 RPD_IFD_NOTIFICATION [junos@1111.1.1.1.1.111 state="UpDown" interface-name="st0.60" interface-index="508" message=" agg_parent 0 address #0 175.16.199.0"] IF_TRACE: EVENT st0.60 index 508 agg_parent 0 address #0 175.16.199.0 diff --git a/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json b/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json index 8e8d9beb83d..502d1e8048d 100644 --- a/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json +++ b/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json @@ -1170,6 +1170,66 @@ "tags": [ "preserve_original_event" ] + }, + { + "@timestamp": "2026-05-07T10:35:17.448Z", + "ecs": { + "version": "8.17.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "<30>1 2026-05-07T12:35:17.448+02:00 AB1234-A-AB-AB01C-ABC rpd 20893 RPD_IFD_NOTIFICATION [junos@1111.1.1.1.1.111 state=\"UpDown\" interface-name=\"st0.60\" interface-index=\"508\" message=\" agg_parent 0 address #0 175.16.199.0\"] IF_TRACE: EVENT st0.60 index 508 agg_parent 0 address #0 175.16.199.0", + "severity": 30 + }, + "host": { + "ip": [ + "175.16.199.0" + ], + "name": "AB1234-A-AB-AB01C-ABC" + }, + "juniper": { + "srx": { + "interface_index": "508", + "log_type": "system", + "message": " agg_parent 0 address #0 175.16.199.0", + "process": "rpd", + "state": "UpDown", + "system": { + "if_trace": "EVENT st0.60 index 508 agg_parent 0 address #0 175.16.199.0" + }, + "tag": "RPD_IFD_NOTIFICATION" + } + }, + "log": { + "level": "informational" + }, + "message": "IF_TRACE: EVENT st0.60 index 508 agg_parent 0 address #0 175.16.199.0", + "observer": { + "ingress": { + "interface": { + "name": "st0.60" + } + }, + "name": "AB1234-A-AB-AB01C-ABC", + "product": "SRX", + "type": "firewall", + "vendor": "Juniper" + }, + "process": { + "name": "rpd", + "pid": 20893 + }, + "related": { + "ip": [ + "175.16.199.0" + ] + }, + "tags": [ + "preserve_original_event" + ] } ] } diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml index 4e2718dd12a..80be322c0ff 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml @@ -308,6 +308,28 @@ processors: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' +# RPD_IFD_NOTIFICATION +# Extract interface address (e.g. "address #0 175.16.199.0") into host.ip and related.ip +- grok: + tag: grok_rpd_ifd_notification_address + if: "ctx.juniper?.srx?.tag == 'RPD_IFD_NOTIFICATION' && ctx.juniper?.srx?.message != null" + field: juniper.srx.message + patterns: + - 'address #%{NONNEGINT} %{IP:_temp_.rpd_ifd.address}' + ignore_failure: true +- append: + tag: append_rpd_ifd_address_to_host_ip + if: "ctx._temp_?.rpd_ifd?.address != null" + field: host.ip + value: '{{{_temp_.rpd_ifd.address}}}' + allow_duplicates: false +- append: + tag: append_rpd_ifd_address_to_related_ip + if: "ctx._temp_?.rpd_ifd?.address != null" + field: related.ip + value: '{{{_temp_.rpd_ifd.address}}}' + allow_duplicates: false + # After System Structured, Structured-Brief, and Unstructured messages: there are optional key-value pairs seperated by ", ". These should now be in `message` field # split k-v fields inside message field. - kv: @@ -901,7 +923,7 @@ processors: field: juniper.srx.message target_field: message ignore_missing: true - if: "ctx.juniper?.srx?.message != null" + if: "ctx.juniper?.srx?.message != null && ctx.message == null" - remove: tag: remove_juniper_srx_process_f8fae6df diff --git a/packages/juniper_srx/data_stream/log/fields/fields.yml b/packages/juniper_srx/data_stream/log/fields/fields.yml index 3f5e6c1f2ef..c13a0a07b13 100644 --- a/packages/juniper_srx/data_stream/log/fields/fields.yml +++ b/packages/juniper_srx/data_stream/log/fields/fields.yml @@ -423,6 +423,8 @@ type: integer - name: direction type: keyword + - name: if_trace + type: keyword - name: ike_version type: integer - name: local @@ -522,6 +524,8 @@ type: keyword - name: function_name type: keyword + - name: interface_index + type: keyword - name: operational_status type: keyword - name: snmp_interface_index diff --git a/packages/juniper_srx/docs/README.md b/packages/juniper_srx/docs/README.md index bd6283ef42f..eedd28ad7a1 100644 --- a/packages/juniper_srx/docs/README.md +++ b/packages/juniper_srx/docs/README.md @@ -592,6 +592,7 @@ The `log` data stream provides events from Juniper SRX devices. These logs inclu | juniper.srx.index | index | keyword | | juniper.srx.index1 | | keyword | | juniper.srx.index2 | | keyword | +| juniper.srx.interface_index | | keyword | | juniper.srx.ip_mon_reth_scan.trigger | | keyword | | juniper.srx.kern_arp_addr_change.ip | | ip | | juniper.srx.kern_arp_addr_change.mac1 | | keyword | @@ -660,6 +661,7 @@ The `log` data stream provides events from Juniper SRX devices. These logs inclu | juniper.srx.sub_category | sub category | keyword | | juniper.srx.system.aux_spi | | integer | | juniper.srx.system.direction | | keyword | +| juniper.srx.system.if_trace | | keyword | | juniper.srx.system.ike_version | | integer | | juniper.srx.system.local | | keyword | | juniper.srx.system.local_gateway | | ip | diff --git a/packages/juniper_srx/manifest.yml b/packages/juniper_srx/manifest.yml index 6f65dc3be23..28fdaaf3a11 100644 --- a/packages/juniper_srx/manifest.yml +++ b/packages/juniper_srx/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: juniper_srx title: Juniper SRX -version: "1.27.0" +version: "1.28.0" description: Collect logs from Juniper SRX devices with Elastic Agent. categories: ["network", "security", "firewall_security"] type: integration From 6ee51ecf2b80bf98f33e4cf9c38ab0a308c9b2c4 Mon Sep 17 00:00:00 2001 From: Ilya Nikokoshev Date: Wed, 1 Jul 2026 20:21:24 +0200 Subject: [PATCH 2/2] [juniper_srx] address review: drop redundant if_trace, map interface index to ECS - Drop juniper.srx.system.if_trace: the generic k-v extractor surfaces it from IF_TRACE messages but it duplicates `message` (message minus the "IF_TRACE: " prefix). Remove the field and its declaration. - Also drop juniper.srx.system when the cleanup leaves it empty. - Map juniper.srx.interface_index (SNMP/IFD index) to ECS observer.ingress.interface.id, alongside the interface name that already lands under observer.ingress for these events. Co-Authored-By: Claude Opus 4.8 --- packages/juniper_srx/changelog.yml | 2 +- .../_dev/test/pipeline/test-system.log-expected.json | 5 +---- .../log/elasticsearch/ingest_pipeline/default.yml | 7 +++++++ .../log/elasticsearch/ingest_pipeline/system.yml | 11 ++++++++++- .../juniper_srx/data_stream/log/fields/fields.yml | 4 ---- packages/juniper_srx/docs/README.md | 2 -- 6 files changed, 19 insertions(+), 12 deletions(-) diff --git a/packages/juniper_srx/changelog.yml b/packages/juniper_srx/changelog.yml index eac1be3edee..fbaf8de1a15 100644 --- a/packages/juniper_srx/changelog.yml +++ b/packages/juniper_srx/changelog.yml @@ -1,7 +1,7 @@ # newer versions go on top - version: "1.28.0" changes: - - description: Extract interface address from RPD_IFD_NOTIFICATION events into host.ip and related.ip, and avoid overwriting an already-populated message field. + - description: Extract interface address from RPD_IFD_NOTIFICATION events into host.ip and related.ip, map the interface index to observer.ingress.interface.id, and avoid overwriting an already-populated message field. type: enhancement link: https://github.com/elastic/integrations/pull/18927 - version: "1.27.2" diff --git a/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json b/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json index 2850bc87511..62769c71f19 100644 --- a/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json +++ b/packages/juniper_srx/data_stream/log/_dev/test/pipeline/test-system.log-expected.json @@ -1192,14 +1192,10 @@ }, "juniper": { "srx": { - "interface_index": "508", "log_type": "system", "message": " agg_parent 0 address #0 175.16.199.0", "process": "rpd", "state": "UpDown", - "system": { - "if_trace": "EVENT st0.60 index 508 agg_parent 0 address #0 175.16.199.0" - }, "tag": "RPD_IFD_NOTIFICATION" } }, @@ -1210,6 +1206,7 @@ "observer": { "ingress": { "interface": { + "id": "508", "name": "st0.60" } }, diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 5e14dffcb02..b4ffa253b93 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -221,6 +221,13 @@ processors: field: juniper.srx.interface_name target_field: observer.ingress.interface.name ignore_missing: true + # interface-index is the SNMP/IFD interface index, which maps to ECS observer.*.interface.id. + # Interface up/down notifications carry no flow direction, so it sits under ingress alongside the interface name. + - rename: + tag: rename_juniper_srx_interface_index_to_observer_ingress_interface_id + field: juniper.srx.interface_index + target_field: observer.ingress.interface.id + ignore_missing: true - rename: tag: rename_juniper_srx_source_zone_name_to_observer_ingress_zone_ebc8c681 field: juniper.srx.source_zone_name diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml index 80be322c0ff..5aa460ba60f 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml @@ -351,6 +351,12 @@ processors: source: >- ctx.juniper.srx.system = ctx.juniper.srx.system.entrySet().stream().collect(Collectors.toMap(e -> e.getKey().replace(' ', '_').replace('-', '_').toLowerCase(), e -> e.getValue().trim())); +# Drop juniper.srx.system.if_trace: the k-v extractor above surfaces it from IF_TRACE messages, but it duplicates `message` (message minus the "IF_TRACE: " prefix). +- remove: + tag: remove_juniper_srx_system_if_trace + field: juniper.srx.system.if_trace + ignore_missing: true + ####################### ## SRX System Fields ## ####################### @@ -412,7 +418,10 @@ processors: - "-" - "Not-Available" source: >- - ctx?.juniper?.srx?.system.entrySet().removeIf(entry -> params.values.contains(entry.getValue())); + ctx.juniper.srx.system.entrySet().removeIf(entry -> params.values.contains(entry.getValue())); + if (ctx.juniper.srx.system.isEmpty()) { + ctx.juniper.srx.remove('system'); + } # Copy `message_brief` into `message` if not already exists. This ensures unpased/unstructured data to be present in `message`. - set: diff --git a/packages/juniper_srx/data_stream/log/fields/fields.yml b/packages/juniper_srx/data_stream/log/fields/fields.yml index c13a0a07b13..3f5e6c1f2ef 100644 --- a/packages/juniper_srx/data_stream/log/fields/fields.yml +++ b/packages/juniper_srx/data_stream/log/fields/fields.yml @@ -423,8 +423,6 @@ type: integer - name: direction type: keyword - - name: if_trace - type: keyword - name: ike_version type: integer - name: local @@ -524,8 +522,6 @@ type: keyword - name: function_name type: keyword - - name: interface_index - type: keyword - name: operational_status type: keyword - name: snmp_interface_index diff --git a/packages/juniper_srx/docs/README.md b/packages/juniper_srx/docs/README.md index 3fb7c3dc042..d6466002bf7 100644 --- a/packages/juniper_srx/docs/README.md +++ b/packages/juniper_srx/docs/README.md @@ -591,7 +591,6 @@ The `log` data stream provides events from Juniper SRX devices. These logs inclu | juniper.srx.index | index | keyword | | juniper.srx.index1 | | keyword | | juniper.srx.index2 | | keyword | -| juniper.srx.interface_index | | keyword | | juniper.srx.ip_mon_reth_scan.trigger | | keyword | | juniper.srx.kern_arp_addr_change.ip | | ip | | juniper.srx.kern_arp_addr_change.mac1 | | keyword | @@ -660,7 +659,6 @@ The `log` data stream provides events from Juniper SRX devices. These logs inclu | juniper.srx.sub_category | sub category | keyword | | juniper.srx.system.aux_spi | | integer | | juniper.srx.system.direction | | keyword | -| juniper.srx.system.if_trace | | keyword | | juniper.srx.system.ike_version | | integer | | juniper.srx.system.local | | keyword | | juniper.srx.system.local_gateway | | ip |