azure: fix Grok processor error for firewall network rule logs (elastic#13882)

navnit-elastic · web-flow · commit 5fc75736d674 · 2025-05-12T18:14:13.000+09:30
This change adds an additional pattern to the Grok processor to correctly parse
"AzureFirewallNetworkRuleLog" in the "firewall_logs" data stream.
diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml
@@ -1,3 +1,8 @@
+- version: "1.23.2"
+  changes:
+    - description: Fix Grok processor error in ingest pipeline for `AzureFirewallNetworkRuleLog` in `azure.firewall_logs`.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/13882
 - version: "1.23.1"
   changes:
     - description: Map `azure.signinlogs.properties.location.state` field to `geo.region_name` instead of `geo.country_name`.
diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-applicationrules-raw.log-expected.json b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-applicationrules-raw.log-expected.json
@@ -489,4 +489,4 @@
             }
         }
     ]
-}
+}
diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-applicationrules-structured-raw.log-expected.json b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-applicationrules-structured-raw.log-expected.json
@@ -89,4 +89,4 @@
             }
         }
     ]
-}
+}
diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-dnsproxy-structured-raw.log-expected.json b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-dnsproxy-structured-raw.log-expected.json
@@ -90,4 +90,4 @@
             ]
         }
     ]
-}
+}
diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-dnsproxyrules-raw.log-expected.json b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-dnsproxyrules-raw.log-expected.json
@@ -248,4 +248,4 @@
             ]
         }
     ]
-}
+}
diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-natrule-structured-raw.log-expected.json b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-natrule-structured-raw.log-expected.json
@@ -79,4 +79,4 @@
             ]
         }
     ]
-}
+}
diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-networkrule-structured-raw.log-expected.json b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-networkrule-structured-raw.log-expected.json
@@ -81,4 +81,4 @@
             ]
         }
     ]
-}
+}
diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-networkrules-raw.log b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-networkrules-raw.log
@@ -7,3 +7,4 @@
 {"category":"AzureFirewallNetworkRule","operationName":"AzureFirewallNetworkRuleLog","properties":{"msg":"TCP request from 192.168.0.2:54854 to 175.16.199.1:1521. Action: alert. Signature: 2102649. IDS: SQL service_name buffer overflow attempt. Priority: 1. Classification: Attempted User Privilege Gain"},"resourceId":"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01","time":"2022-06-08T20:40:56.4525380Z"}
 {"category":"AzureFirewallNetworkRule","operationName":"AzureFirewallNetworkRuleLog","properties":{"msg":"HTTP request from 192.168.0.2:54314 to ocsp.sca1b.amazontrust.com:80. Url: ocsp.sca1b.amazontrust.com. Action: Deny. ThreatIntel: Bot Networks"},"resourceId":"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01","time":"2022-06-08T20:40:56.4525380Z"}
 {"category":"AzureFirewallNetworkRule","operationName":"AzureFirewallNetworkRuleLog","properties":{"msg":"ICMP request from 192.168.0.2: to 175.16.199.1:. Action: alert. Signature: 2100366. IDS: ICMP_INFO PING *NIX. Priority: 3. Classification: Misc activity"},"resourceId":"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01","time":"2022-06-08T20:40:56.4525380Z"}
+{"category":"AzureFirewallNetworkRule","operationName":"AzureFirewallNetworkRuleLog","properties":{"msg":"TCP request from 192.168.0.2:50306 to 89.160.20.156:3389. Action: Allow.. Rule Collection: Permit_RFC1918. Rule: Permit_RFC1918"},"resourceId":"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01","time":"2025-03-13T07:11:59.992099+00:00"}
diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-networkrules-raw.log-expected.json b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-networkrules-raw.log-expected.json
@@ -752,6 +752,93 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2025-03-13T07:11:59.992Z",
+            "azure": {
+                "firewall": {
+                    "action": "Allow.",
+                    "category": "AzureFirewallNetworkRule",
+                    "operation_name": "AzureFirewallNetworkRuleLog"
+                },
+                "resource": {
+                    "group": "TEST-FW-RG",
+                    "id": "/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01",
+                    "name": "TEST-FW01",
+                    "provider": "MICROSOFT.NETWORK/AZUREFIREWALLS"
+                },
+                "subscription_id": "23103928-B2CF-472A-8CDB-0146E2849129"
+            },
+            "cloud": {
+                "account": {
+                    "id": "23103928-B2CF-472A-8CDB-0146E2849129"
+                },
+                "provider": "azure"
+            },
+            "destination": {
+                "address": "89.160.20.156",
+                "as": {
+                    "number": 29518,
+                    "organization": {
+                        "name": "Bredband2 AB"
+                    }
+                },
+                "geo": {
+                    "city_name": "Linköping",
+                    "continent_name": "Europe",
+                    "country_iso_code": "SE",
+                    "country_name": "Sweden",
+                    "location": {
+                        "lat": 58.4167,
+                        "lon": 15.6167
+                    },
+                    "region_iso_code": "SE-E",
+                    "region_name": "Östergötland County"
+                },
+                "ip": "89.160.20.156",
+                "port": 3389
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "network"
+                ],
+                "kind": "event",
+                "original": "{\"category\":\"AzureFirewallNetworkRule\",\"operationName\":\"AzureFirewallNetworkRuleLog\",\"properties\":{\"msg\":\"TCP request from 192.168.0.2:50306 to 89.160.20.156:3389. Action: Allow.. Rule Collection: Permit_RFC1918. Rule: Permit_RFC1918\"},\"resourceId\":\"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01\",\"time\":\"2025-03-13T07:11:59.992099+00:00\"}",
+                "type": [
+                    "connection"
+                ]
+            },
+            "network": {
+                "iana_number": "6",
+                "transport": "tcp"
+            },
+            "observer": {
+                "name": "TEST-FW01",
+                "product": "Network Firewall",
+                "type": "firewall",
+                "vendor": "Azure"
+            },
+            "related": {
+                "ip": [
+                    "192.168.0.2",
+                    "89.160.20.156"
+                ]
+            },
+            "rule": {
+                "name": "Permit_RFC1918",
+                "ruleset": "Permit_RFC1918"
+            },
+            "source": {
+                "address": "192.168.0.2",
+                "ip": "192.168.0.2",
+                "port": 50306
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
-}
+}
diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-sdh3075-raw.log-expected.json b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-sdh3075-raw.log-expected.json
@@ -91,4 +91,4 @@
             ]
         }
     ]
-}
+}
diff --git a/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml
@@ -186,6 +186,7 @@ processors:
       patterns:
         - "^%{DATA:azure.firewall.proto} request from %{IPORHOST:source.address}:%{NUMBER:source.port:long} to %{IPORHOST:destination.address}:%{NUMBER:destination.port:long}. Action: %{DATA:azure.firewall.action}. $" 
         - "^%{DATA:azure.firewall.proto} request from %{IPORHOST:source.address}:%{NUMBER:source.port:long} to %{IPORHOST:destination.address}:%{NUMBER:destination.port:long}. Action: %{DATA:azure.firewall.action}. Policy: %{DATA:azure.firewall.policy}. Rule Collection Group: %{DATA:azure.firewall.rule_collection_group}. Rule Collection: %{DATA:rule.ruleset}. Rule: %{DATA:rule.name}$"
+        - "^%{DATA:azure.firewall.proto} request from %{IPORHOST:source.address}:%{NUMBER:source.port:long} to %{IPORHOST:destination.address}:%{NUMBER:destination.port:long}. Action: %{DATA:azure.firewall.action}. Rule Collection: %{DATA:rule.ruleset}. Rule: %{DATA:rule.name}$"
         - "^%{DATA:azure.firewall.proto} Type=%{DATA:azure.firewall.icmp.request.code} request from %{IPORHOST:source.address} to %{IPORHOST:destination.address}. Action: %{DATA:azure.firewall.action}. $"
         - "^%{DATA:azure.firewall.proto} request from %{IPORHOST:source.address}:%{NUMBER:source.port:long} to %{IPORHOST:destination.address}:%{NUMBER:destination.port:long} was DNAT'ed to %{IP:destination.nat.ip}:%{NUMBER:destination.nat.port:long}$"
         - "^%{DATA:azure.firewall.proto} request from %{IPORHOST:source.address}:%{NUMBER:source.port:long} to %{IPORHOST:destination.address}:%{NUMBER:destination.port:long} was DNAT'ed to %{IP:destination.nat.ip}:%{NUMBER:destination.nat.port:long}. Policy: %{DATA:azure.firewall.policy}. Rule Collection Group: %{DATA:azure.firewall.rule_collection_group}. Rule Collection: %{DATA:rule.ruleset}. Rule: %{DATA:rule.name}$"
diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml
@@ -1,6 +1,6 @@
 name: azure
 title: Azure Logs
-version: "1.23.1"
+version: "1.23.2"
 description: This Elastic integration collects logs from Azure
 type: integration
 icons:

Original file line number	Diff line number	Diff line change
`@@ -489,4 +489,4 @@`
`489`	`489`	`}`
`490`	`490`	`}`
`491`	`491`	`]`
`492`		`-}`
	`492`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -89,4 +89,4 @@`
`89`	`89`	`}`
`90`	`90`	`}`
`91`	`91`	`]`
`92`		`-}`
	`92`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -90,4 +90,4 @@`
`90`	`90`	`]`
`91`	`91`	`}`
`92`	`92`	`]`
`93`		`-}`
	`93`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -248,4 +248,4 @@`
`248`	`248`	`]`
`249`	`249`	`}`
`250`	`250`	`]`
`251`		`-}`
	`251`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -79,4 +79,4 @@`
`79`	`79`	`]`
`80`	`80`	`}`
`81`	`81`	`]`
`82`		`-}`
	`82`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -81,4 +81,4 @@`
`81`	`81`	`]`
`82`	`82`	`}`
`83`	`83`	`]`
`84`		`-}`
	`84`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -91,4 +91,4 @@`
`91`	`91`	`]`
`92`	`92`	`}`
`93`	`93`	`]`
`94`		`-}`
	`94`	`+}`