[elasticsearch] Add security_stats data stream

packages/elasticsearch/_dev/build/docs/README.md

@@ -147,6 +147,16 @@ NOTE: The indices stats are node-specific. That means for example the total numb
 
 {{fields "pending_tasks"}}
 
+### Security stats
+
+`security_stats` interrogates the [Security Stats API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-security-get-stats) endpoint to fetch per-node security statistics. Currently this surfaces the Document Level Security (DLS) bitset cache metrics for each node, which are useful for understanding the memory footprint and cache efficiency of DLS enforcement.
+
+This data stream requires Elasticsearch 9.2.0 or later. On earlier versions the Security Stats API is not available and no metrics will be collected.
+
+{{event "security_stats"}}
+
+{{fields "security_stats"}}
+
 ### Shard
 
 `shard` interrogates the

packages/elasticsearch/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.22.0"
+  changes:
+    - description: Add security_stats data stream to collect per-node Document Level Security (DLS) bitset cache metrics from the Security Stats API (requires Elasticsearch 9.2+).
+      type: enhancement
+      link: https://github.com/elastic/integrations/issues/18680
 - version: "1.21.1"
   changes:
     - description: Bugfix for `querylog` new data stream (docs and dashboard)

packages/elasticsearch/data_stream/security_stats/_dev/deploy/docker/config/elasticsearch.yml

@@ -0,0 +1,9 @@
+network.host: ""
+transport.host: "127.0.0.1"
+http.host: "0.0.0.0"
+indices.id_field_data.enabled: true
+xpack.license.self_generated.type: "trial"
+xpack.security.enabled: true
+xpack.security.authc.api_key.enabled: true
+xpack.security.http.ssl.enabled: false
+ingest.geoip.downloader.enabled: false

packages/elasticsearch/data_stream/security_stats/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,17 @@
+services:
+  elasticsearch:
+    # The Security Stats API (GET /_security/stats) used by this data stream
+    # is only available in Elasticsearch 9.2.0+, so this data stream pins its
+    # own deploy version instead of using the package-level 8.5.0 service.
+    image: "docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-9.2.0}"
+    environment:
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "ELASTIC_PASSWORD=changeme"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "elastic:changeme", "http://127.0.0.1:9200/"]
+      retries: 300
+      interval: 1s
+    volumes:
+      - ./config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+    ports:
+      - "127.0.0.1:9201:9200"

packages/elasticsearch/data_stream/security_stats/_dev/deploy/variants.yml

@@ -0,0 +1,4 @@
+variants:
+  elasticsearch_9.2.0:
+    ELASTIC_VERSION: 9.2.0
+default: elasticsearch_9.2.0

packages/elasticsearch/data_stream/security_stats/_dev/test/system/test-default-config.yml

@@ -0,0 +1,11 @@
+type: elasticsearch/metrics
+dataset: elasticsearch.stack_monitoring.security_stats
+skip:
+  reason: "The security_stats metricset requires Elastic Agent 9.3.5+/9.4.2+/9.5.0+; package CI runs the minimum supported stack (8.10.1), where the metricset is absent and no documents are collected."
+  link: https://github.com/elastic/integrations/issues/18680
+vars:
+  hosts:
+    - "http://{{Hostname}}:9200"
+  username: elastic
+  password: changeme
+data_stream: ~

packages/elasticsearch/data_stream/security_stats/agent/stream/stream.yml.hbs

@@ -0,0 +1,37 @@
+metricsets: ["security_stats"]
+hosts:
+{{#each hosts}}
+  - {{this}}
+{{/each}}
+scope: {{scope}}
+{{#if username}}
+username: {{username}}
+{{/if}}
+{{#if password}}
+password: {{password}}
+{{/if}}
+{{#if api_key}}
+api_key: {{api_key}}
+{{/if}}
+period: {{period}}
+{{#if timeout}}
+timeout: {{timeout}}
+{{/if}}
+{{#if ssl}}
+ssl: {{ssl}}
+{{/if}}
+tags:
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#if leaderelection }}
+{{#if condition }}
+condition: ${kubernetes_leaderelection.leader} == true and {{ condition }}
+{{ else }}
+condition: ${kubernetes_leaderelection.leader} == true
+{{/if}}
+{{ else }}
+{{#if condition }}
+condition: {{ condition }}
+{{/if}}
+{{/if}}

packages/elasticsearch/data_stream/security_stats/fields/base-fields.yml

@@ -0,0 +1,9 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.

packages/elasticsearch/data_stream/security_stats/fields/ecs.yml

@@ -0,0 +1,26 @@
+- name: '@timestamp'
+  external: ecs
+- name: ecs.version
+  external: ecs
+- name: event.dataset
+  external: ecs
+- name: event.duration
+  external: ecs
+- name: event.module
+  external: ecs
+- name: host.name
+  external: ecs
+  dimension: true
+- name: service.address
+  type: keyword
+  dimension: true
+  description: Service address
+- name: service.type
+  external: ecs
+- name: service.name
+  external: ecs
+- name: error.message
+  external: ecs
+- name: agent.id
+  external: ecs
+  dimension: true

packages/elasticsearch/data_stream/security_stats/fields/fields.yml

@@ -0,0 +1,48 @@
+- name: elasticsearch.security.stats
+  type: group
+  description: |
+    Per-node security statistics collected from the Elasticsearch Security Stats
+    API (GET /_security/stats). Available since Elasticsearch 9.2.
+  fields:
+    - name: dls.cache
+      type: group
+      description: |
+        Per-node cache used to materialize the bitsets that enforce Document
+        Level Security (DLS) queries.
+      fields:
+        - name: entries.count
+          type: long
+          metric_type: gauge
+          description: |
+            Current number of cached entries on this node.
+        - name: memory.bytes
+          type: long
+          format: bytes
+          metric_type: gauge
+          description: |
+            Current memory consumed by the DLS cache on this node, in bytes.
+        - name: hits.count
+          type: long
+          metric_type: counter
+          description: |
+            Number of cache lookups served from the cache since node startup.
+        - name: misses.count
+          type: long
+          metric_type: counter
+          description: |
+            Number of cache lookups that had to materialize an entry since node startup.
+        - name: evictions.count
+          type: long
+          metric_type: counter
+          description: |
+            Number of entries evicted (due to size limit or expiration) since node startup.
+        - name: hits.time.ms
+          type: long
+          metric_type: counter
+          description: |
+            Cumulative time spent serving cache hits since node startup, in milliseconds.
+        - name: misses.time.ms
+          type: long
+          metric_type: counter
+          description: |
+            Cumulative time spent materializing entries on cache misses since node startup, in milliseconds.

packages/elasticsearch/data_stream/security_stats/fields/package-fields.yml

@@ -0,0 +1,58 @@
+- name: timestamp
+  type: alias
+  path: '@timestamp'
+- name: cluster_uuid
+  type: alias
+  path: elasticsearch.cluster.id
+- name: source_node
+  type: group
+  fields:
+    - name: uuid
+      type: alias
+      path: elasticsearch.node.id
+    - name: name
+      type: alias
+      path: elasticsearch.node.name
+- name: elasticsearch
+  type: group
+  fields:
+    - name: cluster.name
+      type: keyword
+      description: |
+        Elasticsearch cluster name.
+    - name: cluster.id
+      type: keyword
+      description: |
+        Elasticsearch cluster id.
+    - name: cluster.state.id
+      type: keyword
+      description: |
+        Elasticsearch state id.
+    - name: node
+      type: group
+      fields:
+        - name: id
+          type: keyword
+          dimension: true
+          description: |
+            Node ID
+        - name: name
+          type: keyword
+          description: |
+            Node name.
+        - name: roles
+          type: keyword
+          description: |
+            Node roles.
+        - name: version
+          type: keyword
+          description: |
+            Elasticsearch version reported by the node (for example, `9.2.0`).
+        - name: master
+          type: boolean
+          description: |
+            Is the node the master node?
+        - name: mlockall
+          type: boolean
+          description: |
+            Is mlockall enabled on the node?

packages/elasticsearch/data_stream/security_stats/manifest.yml

@@ -0,0 +1,25 @@
+type: metrics
+title: Elasticsearch security stats metrics
+dataset: elasticsearch.stack_monitoring.security_stats
+elasticsearch:
+  index_template:
+    mappings:
+      dynamic: false
+streams:
+  - input: elasticsearch/metrics
+    title: Security stats metrics
+    description: Collect Elasticsearch per-node security stats (requires Elasticsearch 9.2+)
+    vars:
+      - name: period
+        type: text
+        title: Period
+        multi: false
+        required: true
+        show_user: true
+        default: 10s
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        required: false
+        show_user: false

packages/elasticsearch/data_stream/security_stats/sample_event.json

@@ -0,0 +1,106 @@
+{
+    "@timestamp": "2026-04-27T20:00:00.000Z",
+    "agent": {
+        "hostname": "docker-fleet-agent",
+        "name": "docker-fleet-agent",
+        "id": "60e15e27-7080-4c28-9900-5a087c2ff74c",
+        "type": "metricbeat",
+        "ephemeral_id": "2b6da727-313f-41fc-84af-3cd928f265c1",
+        "version": "9.2.0"
+    },
+    "elastic_agent": {
+        "id": "60e15e27-7080-4c28-9900-5a087c2ff74c",
+        "version": "9.2.0",
+        "snapshot": true
+    },
+    "elasticsearch": {
+        "cluster": {
+            "name": "docker-cluster",
+            "id": "WocBBA0QRma0sGpdQ7vLfQ"
+        },
+        "node": {
+            "id": "f5i3v9hMT_q__q6B9WOo5A",
+            "name": "instance-0000000019",
+            "roles": [
+                "data_hot",
+                "ingest"
+            ],
+            "version": "9.2.0"
+        },
+        "security": {
+            "stats": {
+                "dls": {
+                    "cache": {
+                        "entries": {
+                            "count": 12
+                        },
+                        "memory": {
+                            "bytes": 4096
+                        },
+                        "hits": {
+                            "count": 8421,
+                            "time": {
+                                "ms": 51
+                            }
+                        },
+                        "misses": {
+                            "count": 137,
+                            "time": {
+                                "ms": 219
+                            }
+                        },
+                        "evictions": {
+                            "count": 4
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "ecs": {
+        "version": "8.11.0"
+    },
+    "service": {
+        "address": "http://elasticsearch:9200",
+        "name": "elasticsearch",
+        "type": "elasticsearch"
+    },
+    "data_stream": {
+        "namespace": "default",
+        "type": "metrics",
+        "dataset": "elasticsearch.stack_monitoring.security_stats"
+    },
+    "host": {
+        "hostname": "docker-fleet-agent",
+        "os": {
+            "kernel": "5.11.10-arch1-1",
+            "codename": "Core",
+            "name": "CentOS Linux",
+            "type": "linux",
+            "family": "redhat",
+            "version": "7 (Core)",
+            "platform": "centos"
+        },
+        "containerized": true,
+        "ip": [
+            "172.18.0.7"
+        ],
+        "name": "docker-fleet-agent",
+        "id": "8979eb4aa312c3dccea3823dd92f92f5",
+        "mac": [
+            "02:42:ac:12:00:07"
+        ],
+        "architecture": "x86_64"
+    },
+    "metricset": {
+        "period": 10000,
+        "name": "security_stats"
+    },
+    "event": {
+        "duration": 115000,
+        "agent_id_status": "verified",
+        "ingested": "2026-04-27T20:00:01.000Z",
+        "module": "elasticsearch",
+        "dataset": "elasticsearch.stack_monitoring.security_stats"
+    }
+}