Count number of documents with at least one ignored field

docs/changelog/109146.yaml

@@ -0,0 +1,6 @@
+pr: 109146
+summary: Count number of documents with at least one ignored field
+area: Logs
+type: feature
+issues:
+ - 108092

docs/reference/cluster/stats.asciidoc

@@ -227,6 +227,20 @@ space of deleted Lucene documents when a segment is merged.
 `total_size_in_bytes`::
 (integer)
 Total size in bytes across all primary shards assigned to selected nodes.
+
+`docs_with_ignored_fields`::
+(integer)
+Total number of documents including at least one ignored field.
++
+This number is based on documents in Lucene segments and does not take
+into account deleted documents.
+
+`sum_doc_freq_terms_ignored_fields`::
+(integer)
+Sum of the terms frequencies for the _ignored field over all documents.
++
+This number is based on documents in Lucene segments and does not take
+into account deleted documents.
 =====
 
 `store`::

docs/reference/rest-api/common-parms.asciidoc

@@ -556,6 +556,11 @@ Size of the index in <<byte-units, byte units>>.
 
 `translog`::
 <<index-modules-translog,Translog>> statistics.
+
+`ignored_field`::
+Includes statistics about the number of documents including the number of total documents,
+the number of documents with at least one ignored field and the sum of term frequencies
+for the `_ignored` field over all documents in the index.
 --
 end::index-metric[]
 

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/110_ignored_field.yml

@@ -0,0 +1,176 @@
+setup:
+  - do:
+      indices.create:
+        index: test1
+        body:
+          settings:
+            number_of_shards: 3
+            number_of_replicas: 0
+          mappings:
+            properties:
+              email:
+                type: keyword
+                ignore_above: 15
+              date_of_birth:
+                type: date
+                format: "dd-MM-yyyy"
+                ignore_malformed: true
+              ip_address:
+                type: ip
+                ignore_malformed: true
+
+  - do:
+      indices.create:
+        index: test2
+        body:
+          settings:
+            number_of_shards: 3
+            number_of_replicas: 0
+          mappings:
+            properties:
+              email:
+                type: keyword
+                ignore_above: 15
+              date_of_birth:
+                type: date
+                format: "dd-MM-yyyy"
+                ignore_malformed: true
+              ip_address:
+                type: ip
+                ignore_malformed: true
+
+  - do:
+      indices.create:
+        index: test3
+        body:
+          settings:
+            number_of_shards: 3
+            number_of_replicas: 0
+          mappings:
+            properties:
+              email:
+                type: keyword
+                ignore_above: 15
+              date_of_birth:
+                type: date
+                format: "dd-MM-yyyy"
+                ignore_malformed: true
+              ip_address:
+                type: ip
+                ignore_malformed: true
+
+  - do:
+      bulk:
+        index: test1
+        refresh: true
+        body:
+          - { "index": { "_id": "001" } }
+          - {"email": "[email protected]", "date_of_birth": "10-11-1992", "ip_address": "117.12.45.79" }
+          - { "index": { "_id": "002" } }
+          - { "email": "[email protected]", "date_of_birth": "12-04-1993", "ip_address": "178.22.231.24" }
+
+  - do:
+      bulk:
+        index: test2
+        refresh: true
+        body:
+          - { "index": { "_id": "003" } }
+          - { "email": "[email protected]", "date_of_birth": "09-02-1990", "ip_address": "117.12.45.79" }
+          - { "index": { "_id": "004" } }
+          - { "email": "[email protected]", "date_of_birth": "12-24-1991", "ip_address": "133.45.123.812" }
+
+  - do:
+      bulk:
+        index: test3
+        refresh: false # document not flushed to Lucene segment
+        body:
+          - { "index": { "_id": "005" } }
+          - { "email": "[email protected]", "date_of_birth": "04-05-1992", "ip_address": "123.77.18.488" }
+
+---
+"ignored fields stats with docs metric":
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: GET
+          path: /{index}/_stats
+          capabilities: [ ignored_field_doc_stats_counter ]
+      reason: "Counting docs with ignored fields required"
+
+  - do:
+      indices.stats:
+        metric: [ ignored_field ]
+
+  - match: { _all.primaries.ignored_field.total_docs: 4 }
+  - match: { _all.primaries.ignored_field.docs_with_ignored_fields: 2 }
+  - match: { _all.primaries.ignored_field.sum_doc_freq_terms_ignored_fields: 3 }
+  - match: { _all.total.ignored_field.total_docs: 4 }
+  - match: { _all.total.ignored_field.docs_with_ignored_fields: 2 }
+  - match: { _all.total.ignored_field.sum_doc_freq_terms_ignored_fields: 3 }
+
+  - match: { indices.test1.primaries.ignored_field.total_docs: 2 }
+  - match: { indices.test1.primaries.ignored_field.docs_with_ignored_fields: 0 }
+  - match: { indices.test1.primaries.ignored_field.sum_doc_freq_terms_ignored_fields: 0 }
+  - match: { indices.test1.total.ignored_field.total_docs: 2 }
+  - match: { indices.test1.total.ignored_field.docs_with_ignored_fields: 0 }
+  - match: { indices.test1.total.ignored_field.sum_doc_freq_terms_ignored_fields: 0 }
+
+  - match: { indices.test2.primaries.ignored_field.total_docs: 2 }
+  - match: { indices.test2.primaries.ignored_field.docs_with_ignored_fields: 2 }
+  - match: { indices.test2.primaries.ignored_field.sum_doc_freq_terms_ignored_fields: 3 }
+  - match: { indices.test2.total.ignored_field.total_docs: 2 }
+  - match: { indices.test2.total.ignored_field.docs_with_ignored_fields: 2 }
+  - match: { indices.test2.total.ignored_field.sum_doc_freq_terms_ignored_fields: 3 }
+
+  # Not refreshed
+  - match: { indices.test3.primaries.ignored_field.total_docs: 0 }
+  - match: { indices.test3.primaries.ignored_field.docs_with_ignored_fields: 0 }
+  - match: { indices.test3.primaries.ignored_field.sum_doc_freq_terms_ignored_fields: 0 }
+  - match: { indices.test3.total.ignored_field.total_docs: 0 }
+  - match: { indices.test3.total.ignored_field.docs_with_ignored_fields: 0 }
+  - match: { indices.test3.total.ignored_field.sum_doc_freq_terms_ignored_fields: 0 }
+
+---
+"ignored fields stats with docs metric with all metrics":
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: GET
+          path: /{index}/_stats
+          capabilities: [ ignored_field_doc_stats_counter ]
+      reason: "Counting docs with ignored fields required"
+
+  - do:
+      indices.stats:
+        index: test*
+        metric: _all
+
+  - match: { _all.primaries.ignored_field.total_docs: 4 }
+  - match: { _all.primaries.ignored_field.docs_with_ignored_fields: 2 }
+  - match: { _all.primaries.ignored_field.sum_doc_freq_terms_ignored_fields: 3 }
+  - match: { _all.total.ignored_field.total_docs: 4 }
+  - match: { _all.total.ignored_field.docs_with_ignored_fields: 2 }
+  - match: { _all.total.ignored_field.sum_doc_freq_terms_ignored_fields: 3 }
+
+  - match: { indices.test1.primaries.ignored_field.total_docs: 2 }
+  - match: { indices.test1.primaries.ignored_field.docs_with_ignored_fields: 0 }
+  - match: { indices.test1.primaries.ignored_field.sum_doc_freq_terms_ignored_fields: 0 }
+  - match: { indices.test1.total.ignored_field.total_docs: 2 }
+  - match: { indices.test1.total.ignored_field.docs_with_ignored_fields: 0 }
+  - match: { indices.test1.total.ignored_field.sum_doc_freq_terms_ignored_fields: 0 }
+
+  - match: { indices.test2.primaries.ignored_field.total_docs: 2 }
+  - match: { indices.test2.primaries.ignored_field.docs_with_ignored_fields: 2 }
+  - match: { indices.test2.primaries.ignored_field.sum_doc_freq_terms_ignored_fields: 3 }
+  - match: { indices.test2.total.ignored_field.total_docs: 2 }
+  - match: { indices.test2.total.ignored_field.docs_with_ignored_fields: 2 }
+  - match: { indices.test2.total.ignored_field.sum_doc_freq_terms_ignored_fields: 3 }
+
+  # Not refreshed
+  - match: { indices.test3.primaries.ignored_field.total_docs: 0 }
+  - match: { indices.test3.primaries.ignored_field.docs_with_ignored_fields: 0 }
+  - match: { indices.test3.primaries.ignored_field.sum_doc_freq_terms_ignored_fields: 0 }
+  - match: { indices.test3.total.ignored_field.total_docs: 0 }
+  - match: { indices.test3.total.ignored_field.docs_with_ignored_fields: 0 }
+  - match: { indices.test3.total.ignored_field.sum_doc_freq_terms_ignored_fields: 0 }
+

server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java

@@ -834,7 +834,8 @@ public void testFlagOrdinalOrder() {
             Flag.Bulk,
             Flag.Shards,
             Flag.Mappings,
-            Flag.DenseVector };
+            Flag.DenseVector,
+            Flag.IgnoredFieldStats };
 
         assertThat(flags.length, equalTo(Flag.values().length));
         for (int i = 0; i < flags.length; i++) {
@@ -1000,6 +1001,7 @@ private static void set(Flag flag, IndicesStatsRequestBuilder builder, boolean s
                 // We don't actually expose shards in IndexStats, but this test fails if it isn't handled
                 builder.request().flags().set(Flag.Shards, set);
             case DenseVector -> builder.setDenseVector(set);
+            case IgnoredFieldStats -> builder.setIncludeIgnoredFieldsStats(set);
             default -> fail("new flag? " + flag);
         }
     }
@@ -1046,6 +1048,8 @@ private static boolean isSet(Flag flag, CommonStats response) {
                 return response.getNodeMappings() != null;
             case DenseVector:
                 return response.getDenseVectorStats() != null;
+            case IgnoredFieldStats:
+                return response.getIgnoredFieldStats() != null;
             default:
                 fail("new flag? " + flag);
                 return false;

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -181,6 +181,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_GOOGLE_AI_STUDIO_COMPLETION_ADDED = def(8_672_00_0);
     public static final TransportVersion WATCHER_REQUEST_TIMEOUTS = def(8_673_00_0);
     public static final TransportVersion ML_INFERENCE_ENHANCE_DELETE_ENDPOINT = def(8_674_00_0);
+    public static final TransportVersion IGNORED_FIELDS_STATS = def(8_675_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.shard.DenseVectorStats;
 import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.index.shard.IgnoredFieldStats;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.shard.ShardCountStats;
@@ -109,6 +110,8 @@ public class CommonStats implements Writeable, ToXContentFragment {
 
     @Nullable
     public DenseVectorStats denseVectorStats;
+    @Nullable
+    public IgnoredFieldStats ignoredFieldStats;
 
     public CommonStats() {
         this(CommonStatsFlags.NONE);
@@ -139,6 +142,7 @@ public CommonStats(CommonStatsFlags flags) {
                 case Shards -> shards = new ShardCountStats();
                 case Mappings -> nodeMappings = new NodeMappingStats();
                 case DenseVector -> denseVectorStats = new DenseVectorStats();
+                case IgnoredFieldStats -> ignoredFieldStats = new IgnoredFieldStats();
                 default -> throw new IllegalStateException("Unknown Flag: " + flag);
             }
         }
@@ -182,6 +186,7 @@ public static CommonStats getShardLevelStats(IndicesQueryCache indicesQueryCache
                         // Setting to 1 because the single IndexShard passed to this method implies 1 shard
                         stats.shards = new ShardCountStats(1);
                     case DenseVector -> stats.denseVectorStats = indexShard.denseVectorStats();
+                    case IgnoredFieldStats -> stats.ignoredFieldStats = indexShard.ignoredFieldStats();
                     default -> throw new IllegalStateException("Unknown or invalid flag for shard-level stats: " + flag);
                 }
             } catch (AlreadyClosedException e) {
@@ -219,6 +224,9 @@ public CommonStats(StreamInput in) throws IOException {
         if (in.getTransportVersion().onOrAfter(VERSION_SUPPORTING_DENSE_VECTOR_STATS)) {
             denseVectorStats = in.readOptionalWriteable(DenseVectorStats::new);
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.IGNORED_FIELDS_STATS)) {
+            ignoredFieldStats = in.readOptionalWriteable(IgnoredFieldStats::new);
+        }
     }
 
     @Override
@@ -249,6 +257,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(VERSION_SUPPORTING_DENSE_VECTOR_STATS)) {
             out.writeOptionalWriteable(denseVectorStats);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.IGNORED_FIELDS_STATS)) {
+            out.writeOptionalWriteable(ignoredFieldStats);
+        }
     }
 
     @Override
@@ -275,7 +286,8 @@ public boolean equals(Object o) {
             && Objects.equals(bulk, that.bulk)
             && Objects.equals(shards, that.shards)
             && Objects.equals(nodeMappings, that.nodeMappings)
-            && Objects.equals(denseVectorStats, that.denseVectorStats);
+            && Objects.equals(denseVectorStats, that.denseVectorStats)
+            && Objects.equals(ignoredFieldStats, that.ignoredFieldStats);
     }
 
     @Override
@@ -300,7 +312,8 @@ public int hashCode() {
             bulk,
             shards,
             nodeMappings,
-            denseVectorStats
+            denseVectorStats,
+            ignoredFieldStats
         );
     }
 
@@ -465,6 +478,14 @@ public void add(CommonStats stats) {
         } else {
             denseVectorStats.add(stats.getDenseVectorStats());
         }
+        if (ignoredFieldStats == null) {
+            if (stats.getIgnoredFieldStats() != null) {
+                ignoredFieldStats = new IgnoredFieldStats();
+                ignoredFieldStats.add(stats.ignoredFieldStats);
+            }
+        } else {
+            ignoredFieldStats.add(stats.getIgnoredFieldStats());
+        }
     }
 
     @Nullable
@@ -567,6 +588,11 @@ public DenseVectorStats getDenseVectorStats() {
         return denseVectorStats;
     }
 
+    @Nullable
+    public IgnoredFieldStats getIgnoredFieldStats() {
+        return ignoredFieldStats;
+    }
+
     /**
      * Utility method which computes total memory by adding
      * FieldData, PercolatorCache, Segments (index writer, version map)
@@ -609,6 +635,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         addIfNonNull(builder, params, bulk);
         addIfNonNull(builder, params, nodeMappings);
         addIfNonNull(builder, params, denseVectorStats);
+        addIfNonNull(builder, params, ignoredFieldStats);
         return builder;
     }
 

server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java

@@ -223,7 +223,8 @@ public enum Flag {
         Bulk("bulk", 17),
         Shards("shard_stats", 18),
         Mappings("mappings", 19),
-        DenseVector("dense_vector", 20);
+        DenseVector("dense_vector", 20),
+        IgnoredFieldStats("ignored_field", 21);
 
         private final String restName;
         private final int index;

server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java

@@ -275,10 +275,19 @@ public IndicesStatsRequest denseVector(boolean denseVector) {
         return this;
     }
 
+    public IndicesStatsRequest ignoredFieldsStats(boolean ignoredField) {
+        flags.set(Flag.IgnoredFieldStats, ignoredField);
+        return this;
+    }
+
     public boolean denseVector() {
         return flags.isSet(Flag.DenseVector);
     }
 
+    public boolean ignoredField() {
+        return flags.isSet(Flag.IgnoredFieldStats);
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);

server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java

@@ -159,4 +159,9 @@ public IndicesStatsRequestBuilder setDenseVector(boolean denseVector) {
         request.denseVector(denseVector);
         return this;
     }
+
+    public IndicesStatsRequestBuilder setIncludeIgnoredFieldsStats(boolean includeIgnoredFieldsStats) {
+        request.ignoredFieldsStats(includeIgnoredFieldsStats);
+        return this;
+    }
 }