Enable synthetic source on normalized keyword mappings

docs/changelog/126623.yaml

@@ -0,0 +1,16 @@
+pr: 126623
+summary: Enable synthetic source on normalized keyword mappings
+area: Mapping
+type: "breaking, enhancement"
+issues:
+ - 124369
+ - 121358
+breaking:
+  title: Enable synthetic source on normalized keyword mappings
+  area: Mapping
+  details: |-
+    This changes the default behavior for Synthetic Source on keyword fields using normalizers. Prior to this change, normalized keywords were always stored to allow returning the non-normalized values. Under this change, such field will NOT be stored (i.e they will be synthesized from the index when returning source, like all other synthetic source fields).  This should result in considerable space improvement for this use case.
+    Users can opt out of this behavior on a per-field basis by setting `synthetic_source_keep` to `all` on the field.
+  impact: "By default, normalized keyword fields in synthetic source indices will\
+    \ no longer return the non-normalized value in the source."
+  notable: false

rest-api-spec/build.gradle

@@ -89,4 +89,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("indices.create/21_synthetic_source_stored/field param - keep root array", "Synthetic source keep arrays now stores leaf arrays natively")
   task.skipTest("cluster.info/30_info_thread_pool/Cluster HTTP Info", "The search_throttled thread pool has been removed")
   task.skipTest("synonyms/80_synonyms_from_index/Fail loading synonyms from index if synonyms_set doesn't exist", "Synonyms do no longer fail if the synonyms_set doesn't exist")
+  task.skipTest("mget/90_synthetic_source/keyword with normalizer", "Normalized keywords now use synthetic source")
 })

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml

@@ -6,7 +6,7 @@ setup:
 ---
 keyword:
   - requires:
-      cluster_features: ["gte_v8.4.0"]
+      cluster_features: [ "gte_v8.4.0" ]
       reason: introduced in 8.4.0
 
   - do:
@@ -23,15 +23,15 @@ keyword:
 
   - do:
       index:
-        index:   test
-        id:      1
+        index: test
+        id: 1
         body:
           kwd: foo
 
   - do:
       index:
-        index:   test
-        id:      2
+        index: test
+        id: 2
         body:
           kwd: bar
 
@@ -40,21 +40,24 @@ keyword:
       mget:
         index: test
         body:
-          ids:    [1, 2]
-  - match: {docs.0._index: "test"}
-  - match: {docs.0._id: "1"}
+          ids: [ 1, 2 ]
+  - match: { docs.0._index: "test" }
+  - match: { docs.0._id: "1" }
   - match:
       docs.0._source:
         kwd: foo
 
-  - match: {docs.1._index: "test"}
-  - match: {docs.1._id: "2"}
+  - match: { docs.1._index: "test" }
+  - match: { docs.1._id: "2" }
   - match:
       docs.1._source:
         kwd: bar
 
 ---
 keyword with normalizer:
+  - requires:
+      cluster_features: "mapper.keyword.keyword_normalizer_synthetic_source"
+      reason: "Behavior changed in #126623"
   - do:
       indices.create:
         index: test-keyword-with-normalizer
@@ -113,7 +116,94 @@ keyword with normalizer:
       mget:
         index: test-keyword-with-normalizer
         body:
-          ids:    [ 1, 2, 3 ]
+          ids: [ 1, 2, 3 ]
+  - match: { docs.0._index: "test-keyword-with-normalizer" }
+  - match: { docs.0._id: "1" }
+  - match:
+      docs.0._source:
+        keyword: "the quick brown fox jumps over the lazy dog"
+        keyword_with_ignore_above: "the Quick Brown Fox jumps over the lazy Dog"
+        keyword_without_doc_values: "the Quick Brown Fox jumps over the lazy Dog"
+
+  - match: { docs.1._index: "test-keyword-with-normalizer" }
+  - match: { docs.1._id: "2" }
+  - match:
+      docs.1._source:
+        keyword: "the five boxing wizards jump quickly"
+        keyword_with_ignore_above: "The five BOXING wizards jump Quickly"
+        keyword_without_doc_values: "The five BOXING wizards jump Quickly"
+
+  - match: { docs.2._index: "test-keyword-with-normalizer" }
+  - match: { docs.2._id: "3" }
+  - match:
+      docs.2._source:
+        keyword: [ "do or do not, there is no try", "may the force be with you!" ]
+        keyword_with_ignore_above: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ]
+        keyword_without_doc_values: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ]
+
+---
+keyword with normalizer, source keep mode all:
+  - do:
+      indices.create:
+        index: test-keyword-with-normalizer
+        body:
+          settings:
+            analysis:
+              normalizer:
+                lowercase:
+                  type: custom
+                  filter:
+                    - lowercase
+            index:
+              mapping.source.mode: synthetic
+
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                normalizer: lowercase
+                synthetic_source_keep: all
+              keyword_with_ignore_above:
+                type: keyword
+                normalizer: lowercase
+                ignore_above: 10
+              keyword_without_doc_values:
+                type: keyword
+                normalizer: lowercase
+                doc_values: false
+
+  - do:
+      index:
+        index: test-keyword-with-normalizer
+        id: 1
+        body:
+          keyword: "the Quick Brown Fox jumps over the lazy Dog"
+          keyword_with_ignore_above: "the Quick Brown Fox jumps over the lazy Dog"
+          keyword_without_doc_values: "the Quick Brown Fox jumps over the lazy Dog"
+
+  - do:
+      index:
+        index: test-keyword-with-normalizer
+        id: 2
+        body:
+          keyword: "The five BOXING wizards jump Quickly"
+          keyword_with_ignore_above: "The five BOXING wizards jump Quickly"
+          keyword_without_doc_values: "The five BOXING wizards jump Quickly"
+
+  - do:
+      index:
+        index: test-keyword-with-normalizer
+        id: 3
+        body:
+          keyword: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ]
+          keyword_with_ignore_above: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ]
+          keyword_without_doc_values: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ]
+
+  - do:
+      mget:
+        index: test-keyword-with-normalizer
+        body:
+          ids: [ 1, 2, 3 ]
   - match: { docs.0._index: "test-keyword-with-normalizer" }
   - match: { docs.0._id: "1" }
   - match:
@@ -141,7 +231,7 @@ keyword with normalizer:
 ---
 stored text:
   - requires:
-      cluster_features: ["gte_v8.5.0"]
+      cluster_features: [ "gte_v8.5.0" ]
       reason: introduced in 8.5.0
 
   - do:
@@ -159,39 +249,39 @@ stored text:
 
   - do:
       index:
-        index:   test
-        id:      1
+        index: test
+        id: 1
         body:
           text: the quick brown fox
 
   - do:
       index:
-        index:   test
-        id:      2
+        index: test
+        id: 2
         body:
           text: jumped over the lazy dog
 
   - do:
       mget:
         index: test
         body:
-          ids:    [1, 2]
-  - match: {docs.0._index: "test"}
-  - match: {docs.0._id: "1"}
+          ids: [ 1, 2 ]
+  - match: { docs.0._index: "test" }
+  - match: { docs.0._id: "1" }
   - match:
       docs.0._source:
         text: the quick brown fox
 
-  - match: {docs.1._index: "test"}
-  - match: {docs.1._id: "2"}
+  - match: { docs.1._index: "test" }
+  - match: { docs.1._id: "2" }
   - match:
       docs.1._source:
         text: jumped over the lazy dog
 
 ---
 force_synthetic_source_ok:
   - requires:
-      cluster_features: ["gte_v8.4.0"]
+      cluster_features: [ "gte_v8.4.0" ]
       reason: introduced in 8.4.0
 
   - do:
@@ -210,15 +300,15 @@ force_synthetic_source_ok:
 
   - do:
       index:
-        index:   test
-        id:      1
+        index: test
+        id: 1
         body:
           obj.kwd: foo
 
   - do:
       index:
-        index:   test
-        id:      2
+        index: test
+        id: 2
         body:
           obj:
             kwd: bar
@@ -228,7 +318,7 @@ force_synthetic_source_ok:
       mget:
         index: test
         body:
-          ids: [1, 2]
+          ids: [ 1, 2 ]
   - match:
       docs.0._source:
         obj.kwd: foo

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -39,6 +39,7 @@
 import org.elasticsearch.common.lucene.search.AutomatonQueries;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSortConfig;
 import org.elasticsearch.index.IndexVersion;
@@ -94,6 +95,10 @@
  */
 public final class KeywordFieldMapper extends FieldMapper {
 
+    public static final NodeFeature KEYWORD_NORMALIZER_SYNTHETIC_SOURCE = new NodeFeature(
+        "mapper.keyword.keyword_normalizer_synthetic_source"
+    );
+
     private static final Logger logger = LogManager.getLogger(KeywordFieldMapper.class);
 
     public static final String CONTENT_TYPE = "keyword";
@@ -1276,11 +1281,10 @@ private String originalName() {
 
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
-        if (hasNormalizer()) {
-            // NOTE: no matter if we have doc values or not we use fallback synthetic source
-            // to store the original value whose doc values would be altered by the normalizer
-            return SyntheticSourceSupport.FALLBACK;
-        }
+        /* NOTE: we allow enabling synthetic source on Keyword fields with a Normalizer, even though the returned synthetic value
+        may not perfectly match the original, pre-normalization, value. This reduces the storage space required for normalized keyword
+        fields, which otherwise perform quite poorly in terms of storage.
+         */
 
         if (fieldType.stored() || hasDocValues) {
             return new SyntheticSourceSupport.Native(() -> syntheticFieldLoader(fullPath(), leafName()));

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -70,7 +70,8 @@ public Set<NodeFeature> getTestFeatures() {
             NPE_ON_DIMS_UPDATE_FIX,
             RESCORE_ZERO_VECTOR_QUANTIZED_VECTOR_MAPPING,
             USE_DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ,
-            IVF_FORMAT_CLUSTER_FEATURE
+            IVF_FORMAT_CLUSTER_FEATURE,
+            KeywordFieldMapper.KEYWORD_NORMALIZER_SYNTHETIC_SOURCE
         );
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -300,6 +300,12 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers ind
             // storing the field without requiring users to explicitly set 'store'.
             //
             // If 'store' parameter was explicitly provided we'll reject the request.
+
+            /* NOTE: I am fairly sure the above is strictly not true.  Testing seems
+            show that we do not reject the mapping, even when there is no compatible keyword
+            field and the mapping sets store to false.  --MT 2025-06-11
+             */
+
             this.store = Parameter.storeParam(
                 m -> ((TextFieldMapper) m).store,
                 () -> isSyntheticSourceEnabled && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false

server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.Function;
@@ -34,19 +35,21 @@ protected Object expected(Map<String, Object> fieldMapping, Object value, TestCo
     public static Object expectedValue(Map<String, Object> fieldMapping, Object value, Params params, TestContext testContext) {
         var nullValue = (String) fieldMapping.get("null_value");
 
-        var ignoreAbove = fieldMapping.get("ignore_above") == null
+        int ignoreAbove = fieldMapping.get("ignore_above") == null
             ? Integer.MAX_VALUE
             : ((Number) fieldMapping.get("ignore_above")).intValue();
 
+        String normalizerName = (String) fieldMapping.get("normalizer");
+
         if (value == null) {
-            return convert(null, nullValue, ignoreAbove);
+            return convert(null, nullValue, ignoreAbove, normalizerName);
         }
 
         if (value instanceof String s) {
-            return convert(s, nullValue, ignoreAbove);
+            return convert(s, nullValue, ignoreAbove, normalizerName);
         }
 
-        Function<Stream<String>, Stream<BytesRef>> convertValues = s -> s.map(v -> convert(v, nullValue, ignoreAbove))
+        Function<Stream<String>, Stream<BytesRef>> convertValues = s -> s.map(v -> convert(v, nullValue, ignoreAbove, normalizerName))
             .filter(Objects::nonNull);
 
         boolean hasDocValues = hasDocValues(fieldMapping, true);
@@ -67,15 +70,21 @@ public static Object expectedValue(Map<String, Object> fieldMapping, Object valu
         return maybeFoldList(resultList);
     }
 
-    private static BytesRef convert(String value, String nullValue, int ignoreAbove) {
+    private static BytesRef convert(String value, String nullValue, int ignoreAbove, String normalizer) {
         if (value == null) {
             if (nullValue != null) {
                 value = nullValue;
             } else {
                 return null;
             }
         }
-
+        if (Objects.equals(normalizer, "lowercase")) {
+            // hopefully not Turkish...
+            value = value.toLowerCase(Locale.ROOT);
+        } else if (normalizer != null) {
+            // we probably can't get here anyway, since MapperServiceTestCase only initializes the lowercase normalizer
+            throw new IllegalArgumentException("normalizer [" + normalizer + "] not supported for block loader tests");
+        }
         return value.length() <= ignoreAbove ? new BytesRef(value) : null;
     }
 }