Merge branch 'main' into fix_batched_phase_failure

Author: benchaplin

distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile

@@ -25,7 +25,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:7b18967a1329291d113228149b97382406f039171c6537b1afa4067701a6ba9e AS builder
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:dea9b73186c5565a5bfa820a68f63ff497fdc9f06a0afc777159d819c5f01dc4 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -104,7 +104,7 @@ WORKDIR /usr/share/elasticsearch/config
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:7b18967a1329291d113228149b97382406f039171c6537b1afa4067701a6ba9e
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:dea9b73186c5565a5bfa820a68f63ff497fdc9f06a0afc777159d819c5f01dc4
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile

@@ -25,7 +25,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:46fd486e4716d592aa1ad7dad15bb3e6d0aa5b08b2070e9daec9f6b8c8c990a6 AS builder
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:401d8684fd5932c178d7cf524cb05a5088d0e4f74e886e2d0a3b4b5127349a49 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -80,7 +80,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:46fd486e4716d592aa1ad7dad15bb3e6d0aa5b08b2070e9daec9f6b8c8c990a6
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:401d8684fd5932c178d7cf524cb05a5088d0e4f74e886e2d0a3b4b5127349a49
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

docs/changelog/134359.yaml

@@ -0,0 +1,6 @@
+pr: 134359
+summary: Make `MutableSearchResponse` ref-counted to prevent use-after-close in async
+  search
+area: Search
+type: bug
+issues: []

docs/changelog/137072.yaml

@@ -0,0 +1,5 @@
+pr: 137072
+summary: Adding base64 indexing for vector values
+area: Vector Search
+type: enhancement
+issues: []

docs/changelog/137407.yaml

@@ -0,0 +1,7 @@
+pr: 137407
+summary: Taking additional settings providers into account for data stream effective
+  settings
+area: Data streams
+type: bug
+issues:
+ - 137381

docs/changelog/137434.yaml

@@ -0,0 +1,5 @@
+pr: 137434
+summary: Require basic licence for the Elastic Inference Service
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/137536.yaml

@@ -0,0 +1,5 @@
+pr: 137536
+summary: Use DV rewrites where possible in Keyword queries
+area: Search
+type: enhancement
+issues: []

docs/changelog/137560.yaml

@@ -0,0 +1,5 @@
+pr: 137560
+summary: More friendly exceptions for validation errors
+area: SQL
+type: bug
+issues: []

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamSettingsAction.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
 import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
@@ -38,6 +39,7 @@ public class TransportGetDataStreamSettingsAction extends TransportLocalProjectM
     GetDataStreamSettingsAction.Response> {
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final SettingsFilter settingsFilter;
+    private final MetadataDataStreamsService metadataDataStreamsService;
 
     @Inject
     public TransportGetDataStreamSettingsAction(
@@ -47,7 +49,8 @@ public TransportGetDataStreamSettingsAction(
         SettingsFilter settingsFilter,
         ActionFilters actionFilters,
         ProjectResolver projectResolver,
-        IndexNameExpressionResolver indexNameExpressionResolver
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        MetadataDataStreamsService metadataDataStreamsService
     ) {
         super(
             GetSettingsAction.NAME,
@@ -59,6 +62,7 @@ public TransportGetDataStreamSettingsAction(
         );
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.settingsFilter = settingsFilter;
+        this.metadataDataStreamsService = metadataDataStreamsService;
     }
 
     @Override
@@ -83,7 +87,9 @@ protected void localClusterStateOperation(
         for (String dataStreamName : dataStreamNames) {
             DataStream dataStream = dataStreamMap.get(dataStreamName);
             Settings settings = settingsFilter.filter(dataStream.getSettings());
-            Settings effectiveSettings = settingsFilter.filter(dataStream.getEffectiveSettings(project.metadata()));
+            Settings effectiveSettings = settingsFilter.filter(
+                metadataDataStreamsService.getEffectiveSettings(project.metadata(), dataStream)
+            );
             responseList.add(new GetDataStreamSettingsAction.DataStreamSettingsResponse(dataStreamName, settings, effectiveSettings));
         }
         listener.onResponse(new GetDataStreamSettingsAction.Response(responseList));

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetadataDataStreamsService;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.project.ProjectResolver;
@@ -80,6 +81,7 @@ public class TransportGetDataStreamsAction extends TransportLocalProjectMetadata
     private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
     private final IndexSettingProviders indexSettingProviders;
     private final Client client;
+    private final MetadataDataStreamsService metadataDataStreamsService;
 
     /**
      * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until
@@ -99,7 +101,8 @@ public TransportGetDataStreamsAction(
         DataStreamGlobalRetentionSettings globalRetentionSettings,
         DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
         IndexSettingProviders indexSettingProviders,
-        Client client
+        Client client,
+        MetadataDataStreamsService metadataDataStreamsService
     ) {
         super(
             GetDataStreamAction.NAME,
@@ -116,6 +119,7 @@ public TransportGetDataStreamsAction(
         this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
         this.indexSettingProviders = indexSettingProviders;
         this.client = new OriginSettingClient(client, "stack");
+        this.metadataDataStreamsService = metadataDataStreamsService;
 
         transportService.registerRequestHandler(
             actionName,
@@ -159,7 +163,8 @@ public void onResponse(DataStreamsStatsAction.Response response) {
                             globalRetentionSettings,
                             dataStreamFailureStoreSettings,
                             indexSettingProviders,
-                            maxTimestamps
+                            maxTimestamps,
+                            metadataDataStreamsService
                         )
                     );
                 }
@@ -180,7 +185,8 @@ public void onFailure(Exception e) {
                     globalRetentionSettings,
                     dataStreamFailureStoreSettings,
                     indexSettingProviders,
-                    null
+                    null,
+                    metadataDataStreamsService
                 )
             );
         }
@@ -230,7 +236,8 @@ static GetDataStreamAction.Response innerOperation(
         DataStreamGlobalRetentionSettings globalRetentionSettings,
         DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
         IndexSettingProviders indexSettingProviders,
-        @Nullable Map<String, Long> maxTimestamps
+        @Nullable Map<String, Long> maxTimestamps,
+        MetadataDataStreamsService metadataDataStreamsService
     ) {
         List<DataStream> dataStreams = getDataStreams(state.metadata(), indexNameExpressionResolver, request);
         List<GetDataStreamAction.Response.DataStreamInfo> dataStreamInfos = new ArrayList<>(dataStreams.size());
@@ -265,7 +272,12 @@ static GetDataStreamAction.Response innerOperation(
             } else {
                 indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
                 if (indexTemplate != null) {
-                    Settings settings = dataStream.getEffectiveSettings(state.metadata());
+                    final Settings settings;
+                    try {
+                        settings = metadataDataStreamsService.getEffectiveSettings(state.metadata(), dataStream);
+                    } catch (IOException e) {
+                        throw new RuntimeException("Failed to get effective settings for data stream: " + dataStream.getName(), e);
+                    }
                     ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
                     if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
                         try {
@@ -277,7 +289,7 @@ static GetDataStreamAction.Response innerOperation(
                                 dataStream.getEffectiveIndexTemplate(state.metadata())
                             );
                         } catch (IOException e) {
-                            throw new RuntimeException(e);
+                            throw new RuntimeException("Failed to determine indexMode for data stream: " + dataStream.getName(), e);
                         }
                     }
                     indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);