Merge branch 'main' into script_sort_support_parallel_collection

Author: KunjueYu

.editorconfig

@@ -226,5 +226,8 @@ indent_size = 2
 [*.{xsd,xml}]
 indent_size = 4
 
+[verification-metadata.xml]
+indent_size = 3
+
 [*.{csv,sql}-spec]
 trim_trailing_whitespace = false

.gitattributes

@@ -13,6 +13,9 @@ x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBasePar
 x-pack/plugin/esql/src/main/generated/** linguist-generated=true
 x-pack/plugin/esql/src/main/generated-src/** linguist-generated=true
 
-# ESQL functions docs are autogenerated. More information at `docs/reference/esql/functions/README.md`
-docs/reference/esql/functions/*/** linguist-generated=true
+# ESQL functions docs are autogenerated. More information at `docs/reference/query-languages/esql/README.md`
+docs/reference/query-languages/esql/_snippets/functions/*/** linguist-generated=true
+#docs/reference/query-languages/esql/_snippets/operators/*/** linguist-generated=true
+docs/reference/query-languages/esql/images/** linguist-generated=true
+docs/reference/query-languages/esql/kibana/** linguist-generated=true
 

Could

@@ -25,7 +25,6 @@ base {
   archivesName = 'elasticsearch-benchmarks'
 }
 
-tasks.named("test").configure { enabled = false }
 tasks.named("javadoc").configure { enabled = false }
 
 configurations {
@@ -52,8 +51,10 @@ dependencies {
   api "org.openjdk.jmh:jmh-core:$versions.jmh"
   annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
   // Dependencies of JMH
-  runtimeOnly 'net.sf.jopt-simple:jopt-simple:5.0.4'
+  runtimeOnly 'net.sf.jopt-simple:jopt-simple:5.0.2'
   runtimeOnly 'org.apache.commons:commons-math3:3.6.1'
+
+  testImplementation(project(':test:framework'))
 }
 
 // enable the JMH's BenchmarkProcessor to generate the final benchmark classes

benchmarks/build.gradle

@@ -11,6 +11,7 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
+import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.data.Block;
@@ -28,6 +29,8 @@
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.FoldContext;
@@ -89,9 +92,16 @@ public class EvalBenchmark {
     static final DriverContext driverContext = new DriverContext(BigArrays.NON_RECYCLING_INSTANCE, blockFactory);
 
     static {
+        LogConfigurator.configureESLogging();
         // Smoke test all the expected values and force loading subclasses more like prod
+        selfTest();
+    }
+
+    static void selfTest() {
+        Logger log = LogManager.getLogger(EvalBenchmark.class);
         try {
             for (String operation : EvalBenchmark.class.getField("operation").getAnnotationsByType(Param.class)[0].value()) {
+                log.info("self testing {}", operation);
                 run(operation);
             }
         } catch (NoSuchFieldException e) {

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.compute.operator;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class EvalBenchmarkTests extends ESTestCase {
+    public void testSelfTest() {
+        EvalBenchmark.selfTest();
+    }
+}

benchmarks/src/test/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmarkTests.java

@@ -47,7 +47,7 @@ plugins.withType(InternalJavaRestTestPlugin) {
   tasks.withType(StandaloneRestIntegTestTask).configureEach {
     testClassesDirs = sourceSets.javaRestTest.output.classesDirs
     classpath = sourceSets.javaRestTest.runtimeClasspath
-    usesDefaultDistribution()
+    usesDefaultDistribution("BWC tests require full distribution for now")
   }
 }
 

build-tools-internal/src/main/groovy/elasticsearch.bwc-test.gradle

@@ -17,10 +17,7 @@
 import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
-import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
-import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
-import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 
@@ -81,29 +78,6 @@ public void doCall() {
         if (isModule == false || isXPackModule) {
             addNoticeGeneration(project, extension);
         }
-        project.afterEvaluate(p -> {
-            @SuppressWarnings("unchecked")
-            NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
-                .getExtensions()
-                .getByName(TestClustersPlugin.EXTENSION_NAME);
-            p.getExtensions().getByType(PluginPropertiesExtension.class).getExtendedPlugins().forEach(pluginName -> {
-                // Auto add any dependent modules
-                findModulePath(project, pluginName).ifPresent(
-                    path -> testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.module(path))
-                );
-            });
-        });
-    }
-
-    Optional<String> findModulePath(Project project, String pluginName) {
-        return project.getRootProject()
-            .getAllprojects()
-            .stream()
-            .filter(p -> GradleUtils.isModuleProject(p.getPath()))
-            .filter(p -> p.getPlugins().hasPlugin(PluginBuildPlugin.class))
-            .filter(p -> p.getExtensions().getByType(PluginPropertiesExtension.class).getName().equals(pluginName))
-            .findFirst()
-            .map(Project::getPath);
     }
 
     /**

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -49,8 +49,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-ec2");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-gce");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:mapper-annotated-text");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:mapper-murmur3");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:repository-hdfs");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:store-smb");
         map.put(LegacyRestTestBasePlugin.class, ":qa:ccs-rolling-upgrade-remote-cluster");
         map.put(LegacyRestTestBasePlugin.class, ":qa:mixed-cluster");
@@ -74,11 +72,7 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:fleet");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:logstash");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-constant-keyword");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-unsigned-long");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-version");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:vector-tile");
-        map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:wildcard");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:mixed-tier-cluster");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:repository-old-versions");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:rolling-upgrade");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -198,6 +198,11 @@ public void apply(Project project) {
             task.getExtensions().getExtraProperties().set("usesDefaultDistribution", new Closure<Void>(task) {
                 @Override
                 public Void call(Object... args) {
+                    if (reasonForUsageProvided(args) == false) {
+                        throw new IllegalArgumentException(
+                            "Reason for using `usesDefaultDistribution` required.\nUse usesDefaultDistribution(\"reason why default distro is required here\")."
+                        );
+                    }
                     task.dependsOn(defaultDistro);
                     registerDistributionInputs(task, defaultDistro);
 
@@ -212,6 +217,10 @@ public Void call(Object... args) {
 
                     return null;
                 }
+
+                private static boolean reasonForUsageProvided(Object[] args) {
+                    return args.length == 1 && args[0] instanceof String && ((String) args[0]).isBlank() == false;
+                }
             });
 
             // Add `usesBwcDistribution(version)` extension method to test tasks to indicate they require a BWC distribution

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -18,7 +18,7 @@ netty             = 4.1.118.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
 awsv1sdk            = 1.12.746
-awsv2sdk            = 2.28.13
+awsv2sdk            = 2.30.38
 reactive_streams    = 1.0.4
 
 antlr4            = 4.13.1

build-tools-internal/version.properties

@@ -19,6 +19,7 @@ exec \
   -Des.path.home="$ES_HOME" \
   -Des.path.conf="$ES_PATH_CONF" \
   -Des.distribution.type="$ES_DISTRIBUTION_TYPE" \
+  -Des.java.type="$JAVA_TYPE" \
   -cp "$LAUNCHER_CLASSPATH" \
   org.elasticsearch.launcher.CliToolLauncher \
   "$@"

distribution/src/bin/elasticsearch-cli

@@ -18,6 +18,7 @@ set LAUNCHER_CLASSPATH=%ES_HOME%/lib/*;%ES_HOME%/lib/cli-launcher/*
   -Des.path.home="%ES_HOME%" ^
   -Des.path.conf="%ES_PATH_CONF%" ^
   -Des.distribution.type="%ES_DISTRIBUTION_TYPE%" ^
+  -Des.java.type="%JAVA_TYPE%" ^
   -cp "%LAUNCHER_CLASSPATH%" ^
   org.elasticsearch.launcher.CliToolLauncher ^
   %*

distribution/src/bin/elasticsearch-cli.bat

@@ -55,11 +55,17 @@ else
   JAVA_TYPE="bundled JDK"
 fi
 
-# do not let JAVA_TOOL_OPTIONS slip in (as the JVM does by default)
+# do not let JAVA_TOOL_OPTIONS OR _JAVA_OPTIONS slip in (as the JVM does by default)
 if [ ! -z "$JAVA_TOOL_OPTIONS" ]; then
-  echo "warning: ignoring JAVA_TOOL_OPTIONS=$JAVA_TOOL_OPTIONS"
+  echo -n "warning: ignoring JAVA_TOOL_OPTIONS=$JAVA_TOOL_OPTIONS; "
+  echo "pass JVM parameters via ES_JAVA_OPTS"
   unset JAVA_TOOL_OPTIONS
 fi
+if [ ! -z "$_JAVA_OPTIONS" ]; then
+  echo -n "warning: ignoring _JAVA_OPTIONS=$_JAVA_OPTIONS; "
+  echo "pass JVM parameters via ES_JAVA_OPTS"
+  unset _JAVA_OPTIONS
+fi
 
 # warn that we are not observing the value of JAVA_HOME
 if [ ! -z "$JAVA_HOME" ]; then

distribution/src/bin/elasticsearch-env

@@ -58,11 +58,17 @@ if defined ES_JAVA_HOME (
   set JAVA_TYPE=bundled JDK
 )
 
-rem do not let JAVA_TOOL_OPTIONS slip in (as the JVM does by default)
+rem do not let JAVA_TOOL_OPTIONS or _JAVA_OPTIONS slip in (as the JVM does by default)
 if defined JAVA_TOOL_OPTIONS (
-  echo warning: ignoring JAVA_TOOL_OPTIONS=%JAVA_TOOL_OPTIONS%
+  (echo|set /p=ignoring JAVA_TOOL_OPTIONS=%JAVA_TOOL_OPTIONS%; )
+  echo pass JVM parameters via ES_JAVA_OPTS
   set JAVA_TOOL_OPTIONS=
 )
+if defined _JAVA_OPTIONS (
+  (echo|set /p=ignoring _JAVA_OPTIONS=%_JAVA_OPTIONS%; )
+  echo pass JVM parameters via ES_JAVA_OPTS
+  set _JAVA_OPTIONS=
+)
 
 rem warn that we are not observing the value of $JAVA_HOME
 if defined JAVA_HOME (

distribution/src/bin/elasticsearch-env.bat

@@ -26,6 +26,7 @@ final class SystemJvmOptions {
 
     static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
         String distroType = sysprops.get("es.distribution.type");
+        String javaType = sysprops.get("es.java.type");
         boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
         boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "true"));
         // java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly
@@ -66,8 +67,11 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dlog4j2.disable.jmx=true",
                 "-Dlog4j2.formatMsgNoLookups=true",
                 "-Djava.locale.providers=CLDR",
-                // Pass through distribution type
-                "-Des.distribution.type=" + distroType
+                // Enable vectorization for whatever version we are running. This ensures we use vectorization even when running EA builds.
+                "-Dorg.apache.lucene.vectorization.upperJavaFeatureVersion=" + Runtime.version().feature(),
+                // Pass through distribution type and java type
+                "-Des.distribution.type=" + distroType,
+                "-Des.java.type=" + javaType
             ),
             maybeEnableNativeAccess(useEntitlements),
             maybeOverrideDockerCgroup(distroType),

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -121,6 +121,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
 
   // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
   systemProperty 'es.transport.cname_in_publish_address', 'true'
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
 
 
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")

docs/build.gradle

@@ -0,0 +1,5 @@
+pr: 122459
+summary: Double parameter markers for identifiers
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/122459.yaml

@@ -0,0 +1,7 @@
+pr: 122638
+summary: Expose `input_type` option at root level for `text_embedding` task type in
+  Perform Inference API
+area: Machine Learning
+type: enhancement
+issues:
+ - 117856

docs/changelog/122638.yaml

@@ -0,0 +1,5 @@
+pr: 122852
+summary: Run `TransportGetDataStreamsAction` on local node
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/122852.yaml

@@ -0,0 +1,5 @@
+pr: 122921
+summary: Run `TransportGetMappingsAction` on local node
+area: Indices APIs
+type: enhancement
+issues: []

docs/changelog/122921.yaml

@@ -0,0 +1,5 @@
+pr: 123150
+summary: Limit the number of chunks for semantic text to prevent high memory usage
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/123150.yaml

@@ -0,0 +1,6 @@
+pr: 124177
+summary: "Improve error message for ( and ["
+area: ES|QL
+type: bug
+issues:
+ - 124145

docs/changelog/124177.yaml

@@ -0,0 +1,5 @@
+pr: 124313
+summary: Optimize memory usage in `ShardBulkInferenceActionFilter`
+area: Search
+type: enhancement
+issues: []

docs/changelog/124313.yaml

@@ -0,0 +1,5 @@
+pr: 124335
+summary: Change the order of the optimization rules
+area: ES|QL
+type: bug
+issues: []

docs/changelog/124335.yaml

@@ -0,0 +1,5 @@
+pr: 124477
+summary: Improve downsample performance by buffering docids and do bulk processing
+area: Downsampling
+type: enhancement
+issues: []

docs/changelog/124477.yaml

@@ -0,0 +1,5 @@
+pr: 124581
+summary: New `vector_rescore` parameter as a quantized index type option
+area: Vector Search
+type: enhancement
+issues: []

docs/changelog/124581.yaml

@@ -0,0 +1,5 @@
+pr: 124610
+summary: Remove page alignment in exchange sink
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/124610.yaml

@@ -0,0 +1,5 @@
+pr: 124651
+summary: "Fix system data streams to be restorable from a snapshot"
+area: Infra/Core
+type: bug
+issues: [89261]

docs/changelog/124651.yaml

@@ -0,0 +1,6 @@
+pr: 124676
+summary: TO_LOWER processes all values
+area: ES|QL
+type: bug
+issues:
+ - 124002

docs/changelog/124676.yaml

@@ -0,0 +1,6 @@
+pr: 124732
+summary: Prevent rare starvation bug when using scaling `EsThreadPoolExecutor` with empty core pool size.
+area: Infra/Core
+type: bug
+issues:
+  - 124667

docs/changelog/124732.yaml

@@ -0,0 +1,5 @@
+pr: 124738
+summary: Upgrade AWS v2 SDK to 2.30.38
+area: Machine Learning
+type: upgrade
+issues: []

docs/changelog/124738.yaml

@@ -0,0 +1,5 @@
+pr: 124739
+summary: Improve rolling up metrics
+area: Downsampling
+type: enhancement
+issues: []

docs/changelog/124739.yaml

@@ -0,0 +1,7 @@
+pr: 124769
+summary: Migrate `model_version` to `model_id` when parsing persistent elser inference
+  endpoints
+area: Machine Learning
+type: bug
+issues:
+ - 124675

docs/changelog/124769.yaml

@@ -0,0 +1,6 @@
+pr: 124784
+summary: Merge template mappings properly during validation
+area: Mapping
+type: bug
+issues:
+ - 123372