Merge branch 'main' into field_alias

Author: bowenlan-amzn

CHANGELOG.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Security Manager Replacement] Enhance Java Agent to intercept newByteChannel ([#17989](https://github.com/opensearch-project/OpenSearch/pull/17989))
 - Enabled Async Shard Batch Fetch by default ([#18139](https://github.com/opensearch-project/OpenSearch/pull/18139))
 - Allow to get the search request from the QueryCoordinatorContext ([#17818](https://github.com/opensearch-project/OpenSearch/pull/17818))
+- Improve sort-query performance by retaining the default `totalHitsThreshold` for approximated `match_all` queries ([#18189](https://github.com/opensearch-project/OpenSearch/pull/18189))
 
 ### Changed
 
@@ -25,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update Apache HttpClient5 and HttpCore5 (CVE-2025-27820) ([#18152](https://github.com/opensearch-project/OpenSearch/pull/18152))
 - Bump `org.apache.commons:commons-collections4` from 4.4 to 4.5.0 ([#18101](https://github.com/opensearch-project/OpenSearch/pull/18101))
 - Bump `netty` from 4.1.118.Final to 4.1.121.Final ([#18192](https://github.com/opensearch-project/OpenSearch/pull/18192))
+- Bump `org.apache.commons:commons-configuration2` from 2.11.0 to 2.12.0 ([#18103](https://github.com/opensearch-project/OpenSearch/pull/18103))
 
 ### Deprecated
 
@@ -35,8 +37,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remove package org.opensearch.transport.grpc and replace with org.opensearch.plugin.transport.grpc ([#18031](https://github.com/opensearch-project/OpenSearch/pull/18031))
 - Fix the native plugin installation error cause by the pgp public key change ([#18147](https://github.com/opensearch-project/OpenSearch/pull/18147))
 - Fix object field exists query ([#17843](https://github.com/opensearch-project/OpenSearch/pull/17843))
-- Use Bad Request status for InputCoercionEcception ([#18161](https://github.com/opensearch-project/OpenSearch/pull/18161))
+- Use Bad Request status for InputCoercionException ([#18161](https://github.com/opensearch-project/OpenSearch/pull/18161))
 - Null check field names in QueryStringQueryBuilder ([#18194](https://github.com/opensearch-project/OpenSearch/pull/18194))
+- Avoid NPE if on SnapshotInfo if 'shallow' boolean not present ([#18187](https://github.com/opensearch-project/OpenSearch/issues/18187))
 
 ### Security
 

plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/RemoteStoreKafkaIT.java

@@ -198,8 +198,9 @@ public void testPauseAndResumeIngestion() throws Exception {
         assertTrue(pauseResponse.isShardsAcknowledged());
         waitForState(() -> {
             GetIngestionStateResponse ingestionState = getIngestionState(indexName);
-            return Arrays.stream(ingestionState.getShardStates())
-                .allMatch(state -> state.isPollerPaused() && state.pollerState().equalsIgnoreCase("paused"));
+            return ingestionState.getFailedShards() == 0
+                && Arrays.stream(ingestionState.getShardStates())
+                    .allMatch(state -> state.isPollerPaused() && state.pollerState().equalsIgnoreCase("paused"));
         });
 
         // verify ingestion state is persisted

plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaUtils.java

@@ -53,7 +53,7 @@ public static void createTopic(String topicName, int numOfPartitions, String boo
         }
 
         // validates topic is created
-        await().atMost(10, TimeUnit.SECONDS).until(() -> checkTopicExistence(topicName, bootstrapServers));
+        await().atMost(60, TimeUnit.SECONDS).until(() -> checkTopicExistence(topicName, bootstrapServers));
     }
 
     public static boolean checkTopicExistence(String topicName, String bootstrapServers) {

plugins/repository-hdfs/build.gradle

@@ -74,7 +74,7 @@ dependencies {
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api 'commons-collections:commons-collections:3.2.2'
   api "org.apache.commons:commons-compress:${versions.commonscompress}"
-  api 'org.apache.commons:commons-configuration2:2.11.0'
+  api 'org.apache.commons:commons-configuration2:2.12.0'
   api "commons-io:commons-io:${versions.commonsio}"
   api 'org.apache.commons:commons-lang3:3.17.0'
   implementation 'com.google.re2j:re2j:1.8'

plugins/repository-hdfs/licenses/commons-configuration2-2.11.0.jar.sha1

@@ -0,0 +1 @@
+0cab81350c85ca35db8c47621a19dcddf128883f

plugins/repository-hdfs/licenses/commons-configuration2-2.12.0.jar.sha1

@@ -13,6 +13,7 @@
 import org.opensearch.client.Response;
 import org.opensearch.client.StreamingRequest;
 import org.opensearch.client.StreamingResponse;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.test.rest.OpenSearchRestTestCase;
 import org.junit.After;
 
@@ -22,6 +23,7 @@
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Stream;
 
@@ -44,7 +46,11 @@ public void tearDown() throws Exception {
         super.tearDown();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/15840")
+    @Override
+    protected Settings restClientSettings() {
+        return Settings.builder().put(super.restClientSettings()).put(CLIENT_SOCKET_TIMEOUT, "5s").build();
+    }
+
     public void testCloseClientStreamingRequest() throws Exception {
         final VirtualTimeScheduler scheduler = VirtualTimeScheduler.create(true);
 
@@ -67,17 +73,19 @@ public void testCloseClientStreamingRequest() throws Exception {
         final StreamingResponse<ByteBuffer> streamingResponse = client().streamRequest(streamingRequest);
         scheduler.advanceTimeBy(delay); /* emit first element */
 
-        StepVerifier.create(Flux.from(streamingResponse.getBody()).map(b -> new String(b.array(), StandardCharsets.UTF_8)))
-            .expectNextMatches(s -> s.contains("\"result\":\"created\"") && s.contains("\"_id\":\"1\""))
-            .then(() -> {
-                try {
-                    client().close();
-                } catch (final IOException ex) {
-                    throw new UncheckedIOException(ex);
-                }
-            })
+        StepVerifier.create(
+            Flux.from(streamingResponse.getBody()).timeout(Duration.ofSeconds(5)).map(b -> new String(b.array(), StandardCharsets.UTF_8))
+        ).expectNextMatches(s -> s.contains("\"result\":\"created\"") && s.contains("\"_id\":\"1\"")).then(() -> {
+            try {
+                client().close();
+            } catch (final IOException ex) {
+                throw new UncheckedIOException(ex);
+            }
+        })
             .then(() -> scheduler.advanceTimeBy(delay))
-            .expectErrorMatches(t -> t instanceof InterruptedIOException || t instanceof ConnectionClosedException)
+            .expectErrorMatches(
+                t -> t instanceof InterruptedIOException || t instanceof ConnectionClosedException || t instanceof TimeoutException
+            )
             .verify(Duration.ofSeconds(10));
     }
 }

plugins/transport-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4StreamingStressIT.java

@@ -1145,7 +1145,7 @@ public void testSortMissingNumbersMinMax() throws Exception {
             .get();
         assertNoFailures(searchResponse);
 
-        assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(2L));
+        assertThat(searchResponse.getHits().getTotalHits().value(), equalTo(3L));
         assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("1"));
         // The order here could be unstable (depends on document order) since missing == field value
         assertThat(searchResponse.getHits().getAt(1).getId(), is(oneOf("3", "2")));

server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java

@@ -4568,7 +4568,7 @@ public IndexShardSnapshot loadShardSnapshot(BlobContainer shardContainer, Snapsh
             SnapshotId snapshotId = snapshotInfo.snapshotId();
             if (snapshotInfo.getPinnedTimestamp() != 0) {
                 return () -> IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0, 0, "1");
-            } else if (snapshotInfo.isRemoteStoreIndexShallowCopyEnabled()) {
+            } else if (Boolean.TRUE.equals(snapshotInfo.isRemoteStoreIndexShallowCopyEnabled())) {
                 if (shardContainer.blobExists(REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.blobName(snapshotId.getUUID()))) {
                     return REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.read(
                         shardContainer,

server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java

@@ -449,7 +449,7 @@ public boolean canApproximate(SearchContext context) {
         if (context.from() + context.size() == 0) {
             this.setSize(SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO);
         } else {
-            this.setSize(Math.max(context.from() + context.size(), context.trackTotalHitsUpTo() + 1));
+            this.setSize(Math.max(context.from() + context.size(), context.trackTotalHitsUpTo()));
         }
         if (context.request() != null && context.request().source() != null) {
             FieldSortBuilder primarySortField = FieldSortBuilder.getPrimaryFieldSortOrNull(context.request().source());

server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java

@@ -12,6 +12,7 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryVisitor;
+import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Weight;
 import org.opensearch.search.internal.SearchContext;
 
@@ -47,7 +48,11 @@ public Query rewrite(IndexSearcher indexSearcher) throws IOException {
             // Default to the original query. This suggests that we were not called from ContextIndexSearcher.
             return originalQuery.rewrite(indexSearcher);
         }
-        return resolvedQuery.rewrite(indexSearcher);
+        Query rewritten = resolvedQuery.rewrite(indexSearcher);
+        if (rewritten != resolvedQuery) {
+            resolvedQuery = rewritten;
+        }
+        return this;
     }
 
     public void setContext(SearchContext context) {
@@ -78,6 +83,15 @@ public boolean equals(Object o) {
         return true;
     }
 
+    @Override
+    public Weight createWeight(IndexSearcher indexSearcher, ScoreMode scoreMode, float boost) throws IOException {
+        if (resolvedQuery == null) {
+            // Default to the original query.
+            return originalQuery.createWeight(indexSearcher, scoreMode, boost);
+        }
+        return resolvedQuery.createWeight(indexSearcher, scoreMode, boost);
+    }
+
     @Override
     public int hashCode() {
         int h = classHash();

server/src/main/java/org/opensearch/search/approximate/ApproximateScoreQuery.java

@@ -74,6 +74,7 @@
 import org.opensearch.common.util.CachedSupplier;
 import org.opensearch.index.search.OpenSearchToParentBlockJoinQuery;
 import org.opensearch.search.DocValueFormat;
+import org.opensearch.search.approximate.ApproximateScoreQuery;
 import org.opensearch.search.collapse.CollapseContext;
 import org.opensearch.search.internal.ScrollContext;
 import org.opensearch.search.internal.SearchContext;
@@ -724,6 +725,8 @@ static int shortcutTotalHitCount(IndexReader reader, Query query) throws IOExcep
                 query = ((ConstantScoreQuery) query).getQuery();
             } else if (query instanceof BoostQuery) {
                 query = ((BoostQuery) query).getQuery();
+            } else if (query instanceof ApproximateScoreQuery) {
+                query = ((ApproximateScoreQuery) query).getOriginalQuery();
             } else {
                 break;
             }

server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java

@@ -287,13 +287,20 @@ public void testRangeQuery() throws IOException {
         String date2 = "2016-04-28T11:33:52";
         long instant1 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date1)).toInstant().toEpochMilli();
         long instant2 = DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(date2)).toInstant().toEpochMilli() + 999;
-        Query expected = new ApproximatePointRangeQuery(
+        ApproximatePointRangeQuery approximatePointRangeQuery = new ApproximatePointRangeQuery(
             "field",
             pack(new long[] { instant1 }).bytes,
             pack(new long[] { instant2 }).bytes,
             new long[] { instant1 }.length,
             ApproximatePointRangeQuery.LONG_FORMAT
         );
+        Query expected = new ApproximateScoreQuery(
+            new IndexOrDocValuesQuery(
+                LongPoint.newRangeQuery("field", instant1, instant2),
+                SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2)
+            ),
+            approximatePointRangeQuery
+        );
         Query rangeQuery = ft.rangeQuery(date1, date2, true, true, null, null, null, context);
         assertTrue(rangeQuery instanceof ApproximateScoreQuery);
         ((ApproximateScoreQuery) rangeQuery).setContext(new TestSearchContext(context));