SONAR-26417 Migrate RuleIndex to the ES8 Java Api Client methods

Co-authored-by: Emma Mansalier <[email protected]>
Co-authored-by: Matteo Mara <[email protected]>

Author: 2 people

server/sonar-server-common/src/it/java/org/sonar/server/rule/index/RuleIndexIT.java

@@ -0,0 +1,197 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2025 SonarSource Sàrl
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import co.elastic.clients.elasticsearch._types.FieldValue;
+import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode;
+import co.elastic.clients.elasticsearch._types.query_dsl.Operator;
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+
+/**
+ * Helper class for building ES 8 Query objects.
+ * Provides methods similar to ES 7's QueryBuilders.
+ */
+public final class ES8QueryHelper {
+
+  private ES8QueryHelper() {
+    // Utility class
+  }
+
+  // ========== MATCH ALL ==========
+
+  public static Query matchAllQuery() {
+    return Query.of(q -> q.matchAll(m -> m));
+  }
+
+  // ========== TERM QUERIES ==========
+
+  public static Query termQuery(String field, String value) {
+    return Query.of(q -> q.term(t -> t.field(field).value(value)));
+  }
+
+  public static Query termQuery(String field, long value) {
+    return Query.of(q -> q.term(t -> t.field(field).value(value)));
+  }
+
+  public static Query termQuery(String field, boolean value) {
+    return Query.of(q -> q.term(t -> t.field(field).value(value)));
+  }
+
+  public static Query termsQuery(String field, String... values) {
+    List<FieldValue> fieldValues = Stream.of(values)
+      .map(FieldValue::of)
+      .toList();
+    return Query.of(q -> q.terms(t -> t.field(field).terms(tv -> tv.value(fieldValues))));
+  }
+
+  public static Query termsQuery(String field, @Nullable Collection<String> values) {
+    if (values == null || values.isEmpty()) {
+      return matchAllQuery();
+    }
+    List<FieldValue> fieldValues = values.stream()
+      .map(FieldValue::of)
+      .toList();
+    return Query.of(q -> q.terms(t -> t.field(field).terms(tv -> tv.value(fieldValues))));
+  }
+
+  // ========== BOOL QUERIES ==========
+
+  public static Query boolQuery() {
+    return Query.of(q -> q.bool(b -> b));
+  }
+
+  public static Query boolQuery(Consumer<BoolQuery.Builder> fn) {
+    return Query.of(q -> q.bool(b -> {
+      fn.accept(b);
+      return b;
+    }));
+  }
+
+  // ========== MATCH QUERIES ==========
+
+  public static Query matchQuery(String field, String text) {
+    return Query.of(q -> q.match(m -> m.field(field).query(text)));
+  }
+
+  public static Query matchQuery(String field, String text, Operator operator) {
+    return Query.of(q -> q.match(m -> m.field(field).query(text).operator(operator)));
+  }
+
+  // ========== RANGE QUERIES ==========
+
+  /**
+   * ES 8: Creates a range query for a specific field.
+   * Note: In ES8, we use the NumberRangeQuery variant for numeric comparisons.
+   */
+  public static Query rangeQuery(String field) {
+    return Query.of(q -> q.range(r -> r.number(n -> n.field(field))));
+  }
+
+  public static Query rangeQueryGte(String field, long value) {
+    return Query.of(q -> q.range(r -> r.number(n -> n.field(field).gte((double) value))));
+  }
+
+  public static Query rangeQueryLte(String field, long value) {
+    return Query.of(q -> q.range(r -> r.number(n -> n.field(field).lte((double) value))));
+  }
+
+  public static Query rangeQueryGt(String field, long value) {
+    return Query.of(q -> q.range(r -> r.number(n -> n.field(field).gt((double) value))));
+  }
+
+  public static Query rangeQueryLt(String field, long value) {
+    return Query.of(q -> q.range(r -> r.number(n -> n.field(field).lt((double) value))));
+  }
+
+  public static Query rangeQueryBetween(String field, long from, long to) {
+    return Query.of(q -> q.range(r -> r.number(n -> n.field(field).gte((double) from).lte((double) to))));
+  }
+
+  public static Query rangeQueryGte(String field, double value) {
+    return Query.of(q -> q.range(r -> r.number(n -> n.field(field).gte(value))));
+  }
+
+  public static Query rangeQueryLte(String field, double value) {
+    return Query.of(q -> q.range(r -> r.number(n -> n.field(field).lte(value))));
+  }
+
+  public static Query rangeQueryGt(String field, double value) {
+    return Query.of(q -> q.range(r -> r.number(n -> n.field(field).gt(value))));
+  }
+
+  public static Query rangeQueryLt(String field, double value) {
+    return Query.of(q -> q.range(r -> r.number(n -> n.field(field).lt(value))));
+  }
+
+  // ========== EXISTS QUERIES ==========
+
+  public static Query existsQuery(String field) {
+    return Query.of(q -> q.exists(e -> e.field(field)));
+  }
+
+  // ========== PREFIX QUERIES ==========
+
+  public static Query prefixQuery(String field, String prefix) {
+    return Query.of(q -> q.prefix(p -> p.field(field).value(prefix)));
+  }
+
+  // ========== WILDCARD QUERIES ==========
+
+  public static Query wildcardQuery(String field, String value) {
+    return Query.of(q -> q.wildcard(w -> w.field(field).value(value)));
+  }
+
+  // ========== NESTED QUERIES ==========
+
+  public static Query nestedQuery(String path, Query query) {
+    return Query.of(q -> q.nested(n -> n.path(path).query(query)));
+  }
+
+  public static Query nestedQuery(String path, Query query, ChildScoreMode scoreMode) {
+    return Query.of(q -> q.nested(n -> n.path(path).query(query).scoreMode(scoreMode)));
+  }
+
+  // ========== HAS CHILD/PARENT QUERIES ==========
+
+  public static Query hasChildQuery(String type, Query query) {
+    return Query.of(q -> q.hasChild(h -> h.type(type).query(query)));
+  }
+
+  public static Query hasChildQuery(String type, Query query, ChildScoreMode scoreMode) {
+    return Query.of(q -> q.hasChild(h -> h.type(type).query(query).scoreMode(scoreMode)));
+  }
+
+  public static Query hasParentQuery(String parentType, Query query) {
+    return Query.of(q -> q.hasParent(h -> h.parentType(parentType).query(query)));
+  }
+
+  /**
+   * Wraps a query with a boost
+   */
+  public static Query withBoost(Query query, float boost) {
+    return Query.of(q -> q.bool(b -> b.must(query).boost(boost)));
+  }
+}

server/sonar-server-common/src/main/java/org/sonar/server/es/ES8QueryHelper.java

@@ -19,6 +19,11 @@
  */
 package org.sonar.server.es;
 
+import co.elastic.clients.elasticsearch._types.FieldValue;
+import co.elastic.clients.elasticsearch._types.SortOptions;
+import co.elastic.clients.elasticsearch._types.query_dsl.Query;
+import co.elastic.clients.elasticsearch.core.SearchRequest;
+import co.elastic.clients.elasticsearch.core.search.Hit;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -46,9 +51,10 @@
 public class EsUtils {
 
   public static final int SCROLL_TIME_IN_MINUTES = 3;
+  public static final int SEARCH_AFTER_PAGE_SIZE = 500;
 
   /**
-   * See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-regexp-query.html
+   * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-regexp-query.html">Elasticsearch Regexp query documentation</a>
    */
   private static final Pattern SPECIAL_REGEX_CHARS = Pattern.compile("[#@&~<>\"{}()\\[\\].+*?^$\\\\|]");
 
@@ -99,12 +105,29 @@ public static String formatDateTime(@Nullable Date date) {
 
   /**
    * Optimize scolling, by specifying document sorting.
-   * See https://www.elastic.co/guide/en/elasticsearch/reference/2.4/search-request-scroll.html#search-request-scroll
+   * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/2.4/search-request-scroll.html#search-request-scroll">Elasticsearch scroll documentation</a>
    */
+  @Deprecated(since = "2025.6", forRemoval = true)
   public static void optimizeScrollRequest(SearchSourceBuilder esSearch) {
     esSearch.sort("_doc", SortOrder.ASC);
   }
 
+  /**
+   * ES 8: Optimize search_after pagination by specifying document sorting.
+   * This provides efficient pagination without the overhead of maintaining scroll contexts.
+   * Note: _doc and _id sorting cannot be used with search_after:
+   * <ul>
+   * <li>_doc is only for scroll API</li>
+   * <li>_id requires fielddata to be enabled (disabled by default in ES 8)</li>
+   * </ul>
+   * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html#search-after">Elasticsearch search_after documentation</a>
+   */
+  public static void optimizeSearchAfterRequest(SearchRequest.Builder esSearch, String sortField) {
+    // Sort by sortField to ensure deterministic ordering for search_after pagination
+    // This sortField has doc_values enabled and provides a unique sort key for reliable pagination
+    esSearch.sort(s -> s.field(f -> f.field(sortField).order(co.elastic.clients.elasticsearch._types.SortOrder.Asc)));
+  }
+
   /**
    * Escapes regexp special characters so that text can be forwarded from end-user input
    * to Elasticsearch regexp query (for instance attributes "include" and "exclude" of
@@ -114,10 +137,21 @@ public static String escapeSpecialRegexChars(String str) {
     return SPECIAL_REGEX_CHARS.matcher(str).replaceAll("\\\\$0");
   }
 
+  @Deprecated(since = "2025.6", forRemoval = true)
   public static <I> Iterator<I> scrollIds(EsClient esClient, SearchResponse scrollResponse, Function<String, I> idConverter) {
     return new IdScrollIterator<>(esClient, scrollResponse, idConverter);
   }
 
+  /**
+   * ES 8: Iterate through search results using search_after API
+   * This is the recommended approach for deep pagination in ES 8.
+   * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html#search-after">Elasticsearch search_after documentation</a>
+   */
+  public static <T, I> Iterator<I> searchAfterIds(EsClient esClient, SearchRequest initialRequest, Class<T> tClass, Function<String, I> idConverter) {
+    return new SearchAfterIterator<>(esClient, initialRequest, tClass, idConverter);
+  }
+
+  @Deprecated(since = "2025.6", forRemoval = true)
   private static class IdScrollIterator<I> implements Iterator<I> {
 
     private final EsClient esClient;
@@ -156,4 +190,100 @@ public void remove() {
       throw new UnsupportedOperationException("Cannot remove item when scrolling");
     }
   }
+
+  private static class SearchAfterIterator<T, I> implements Iterator<I> {
+
+    private final EsClient esClient;
+    private final SearchRequest initialRequest;
+    private final Class<T> tClass;
+    private final Function<String, I> idConverter;
+
+    private final Queue<Hit<T>> hits = new ArrayDeque<>();
+    private List<FieldValue> lastSortValues = null;
+    private boolean hasMore = true;
+    private boolean isFirstRequest = true;
+
+    private SearchAfterIterator(EsClient esClient, SearchRequest initialRequest, Class<T> tClass, Function<String, I> idConverter) {
+      this.esClient = esClient;
+      this.initialRequest = initialRequest;
+      this.tClass = tClass;
+      this.idConverter = idConverter;
+      fetchNextBatch();
+    }
+
+    private void fetchNextBatch() {
+      if (!hasMore) {
+        return;
+      }
+
+      co.elastic.clients.elasticsearch.core.SearchResponse<T> response;
+
+      if (isFirstRequest) {
+        // For the first request, use the initial request but disable source fetching
+        // since we're using Void.class and only need document IDs
+        response = esClient.searchV2(b -> b
+          .index(initialRequest.index())
+          .query(initialRequest.query())
+          .size(initialRequest.size())
+          .sort(initialRequest.sort())
+          .source(s -> s.fetch(false)),
+          tClass);
+        isFirstRequest = false;
+      } else {
+        // For subsequent requests, add search_after with the last sort values
+        final List<String> indexNames = initialRequest.index();
+        final Query query = initialRequest.query();
+        final Integer size = initialRequest.size();
+        final List<SortOptions> sortOptions = initialRequest.sort();
+
+        response = esClient.searchV2(b -> {
+          b.index(indexNames)
+            .query(query)
+            .size(size)
+            .source(s -> s.fetch(false));
+
+          // Apply the same sort configuration as the initial request
+          if (sortOptions != null && !sortOptions.isEmpty()) {
+            b.sort(sortOptions);
+          }
+
+          // Add search_after for pagination
+          b.searchAfter(lastSortValues);
+
+          return b;
+        }, tClass);
+      }
+
+      if (response.hits() != null && response.hits().hits() != null && !response.hits().hits().isEmpty()) {
+        List<Hit<T>> newHits = response.hits().hits();
+        hits.addAll(newHits);
+        // Save the sort values from the last hit for the next request
+        Hit<T> lastHit = newHits.get(newHits.size() - 1);
+        lastSortValues = lastHit.sort();
+      } else {
+        hasMore = false;
+      }
+    }
+
+    @Override
+    public boolean hasNext() {
+      if (hits.isEmpty() && hasMore) {
+        fetchNextBatch();
+      }
+      return !hits.isEmpty();
+    }
+
+    @Override
+    public I next() {
+      if (!hasNext()) {
+        throw new NoSuchElementException();
+      }
+      return idConverter.apply(hits.poll().id());
+    }
+
+    @Override
+    public void remove() {
+      throw new UnsupportedOperationException("Cannot remove item when iterating");
+    }
+  }
 }