OAK-11352 (oak-search-elastic) add useInFullTextQuery flag for dynamic boosted values (#1949)

* OAK-11352 (oak-search-elastic) add useInFullTextQuery flag for dynamic boosted values

* OAK-11352 (minor) make useInFullTextQuery final

* OAK-11352 (test) fix dynamicBoostNotIncludedInFullText

Author: fabriziofortino

oak-doc/src/site/markdown/query/elastic.md

@@ -48,9 +48,19 @@ however there are differences:
 * `useInExcerpt` does not support regexp relative properties.
 * For property definitions, `sync` and `unique` are ignored.
   Synchronous indexing, and enforcing uniqueness constraints is not currently supported in elastic indexes.
-* The behavior for `dynamicBoost` is slightly different: 
-  For Lucene indexes, boosting is done in indexing, while for Elastic it is done at query time.
-* The behavior for `suggest` is slightly different:
+* The behavior of `dynamicBoost` differs slightly between Lucene and Elasticsearch:  
+  - **Lucene**: Boosting is applied at indexing time.  
+  - **Elasticsearch**: Boosting is applied at query time.  
+
+Full-text queries automatically use dynamically boosted values to match relevant results, but this behavior may not always be desirable.
+To use these values exclusively for influencing relevance without affecting matching, configure the property definition as follows:
+```json
+{
+  "dynamicBoost": true,
+  "useInFullTextQuery": false
+}
+```
+* The behavior of `suggest` is slightly different:
   For Lucene indexes, the suggestor is updated every 10 minutes by default and the frequency
   can be changed by `suggestUpdateFrequencyMinutes` property in suggestion node under the index definition node.
   In Elastic indexes, there is no such delay and thus no need for the above config property. This is an improvement in ES over lucene.

oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java

@@ -187,7 +187,7 @@ public class ElasticIndexDefinition extends IndexDefinition {
     public final InferenceDefinition inferenceDefinition;
 
     private final Map<String, List<PropertyDefinition>> propertiesByName;
-    private final List<PropertyDefinition> dynamicBoostProperties;
+    private final List<ElasticPropertyDefinition> dynamicBoostProperties;
     private final List<PropertyDefinition> similarityProperties;
     private final List<PropertyDefinition> similarityTagsProperties;
     private final String[] similarityTagsFields;
@@ -238,6 +238,7 @@ public ElasticIndexDefinition(NodeState root, NodeState defn, String indexPath,
                 .stream()
                 .flatMap(IndexingRule::getNamePatternsProperties)
                 .filter(pd -> pd.dynamicBoost)
+                .map(pd -> (ElasticPropertyDefinition) pd)
                 .collect(Collectors.toList());
 
         this.similarityProperties = getDefinedRules()
@@ -278,7 +279,7 @@ public Map<String, List<PropertyDefinition>> getPropertiesByName() {
         return propertiesByName;
     }
 
-    public List<PropertyDefinition> getDynamicBoostProperties() {
+    public List<ElasticPropertyDefinition> getDynamicBoostProperties() {
         return dynamicBoostProperties;
     }
 

oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticPropertyDefinition.java

@@ -34,6 +34,12 @@ public class ElasticPropertyDefinition extends PropertyDefinition {
   private static final int DEFAULT_CANDIDATES = 500;
   private KnnSearchParameters knnSearchParameters;
 
+  /**
+   * Whether to use dynamic boosted values in full text queries, default is true
+   */
+  private static final String PROP_USE_IN_FULL_TEXT_QUERY = "useInFullTextQuery";
+  private final boolean useInFullTextQuery;
+
   public ElasticPropertyDefinition(IndexDefinition.IndexingRule idxDefn, String nodeName, NodeState defn) {
     super(idxDefn, nodeName, defn);
     if (this.useInSimilarity) {
@@ -43,12 +49,17 @@ public ElasticPropertyDefinition(IndexDefinition.IndexingRule idxDefn, String no
           getOptionalValue(defn, PROP_K, DEFAULT_K),
           getOptionalValue(defn, PROP_CANDIDATES, DEFAULT_CANDIDATES));
     }
+    this.useInFullTextQuery = this.dynamicBoost && getOptionalValue(defn, PROP_USE_IN_FULL_TEXT_QUERY, true);
   }
 
   public KnnSearchParameters getKnnSearchParameters() {
     return knnSearchParameters;
   }
 
+    public boolean useInFullTextQuery() {
+        return useInFullTextQuery;
+    }
+
   /**
    * Class for defining parameters of approximate knn search on dense_vector fields
    * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/dense-vector.html">...</a> and

oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/ElasticRequestHandler.java

@@ -556,13 +556,17 @@ private boolean visitTerm(String propertyName, String text, String boost, boolea
                     QueryStringQuery.Builder qsqBuilder = fullTextQuery(text, ElasticIndexDefinition.DYNAMIC_PROPERTIES + ".value", pr, false);
                     bqBuilder.must(m -> m.nested(nf -> nf.path(ElasticIndexDefinition.DYNAMIC_PROPERTIES).query(Query.of(q -> q.queryString(qsqBuilder.build())))));
                 } else {
-                    boolean dbEnabled = !elasticIndexDefinition.getDynamicBoostProperties().isEmpty();
+                    // TODO: we include dynamic boosted values in the full-text query if there is at least one dynamic property with useInFullTextQuery set to true
+                    // This might not be ideal when there are multiple dynamic properties with different useInFullTextQuery settings (very unlikely)
+                    // A better approach would be to include the values at index time (requires a refactoring of the DocumentMaker to access the ElasticIndexDefinition)
+                    boolean includeDynamicBoostedValues = !elasticIndexDefinition.getDynamicBoostProperties().isEmpty() &&
+                            elasticIndexDefinition.getDynamicBoostProperties().stream().anyMatch(ElasticPropertyDefinition::useInFullTextQuery);
 
                     // Experimental support for inference queries
                     if (elasticIndexDefinition.inferenceDefinition != null && elasticIndexDefinition.inferenceDefinition.queries != null) {
-                        bqBuilder.must(m -> m.bool(b -> inference(b, propertyName, text, pr, dbEnabled)));
+                        bqBuilder.must(m -> m.bool(b -> inference(b, propertyName, text, pr, includeDynamicBoostedValues)));
                     } else {
-                        QueryStringQuery.Builder qsqBuilder = fullTextQuery(text, getElasticFieldName(propertyName), pr, dbEnabled);
+                        QueryStringQuery.Builder qsqBuilder = fullTextQuery(text, getElasticFieldName(propertyName), pr, includeDynamicBoostedValues);
                         bqBuilder.must(m -> m.queryString(qsqBuilder.build()));
                     }
                 }
@@ -878,7 +882,7 @@ private static Query referenceConstraint(String uuid) {
         return Query.of(q -> q.multiMatch(m -> m.fields(uuid)));
     }
 
-    private static QueryStringQuery.Builder fullTextQuery(String text, String fieldName, PlanResult pr, boolean dynamicBoostEnabled) {
+    private static QueryStringQuery.Builder fullTextQuery(String text, String fieldName, PlanResult pr, boolean includeDynamicBoostedValues) {
         LOG.debug("fullTextQuery for text: '{}', fieldName: '{}'", text, fieldName);
         QueryStringQuery.Builder qsqBuilder = new QueryStringQuery.Builder()
                 .query(FulltextIndex.rewriteQueryText(text))
@@ -890,7 +894,7 @@ private static QueryStringQuery.Builder fullTextQuery(String text, String fieldN
                 qsqBuilder.fields(pd.name + "^" + pd.boost);
             }
             // dynamic boost is included only for :fulltext field
-            if (dynamicBoostEnabled) {
+            if (includeDynamicBoostedValues) {
                 qsqBuilder.fields(ElasticIndexDefinition.DYNAMIC_BOOST_FULLTEXT + "^" + DYNAMIC_BOOST_WEIGHT);
             }
         }

oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticDynamicBoostTest.java

@@ -84,4 +84,39 @@ public void dynamicBoostAnalyzed() throws Exception {
                     List.of("/test/asset2", "/test/asset1"));
         });
     }
+
+    @Test
+    public void dynamicBoostNotIncludedInFullText() throws Exception {
+        createAssetsIndexAndProperties(false, false, false);
+
+        Tree testParent = createNodeWithType(root.getTree("/"), "test", JcrConstants.NT_UNSTRUCTURED, "");
+
+        Tree predicted1 = createAssetNodeWithPredicted(testParent, "asset1", "flower with a lot of red and a bit of blue");
+        createPredictedTag(predicted1, "fooTag", 100.0);
+        createPredictedTag(predicted1, "barTag", 1.0);
+        createPredictedTag(predicted1, "red", 9.0);
+        createPredictedTag(predicted1, "blue", 1.0);
+
+        Tree predicted2 = createAssetNodeWithPredicted(testParent, "asset2", "flower with a lot of blue and a bit of red");
+        createPredictedTag(predicted2, "fooTag", 1.0);
+        createPredictedTag(predicted2, "barTag", 100.0);
+        createPredictedTag(predicted2, "red", 1.0);
+        createPredictedTag(predicted2, "blue", 9.0);
+
+        Tree predicted3 = createAssetNodeWithPredicted(testParent, "asset3", "this is a not matching asset");
+        createPredictedTag(predicted3, "fooTag", 1.0);
+        createPredictedTag(predicted3, "barTag", 1.0);
+
+        root.commit();
+
+        assertEventually(() -> {
+            // with this test we are checking that the dynamic boost is not included in the fulltext search
+            assertQuery("//element(*, dam:Asset)[jcr:contains(., 'fooTag')]", XPATH, List.of());
+            assertOrderedQuery("select [jcr:path] from [dam:Asset] where contains(*, 'flower OR fooTag')",
+                    List.of("/test/asset1", "/test/asset2"));
+            assertOrderedQuery("select [jcr:path] from [dam:Asset] where contains(*, 'flower OR barTag')",
+                    List.of("/test/asset2", "/test/asset1"));
+        });
+
+    }
 }

oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/DynamicBoostCommonTest.java

@@ -230,13 +230,18 @@ protected void createPredictedTag(Tree parent, String tagName, double confidence
     }
 
     protected void createAssetsIndexAndProperties(boolean lite, boolean similarityTags) throws Exception {
+        createAssetsIndexAndProperties(lite, similarityTags, true);
+    }
+
+    protected void createAssetsIndexAndProperties(boolean lite, boolean similarityTags, boolean useInFullTextQuery) throws Exception {
         NodeTypeRegistry.register(root, new ByteArrayInputStream(ASSET_NODE_TYPE.getBytes()), "test nodeType");
         Tree indexRuleProps = createIndex("dam:Asset", lite);
 
         Tree predictedTagsDynamicBoost = createNodeWithType(indexRuleProps, "predictedTagsDynamicBoost", JcrConstants.NT_UNSTRUCTURED, "");
         predictedTagsDynamicBoost.setProperty("name", "jcr:content/metadata/predictedTags/.*");
         predictedTagsDynamicBoost.setProperty("isRegexp", true);
         predictedTagsDynamicBoost.setProperty("dynamicBoost", true);
+        predictedTagsDynamicBoost.setProperty("useInFullTextQuery", useInFullTextQuery);
 
         if (similarityTags) {
             Tree predictedTags = createNodeWithType(indexRuleProps, "predictedTags", JcrConstants.NT_UNSTRUCTURED, "");