Merge pull request #6454 from joewiz/bugfix/lucene-facet-drilldown-highlight

[bugfix] Preserve ft:highlight-field-matches under facet drill-down

Author: duncdrum

extensions/indexes/lucene/src/main/java/org/exist/indexing/lucene/LuceneUtil.java

@@ -36,6 +36,7 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.queries.function.FunctionScoreQuery;
 import org.apache.lucene.search.*;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
@@ -163,6 +164,8 @@ public static void extractTerms(final Query query, final Map<Object, Query> term
                     extractTermsFromTermRange(termRangeQuery, terms, reader, includeFields);
             case DrillDownQuery drillDownQuery ->
                     extractTermsFromDrillDown(drillDownQuery, terms, reader, includeFields);
+            case FunctionScoreQuery functionScoreQuery ->
+                    extractTerms(functionScoreQuery.getWrappedQuery(), terms, reader, includeFields);
             case null, default -> {
                 query.visit(new QueryVisitor() {
                     @Override
@@ -181,8 +184,11 @@ public void consumeTerms(Query query, Term... termsArray) {
     }
 
     private static void extractTermsFromDrillDown(DrillDownQuery query, Map<Object, Query> terms, IndexReader reader, boolean includeFields) throws IOException {
-        final Query rewritten = query.rewrite(new IndexSearcher(reader));
-        extractTerms(rewritten, terms, reader, includeFields);
+        // Extract terms from the base (content) query only. Rewriting a DrillDownQuery expands it
+        // into a BooleanQuery that also carries the internal dimension-filter clauses (e.g.
+        // $facets:kind$para), whose terms don't appear in document text and prevent correct
+        // highlight extraction. getBaseQuery() returns the content query directly.
+        extractTerms(query.getBaseQuery(), terms, reader, includeFields);
     }
 
     private static void extractTermsFromBoolean(final BooleanQuery query, final Map<Object, Query> terms, final IndexReader reader, final boolean includeFields) throws IOException {

extensions/indexes/lucene/src/test/xquery/lucene/facet-drilldown-highlight.xqm

@@ -0,0 +1,131 @@
+(:
+ : eXist-db Open Source Native XML Database
+ : Copyright (C) 2001 The eXist-db Authors
+ :
+ : info@exist-db.org
+ : http://www.exist-db.org
+ :
+ : This library is free software; you can redistribute it and/or
+ : modify it under the terms of the GNU Lesser General Public
+ : License as published by the Free Software Foundation; either
+ : version 2.1 of the License, or (at your option) any later version.
+ :
+ : This library is distributed in the hope that it will be useful,
+ : but WITHOUT ANY WARRANTY; without even the implied warranty of
+ : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ : Lesser General Public License for more details.
+ :
+ : You should have received a copy of the GNU Lesser General Public
+ : License along with this library; if not, write to the Free Software
+ : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ :)
+xquery version "3.1";
+
+(:~
+ : Regression test: facet drill-down must not disable ft:highlight-field-matches.
+ :
+ : A faceted ft:query wraps the content query in a Lucene DrillDownQuery, which is opaque to the
+ : term extraction ft:highlight-field-matches relies on. Storing that wrapped query on the match
+ : silently produced empty highlights for every faceted search. The fix keeps the pre-drill-down
+ : query for match/highlight extraction while searching with the drill-down query.
+ :)
+module namespace fdh = "http://exist-db.org/xquery/lucene/test/facet-drilldown-highlight";
+
+declare namespace test = "http://exist-db.org/xquery/xqsuite";
+declare namespace exist = "http://exist.sourceforge.net/NS/exist";
+
+import module namespace ft = "http://exist-db.org/xquery/lucene";
+
+declare variable $fdh:COLLECTION := "/db/lucene-test-facet-highlight";
+declare variable $fdh:CONFIG := "/db/system/config/db/lucene-test-facet-highlight";
+
+declare variable $fdh:XCONF :=
+    <collection xmlns="http://exist-db.org/collection-config/1.0">
+        <index>
+            <lucene>
+                <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer"/>
+                <text qname="para">
+                    <field name="content" expression="."/>
+                    <facet dimension="kind" expression="'para'"/>
+                </text>
+                <text qname="caption">
+                    <field name="content" expression="."/>
+                    <facet dimension="kind" expression="'caption'"/>
+                </text>
+            </lucene>
+        </index>
+    </collection>;
+
+declare variable $fdh:DOC1 :=
+    <article>
+        <section><para>The eXist-db array functions let you map and filter array members.</para></section>
+        <figure><caption>An array diagram</caption></figure>
+    </article>;
+
+declare variable $fdh:DOC2 :=
+    <article>
+        <section><para>map:merge combines maps; array and map are XDM types.</para></section>
+    </article>;
+
+declare
+    %test:setUp
+function fdh:setup() {
+    let $_ := (xmldb:create-collection("/db/system", "config"), xmldb:create-collection("/db/system/config", "db"))
+    let $conf := xmldb:create-collection("/db/system/config/db", "lucene-test-facet-highlight")
+    let $col := xmldb:create-collection("/db", "lucene-test-facet-highlight")
+    return (
+        xmldb:store($conf, "collection.xconf", $fdh:XCONF),
+        xmldb:store($col, "doc1.xml", $fdh:DOC1),
+        xmldb:store($col, "doc2.xml", $fdh:DOC2),
+        xmldb:reindex($col)
+    )
+};
+
+declare
+    %test:tearDown
+function fdh:tearDown() {
+    if (xmldb:collection-available($fdh:COLLECTION)) then xmldb:remove($fdh:COLLECTION) else (),
+    if (xmldb:collection-available($fdh:CONFIG)) then xmldb:remove($fdh:CONFIG) else ()
+};
+
+(: control: a plain (unfaceted) query highlights fine :)
+declare
+    %test:assertTrue
+function fdh:plain-query-highlights() {
+    let $hit := (collection($fdh:COLLECTION)//para[ft:query(., "content:(array)")])[1]
+    return exists(ft:highlight-field-matches($hit, "content")//exist:match)
+};
+
+(: the regression: a facet drill-down query must STILL highlight (was empty before the fix) :)
+declare
+    %test:assertTrue
+function fdh:faceted-query-highlights() {
+    let $opts := map { "facets": map { "kind": "para" } }
+    let $hit := (collection($fdh:COLLECTION)//para[ft:query(., "content:(array)", $opts)])[1]
+    return exists(ft:highlight-field-matches($hit, "content")//exist:match)
+};
+
+(: the highlighted term is the queried term, under drill-down :)
+declare
+    %test:assertTrue
+function fdh:faceted-highlight-marks-the-term() {
+    let $opts := map { "facets": map { "kind": "para" } }
+    let $hit := (collection($fdh:COLLECTION)//para[ft:query(., "content:(array)", $opts)])[1]
+    return lower-case(string(ft:highlight-field-matches($hit, "content")//exist:match[1])) = "array"
+};
+
+(: sanity: facet drill-down still SELECTS by facet value (kind=caption keeps the caption hit) :)
+declare
+    %test:assertEquals(1)
+function fdh:drilldown-selects-facet() {
+    let $opts := map { "facets": map { "kind": "caption" } }
+    return count(collection($fdh:COLLECTION)//caption[ft:query(., "content:(array)", $opts)])
+};
+
+(: sanity: facet drill-down still EXCLUDES other facet values (kind=para drops the caption) :)
+declare
+    %test:assertEquals(0)
+function fdh:drilldown-excludes-other-facet() {
+    let $opts := map { "facets": map { "kind": "para" } }
+    return count(collection($fdh:COLLECTION)//caption[ft:query(., "content:(array)", $opts)])
+};