BREAKING: Add cancellation support to IndexSearcher, apache#922 (apache#1080)

* Add CancellationToken parameter to IndexSearcher methods, apache#922

* Pass cancellation token and throw if requested, apache#922

* Pass cancellation token further down in multithreaded code, apache#922

* PR feedback

* Apply suggestions from code review

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Add unit tests for cancellation support

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: paulirwin

src/Lucene.Net.Grouping/package.md

@@ -166,4 +166,4 @@ int maxDoc = s.MaxDoc;
 FixedBitSet groupHeadsBitSet = c.RetrieveGroupHeads(maxDoc);
 ```
 
-For each of the above collector types there is also a variant that works with `ValueSource` instead of of fields. Concretely this means that these variants can work with functions. These variants are slower than there term based counter parts. These implementations are located in the `Lucene.Net.Search.Grouping.Function` package, but can also be used with the `GroupingSearch` convenience utility
+For each of the above collector types there is also a variant that works with `ValueSource` instead of fields. Concretely, this means that these variants can work with functions. These variants are slower than their term-based counterparts. These implementations are located in the `Lucene.Net.Search.Grouping.Function` package, but can also be used with the `GroupingSearch` convenience utility.

src/Lucene.Net.Misc/Index/Sorter/BlockJoinComparatorSource.cs

@@ -2,6 +2,7 @@
 using Lucene.Net.Util;
 using System;
 using System.IO;
+using System.Threading;
 
 namespace Lucene.Net.Index.Sorter
 {
@@ -28,7 +29,7 @@ namespace Lucene.Net.Index.Sorter
     /// Note that this class is intended to used with <see cref="SortingMergePolicy"/>,
     /// and for other purposes has some limitations:
     /// <list type="bullet">
-    ///    <item><description>Cannot yet be used with <see cref="IndexSearcher.SearchAfter(ScoreDoc, Query, Filter, int, Sort)">
+    ///    <item><description>Cannot yet be used with <see cref="IndexSearcher.SearchAfter(ScoreDoc, Query, Filter, int, Sort, CancellationToken)">
     ///    IndexSearcher.SearchAfter</see></description></item>
     ///    <item><description>Filling sort field values is not yet supported.</description></item>
     /// </list>

src/Lucene.Net.TestFramework/Search/AssertingIndexSearcher.cs

@@ -3,6 +3,7 @@
 using RandomizedTesting.Generators;
 using System;
 using System.Collections.Generic;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace Lucene.Net.Search
@@ -100,10 +101,10 @@ protected override Query WrapFilter(Query query, Filter filter)
             return (filter is null) ? query : new FilteredQuery(query, filter, TestUtil.RandomFilterStrategy(random));
         }
 
-        protected override void Search(IList<AtomicReaderContext> leaves, Weight weight, ICollector collector)
+        protected override void Search(IList<AtomicReaderContext> leaves, Weight weight, ICollector collector, CancellationToken cancellationToken = default)
         {
             // TODO: shouldn't we AssertingCollector.wrap(collector) here?
-            base.Search(leaves, AssertingWeight.Wrap(random, weight), collector);
+            base.Search(leaves, AssertingWeight.Wrap(random, weight), collector, cancellationToken);
         }
 
         public override string ToString()

src/Lucene.Net.TestFramework/Search/CheckHits.cs

@@ -5,6 +5,7 @@
 using System.Diagnostics;
 using System.Globalization;
 using System.Text;
+using System.Threading;
 using JCG = J2N.Collections.Generic;
 using Assert = Lucene.Net.TestFramework.Assert;
 
@@ -450,28 +451,28 @@ protected virtual void CheckExplanations(Query q)
             base.Search(q, null, new ExplanationAsserter(q, null, this));
         }
 
-        public override TopFieldDocs Search(Query query, Filter filter, int n, Sort sort)
+        public override TopFieldDocs Search(Query query, Filter filter, int n, Sort sort, CancellationToken cancellationToken = default)
         {
             CheckExplanations(query);
-            return base.Search(query, filter, n, sort);
+            return base.Search(query, filter, n, sort, cancellationToken);
         }
 
-        public override void Search(Query query, ICollector results)
+        public override void Search(Query query, ICollector results, CancellationToken cancellationToken = default)
         {
             CheckExplanations(query);
-            base.Search(query, results);
+            base.Search(query, results, cancellationToken);
         }
 
-        public override void Search(Query query, Filter filter, ICollector results)
+        public override void Search(Query query, Filter filter, ICollector results, CancellationToken cancellationToken = default)
         {
             CheckExplanations(query);
-            base.Search(query, filter, results);
+            base.Search(query, filter, results, cancellationToken);
         }
 
-        public override TopDocs Search(Query query, Filter filter, int n)
+        public override TopDocs Search(Query query, Filter filter, int n, CancellationToken cancellationToken = default)
         {
             CheckExplanations(query);
-            return base.Search(query, filter, n);
+            return base.Search(query, filter, n, cancellationToken);
         }
     }
 

src/Lucene.Net.TestFramework/Search/ShardSearchingTestBase.cs

@@ -11,6 +11,7 @@
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
+using System.Threading;
 using JCG = J2N.Collections.Generic;
 #if FEATURE_SERIALIZABLE_EXCEPTIONS
 using System.ComponentModel;
@@ -192,7 +193,7 @@ internal virtual void BroadcastNodeReopen(int nodeID, long version, IndexSearche
         // MOCK: in a real env you have to hit the wire
         // (send this query to all remote nodes
         // concurrently):
-        internal virtual TopDocs SearchNode(int nodeID, long[] nodeVersions, Query q, Sort sort, int numHits, ScoreDoc searchAfter)
+        internal virtual TopDocs SearchNode(int nodeID, long[] nodeVersions, Query q, Sort sort, int numHits, ScoreDoc searchAfter, CancellationToken cancellationToken = default)
         {
             NodeState.ShardIndexSearcher s = m_nodes[nodeID].Acquire(nodeVersions);
             try
@@ -201,17 +202,17 @@ internal virtual TopDocs SearchNode(int nodeID, long[] nodeVersions, Query q, So
                 {
                     if (searchAfter != null)
                     {
-                        return s.LocalSearchAfter(searchAfter, q, numHits);
+                        return s.LocalSearchAfter(searchAfter, q, numHits, cancellationToken);
                     }
                     else
                     {
-                        return s.LocalSearch(q, numHits);
+                        return s.LocalSearch(q, numHits, cancellationToken);
                     }
                 }
                 else
                 {
                     if (Debugging.AssertsEnabled) Debugging.Assert(searchAfter is null); // not supported yet
-                    return s.LocalSearch(q, numHits, sort);
+                    return s.LocalSearch(q, numHits, sort, cancellationToken);
                 }
             }
             finally
@@ -448,7 +449,7 @@ public override CollectionStatistics CollectionStatistics(string field)
                     return new CollectionStatistics(field, maxDoc, docCount, sumTotalTermFreq, sumDocFreq);
                 }
 
-                public override TopDocs Search(Query query, int numHits)
+                public override TopDocs Search(Query query, int numHits, CancellationToken cancellationToken = default)
                 {
                     TopDocs[] shardHits = new TopDocs[nodeVersions.Length];
                     for (int nodeID = 0; nodeID < nodeVersions.Length; nodeID++)
@@ -457,7 +458,7 @@ public override TopDocs Search(Query query, int numHits)
                         {
                             // My node; run using local shard searcher we
                             // already aquired:
-                            shardHits[nodeID] = LocalSearch(query, numHits);
+                            shardHits[nodeID] = LocalSearch(query, numHits, cancellationToken);
                         }
                         else
                         {
@@ -469,12 +470,12 @@ public override TopDocs Search(Query query, int numHits)
                     return TopDocs.Merge(null, numHits, shardHits);
                 }
 
-                public virtual TopDocs LocalSearch(Query query, int numHits)
+                public virtual TopDocs LocalSearch(Query query, int numHits, CancellationToken cancellationToken = default)
                 {
-                    return base.Search(query, numHits);
+                    return base.Search(query, numHits, cancellationToken);
                 }
 
-                public override TopDocs SearchAfter(ScoreDoc after, Query query, int numHits)
+                public override TopDocs SearchAfter(ScoreDoc after, Query query, int numHits, CancellationToken cancellationToken = default)
                 {
                     TopDocs[] shardHits = new TopDocs[nodeVersions.Length];
                     // results are merged in that order: score, shardIndex, doc. therefore we set
@@ -521,7 +522,7 @@ public override TopDocs SearchAfter(ScoreDoc after, Query query, int numHits)
                         {
                             // My node; run using local shard searcher we
                             // already aquired:
-                            shardHits[nodeID] = LocalSearchAfter(shardAfter, query, numHits);
+                            shardHits[nodeID] = LocalSearchAfter(shardAfter, query, numHits, cancellationToken);
                         }
                         else
                         {
@@ -534,12 +535,12 @@ public override TopDocs SearchAfter(ScoreDoc after, Query query, int numHits)
                     return TopDocs.Merge(null, numHits, shardHits);
                 }
 
-                public virtual TopDocs LocalSearchAfter(ScoreDoc after, Query query, int numHits)
+                public virtual TopDocs LocalSearchAfter(ScoreDoc after, Query query, int numHits, CancellationToken cancellationToken = default)
                 {
-                    return base.SearchAfter(after, query, numHits);
+                    return base.SearchAfter(after, query, numHits, cancellationToken);
                 }
 
-                public override TopFieldDocs Search(Query query, int numHits, Sort sort)
+                public override TopFieldDocs Search(Query query, int numHits, Sort sort, CancellationToken cancellationToken = default)
                 {
                     if (Debugging.AssertsEnabled) Debugging.Assert(sort != null);
                     TopDocs[] shardHits = new TopDocs[nodeVersions.Length];
@@ -549,21 +550,21 @@ public override TopFieldDocs Search(Query query, int numHits, Sort sort)
                         {
                             // My node; run using local shard searcher we
                             // already aquired:
-                            shardHits[nodeID] = LocalSearch(query, numHits, sort);
+                            shardHits[nodeID] = LocalSearch(query, numHits, sort, cancellationToken);
                         }
                         else
                         {
-                            shardHits[nodeID] = outerInstance.outerInstance.SearchNode(nodeID, nodeVersions, query, sort, numHits, null);
+                            shardHits[nodeID] = outerInstance.outerInstance.SearchNode(nodeID, nodeVersions, query, sort, numHits, null, cancellationToken);
                         }
                     }
 
                     // Merge:
                     return (TopFieldDocs)TopDocs.Merge(sort, numHits, shardHits);
                 }
 
-                public virtual TopFieldDocs LocalSearch(Query query, int numHits, Sort sort)
+                public virtual TopFieldDocs LocalSearch(Query query, int numHits, Sort sort, CancellationToken cancellationToken = default)
                 {
-                    return base.Search(query, numHits, sort);
+                    return base.Search(query, numHits, sort, cancellationToken);
                 }
             }
 

src/Lucene.Net.Tests/Search/TestBooleanQuery.cs

@@ -3,6 +3,7 @@
 using NUnit.Framework;
 using System;
 using System.Collections.Generic;
+using System.Threading;
 using System.Threading.Tasks;
 using JCG = J2N.Collections.Generic;
 using Assert = Lucene.Net.TestFramework.Assert;
@@ -395,10 +396,10 @@ public IndexSearcherAnonymousClass(IndexReader r)
             {
             }
 
-            protected override void Search(IList<AtomicReaderContext> leaves, Weight weight, ICollector collector)
+            protected override void Search(IList<AtomicReaderContext> leaves, Weight weight, ICollector collector, CancellationToken cancellationToken = default)
             {
                 Assert.AreEqual(-1, collector.GetType().Name.IndexOf("OutOfOrder", StringComparison.Ordinal));
-                base.Search(leaves, weight, collector);
+                base.Search(leaves, weight, collector, cancellationToken);
             }
         }
     }

src/Lucene.Net.Tests/Search/TestCustomSearcherSort.cs

@@ -3,6 +3,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using System.Threading;
 using Assert = Lucene.Net.TestFramework.Assert;
 using JCG = J2N.Collections.Generic;
 
@@ -212,20 +213,20 @@ public CustomSearcher(IndexReader r, int switcher)
                 this.switcher = switcher;
             }
 
-            public override TopFieldDocs Search(Query query, Filter filter, int nDocs, Sort sort)
+            public override TopFieldDocs Search(Query query, Filter filter, int nDocs, Sort sort, CancellationToken cancellationToken = default)
             {
                 BooleanQuery bq = new BooleanQuery();
                 bq.Add(query, Occur.MUST);
                 bq.Add(new TermQuery(new Term("mandant", Convert.ToString(switcher))), Occur.MUST);
-                return base.Search(bq, filter, nDocs, sort);
+                return base.Search(bq, filter, nDocs, sort, cancellationToken);
             }
 
-            public override TopDocs Search(Query query, Filter filter, int nDocs)
+            public override TopDocs Search(Query query, Filter filter, int nDocs, CancellationToken cancellationToken = default)
             {
                 BooleanQuery bq = new BooleanQuery();
                 bq.Add(query, Occur.MUST);
                 bq.Add(new TermQuery(new Term("mandant", Convert.ToString(switcher))), Occur.MUST);
-                return base.Search(bq, filter, nDocs);
+                return base.Search(bq, filter, nDocs, cancellationToken);
             }
         }