Skip to content

Commit d4d960f

Browse files
paulirwinclaude
paulirwin
and
claude
authored
BREAKING: More cancellation support in APIs that call IndexSearcher, #922 (#1261)
* Add cancellation support to Classification * Add cancellation support to SpellChecker * Add cancellation support to GroupingSearch * Add cancellation support to DrillSideways * Add cancellation support to FacetsCollector * Add cancellation support to JoinUtil * Add cancellation support to MemoryIndex * Add cancellation support to Suggest * PR feedback * PR feedback * Add cancellation token tests for higher-level search APIs Verify that CancellationToken is properly plumbed through to IndexSearcher.Search for MemoryIndex, DrillSideways, FacetsCollector, GroupingSearch, JoinUtil, SpellChecker, and AnalyzingInfixSuggester. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Fix DatasetSplitter cancellation exception handling - Add catch for OperationCanceledException before the generic catch so cancellation isn't wrapped in IOException - Replace ThrowIfCancellationRequested in finally with a conditional check to avoid masking exceptions during unwinding - Fix param doc to say "split operation" instead of "search" Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 57e0b8c commit d4d960f

26 files changed

Lines changed: 670 additions & 146 deletions

File tree

‎src/Lucene.Net.Classification/ClassificationResult.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
using System.Threading;
2+
13
namespace Lucene.Net.Classification
24
{
35
/*
@@ -18,7 +20,7 @@ namespace Lucene.Net.Classification
1820
*/
1921

2022
/// <summary>
21-
/// The result of a call to <see cref="IClassifier{T}.AssignClass(string)"/> holding an assigned class of type <typeparam name="T"/> and a score.
23+
/// The result of a call to <see cref="IClassifier{T}.AssignClass(string, CancellationToken)"/> holding an assigned class of type <typeparam name="T"/> and a score.
2224
/// @lucene.experimental
2325
/// </summary>
2426
public class ClassificationResult<T>

‎src/Lucene.Net.Classification/IClassifier.cs‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using Lucene.Net.Analysis;
22
using Lucene.Net.Index;
33
using Lucene.Net.Search;
4+
using System.Threading;
45

56
namespace Lucene.Net.Classification
67
{
@@ -33,8 +34,9 @@ public interface IClassifier<T>
3334
/// Assign a class (with score) to the given text string
3435
/// </summary>
3536
/// <param name="text">a string containing text to be classified</param>
37+
/// <param name="cancellationToken">a cancellation token to cancel the classification operation. LUCENENET specific.</param>
3638
/// <returns>a <see cref="ClassificationResult{T}"/> holding assigned class of type <typeparamref name="T"/> and score</returns>
37-
ClassificationResult<T> AssignClass(string text);
39+
ClassificationResult<T> AssignClass(string text, CancellationToken cancellationToken = default);
3840

3941
/// <summary>
4042
/// Train the classifier using the underlying Lucene index
@@ -43,23 +45,42 @@ public interface IClassifier<T>
4345
/// <param name="atomicReader">the reader to use to access the Lucene index</param>
4446
/// <param name="classFieldName">the name of the field containing the class assigned to documents</param>
4547
/// <param name="textFieldName">the name of the field used to compare documents</param>
46-
void Train(AtomicReader atomicReader, string textFieldName, string classFieldName, Analyzer analyzer);
48+
/// <param name="cancellationToken">a cancellation token to cancel the training operation. LUCENENET specific.
49+
/// Note that not all implementations are actually cancelable.</param>
50+
void Train(AtomicReader atomicReader,
51+
string textFieldName,
52+
string classFieldName,
53+
Analyzer analyzer,
54+
CancellationToken cancellationToken = default);
4755

4856
/// <summary>Train the classifier using the underlying Lucene index</summary>
4957
/// <param name="analyzer">the analyzer used to tokenize / filter the unseen text</param>
5058
/// <param name="atomicReader">the reader to use to access the Lucene index</param>
5159
/// <param name="classFieldName">the name of the field containing the class assigned to documents</param>
5260
/// <param name="query">the query to filter which documents use for training</param>
5361
/// <param name="textFieldName">the name of the field used to compare documents</param>
54-
void Train(AtomicReader atomicReader, string textFieldName, string classFieldName, Analyzer analyzer, Query query);
62+
/// <param name="cancellationToken">a cancellation token to cancel the training operation. LUCENENET specific.
63+
/// Note that not all implementations are actually cancelable.</param>
64+
void Train(AtomicReader atomicReader,
65+
string textFieldName,
66+
string classFieldName,
67+
Analyzer analyzer,
68+
Query query,
69+
CancellationToken cancellationToken = default);
5570

5671
/// <summary>Train the classifier using the underlying Lucene index</summary>
5772
/// <param name="analyzer">the analyzer used to tokenize / filter the unseen text</param>
5873
/// <param name="atomicReader">the reader to use to access the Lucene index</param>
5974
/// <param name="classFieldName">the name of the field containing the class assigned to documents</param>
6075
/// <param name="query">the query to filter which documents use for training</param>
6176
/// <param name="textFieldNames">the names of the fields to be used to compare documents</param>
62-
void Train(AtomicReader atomicReader, string[] textFieldNames, string classFieldName, Analyzer analyzer,
63-
Query query);
77+
/// <param name="cancellationToken">a cancellation token to cancel the training operation. LUCENENET specific.
78+
/// Note that not all implementations are actually cancelable.</param>
79+
void Train(AtomicReader atomicReader,
80+
string[] textFieldNames,
81+
string classFieldName,
82+
Analyzer analyzer,
83+
Query query,
84+
CancellationToken cancellationToken = default);
6485
}
6586
}

‎src/Lucene.Net.Classification/KNearestNeighborClassifier.cs‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using Lucene.Net.Util;
66
using System.Collections.Generic;
77
using System.IO;
8+
using System.Threading;
89

910
namespace Lucene.Net.Classification
1011
{
@@ -34,7 +35,6 @@ namespace Lucene.Net.Classification
3435
/// </summary>
3536
public class KNearestNeighborClassifier : IClassifier<BytesRef>
3637
{
37-
3838
private MoreLikeThis mlt;
3939
private string[] textFieldNames;
4040
private string classFieldName;
@@ -67,8 +67,9 @@ public KNearestNeighborClassifier(int k, int minDocsFreq, int minTermFreq)
6767
/// Assign a class (with score) to the given text string
6868
/// </summary>
6969
/// <param name="text">a string containing text to be classified</param>
70+
/// <param name="cancellationToken">a cancellation token to cancel the search. LUCENENET specific.</param>
7071
/// <returns>a <see cref="ClassificationResult{BytesRef}"/> holding assigned class of type <see cref="BytesRef"/> and score</returns>
71-
public virtual ClassificationResult<BytesRef> AssignClass(string text)
72+
public virtual ClassificationResult<BytesRef> AssignClass(string text, CancellationToken cancellationToken = default)
7273
{
7374
if (mlt is null)
7475
{
@@ -86,7 +87,7 @@ public virtual ClassificationResult<BytesRef> AssignClass(string text)
8687
{
8788
mltQuery.Add(query, Occur.MUST);
8889
}
89-
TopDocs topDocs = indexSearcher.Search(mltQuery, k);
90+
TopDocs topDocs = indexSearcher.Search(mltQuery, k, cancellationToken);
9091
return SelectClassFromNeighbors(topDocs);
9192
}
9293

@@ -129,9 +130,15 @@ private ClassificationResult<BytesRef> SelectClassFromNeighbors(TopDocs topDocs)
129130
/// <param name="atomicReader">the reader to use to access the Lucene index</param>
130131
/// <param name="classFieldName">the name of the field containing the class assigned to documents</param>
131132
/// <param name="textFieldName">the name of the field used to compare documents</param>
132-
public virtual void Train(AtomicReader atomicReader, string textFieldName, string classFieldName, Analyzer analyzer)
133+
/// <param name="cancellationToken">a cancellation token to cancel the training operation. LUCENENET specific.
134+
/// Note that at this time, this implementation is not actually cancelable.</param>
135+
public virtual void Train(AtomicReader atomicReader,
136+
string textFieldName,
137+
string classFieldName,
138+
Analyzer analyzer,
139+
CancellationToken cancellationToken = default)
133140
{
134-
Train(atomicReader, textFieldName, classFieldName, analyzer, null);
141+
Train(atomicReader, textFieldName, classFieldName, analyzer, null, cancellationToken);
135142
}
136143

137144
/// <summary>Train the classifier using the underlying Lucene index</summary>
@@ -140,9 +147,16 @@ public virtual void Train(AtomicReader atomicReader, string textFieldName, strin
140147
/// <param name="classFieldName">the name of the field containing the class assigned to documents</param>
141148
/// <param name="query">the query to filter which documents use for training</param>
142149
/// <param name="textFieldName">the name of the field used to compare documents</param>
143-
public virtual void Train(AtomicReader atomicReader, string textFieldName, string classFieldName, Analyzer analyzer, Query query)
150+
/// <param name="cancellationToken">a cancellation token to cancel the training operation. LUCENENET specific.
151+
/// Note that at this time, this implementation is not actually cancelable.</param>
152+
public virtual void Train(AtomicReader atomicReader,
153+
string textFieldName,
154+
string classFieldName,
155+
Analyzer analyzer,
156+
Query query,
157+
CancellationToken cancellationToken = default)
144158
{
145-
Train(atomicReader, new string[] { textFieldName }, classFieldName, analyzer, query);
159+
Train(atomicReader, new[] { textFieldName }, classFieldName, analyzer, query, cancellationToken);
146160
}
147161

148162
/// <summary>Train the classifier using the underlying Lucene index</summary>
@@ -151,8 +165,17 @@ public virtual void Train(AtomicReader atomicReader, string textFieldName, strin
151165
/// <param name="classFieldName">the name of the field containing the class assigned to documents</param>
152166
/// <param name="query">the query to filter which documents use for training</param>
153167
/// <param name="textFieldNames">the names of the fields to be used to compare documents</param>
154-
public virtual void Train(AtomicReader atomicReader, string[] textFieldNames, string classFieldName, Analyzer analyzer, Query query)
168+
/// <param name="cancellationToken">a cancellation token to cancel the training operation. LUCENENET specific.
169+
/// Note that at this time, this implementation is not actually cancelable.</param>
170+
public virtual void Train(AtomicReader atomicReader,
171+
string[] textFieldNames,
172+
string classFieldName,
173+
Analyzer analyzer,
174+
Query query,
175+
CancellationToken cancellationToken = default)
155176
{
177+
// LUCENENET: cancellationToken is present for IClassifier interface compliance;
178+
// can be utilized here if a suitable cancellation point is added in the future.
156179
this.textFieldNames = textFieldNames;
157180
this.classFieldName = classFieldName;
158181
mlt = new MoreLikeThis(atomicReader);

‎src/Lucene.Net.Classification/SimpleNaiveBayesClassifier.cs‎

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using System;
77
using System.Collections.Generic;
88
using System.IO;
9+
using System.Threading;
910

1011
namespace Lucene.Net.Classification
1112
{
@@ -44,7 +45,7 @@ public class SimpleNaiveBayesClassifier : IClassifier<BytesRef>
4445

4546
/// <summary>
4647
/// Creates a new NaiveBayes classifier.
47-
/// Note that you must call <see cref="Train(AtomicReader, string, string, Analyzer)"/> before you can
48+
/// Note that you must call <see cref="Train(AtomicReader, string, string, Analyzer, CancellationToken)"/> before you can
4849
/// classify any documents.
4950
/// </summary>
5051
public SimpleNaiveBayesClassifier()
@@ -58,9 +59,14 @@ public SimpleNaiveBayesClassifier()
5859
/// <param name="atomicReader">the reader to use to access the Lucene index</param>
5960
/// <param name="classFieldName">the name of the field containing the class assigned to documents</param>
6061
/// <param name="textFieldName">the name of the field used to compare documents</param>
61-
public virtual void Train(AtomicReader atomicReader, string textFieldName, string classFieldName, Analyzer analyzer)
62+
/// <param name="cancellationToken">a cancellation token to cancel the training operation. LUCENENET specific.</param>
63+
public virtual void Train(AtomicReader atomicReader,
64+
string textFieldName,
65+
string classFieldName,
66+
Analyzer analyzer,
67+
CancellationToken cancellationToken = default)
6268
{
63-
Train(atomicReader, textFieldName, classFieldName, analyzer, null);
69+
Train(atomicReader, textFieldName, classFieldName, analyzer, null, cancellationToken);
6470
}
6571

6672
/// <summary>Train the classifier using the underlying Lucene index</summary>
@@ -69,9 +75,15 @@ public virtual void Train(AtomicReader atomicReader, string textFieldName, strin
6975
/// <param name="classFieldName">the name of the field containing the class assigned to documents</param>
7076
/// <param name="query">the query to filter which documents use for training</param>
7177
/// <param name="textFieldName">the name of the field used to compare documents</param>
72-
public virtual void Train(AtomicReader atomicReader, string textFieldName, string classFieldName, Analyzer analyzer, Query query)
78+
/// <param name="cancellationToken">a cancellation token to cancel the training operation. LUCENENET specific.</param>
79+
public virtual void Train(AtomicReader atomicReader,
80+
string textFieldName,
81+
string classFieldName,
82+
Analyzer analyzer,
83+
Query query,
84+
CancellationToken cancellationToken = default)
7385
{
74-
Train(atomicReader, new string[] { textFieldName }, classFieldName, analyzer, query);
86+
Train(atomicReader, new[] { textFieldName }, classFieldName, analyzer, query, cancellationToken);
7587
}
7688

7789
/// <summary>Train the classifier using the underlying Lucene index</summary>
@@ -80,18 +92,24 @@ public virtual void Train(AtomicReader atomicReader, string textFieldName, strin
8092
/// <param name="classFieldName">the name of the field containing the class assigned to documents</param>
8193
/// <param name="query">the query to filter which documents use for training</param>
8294
/// <param name="textFieldNames">the names of the fields to be used to compare documents</param>
83-
public virtual void Train(AtomicReader atomicReader, string[] textFieldNames, string classFieldName, Analyzer analyzer, Query query)
95+
/// <param name="cancellationToken">a cancellation token to cancel the training operation. LUCENENET specific.</param>
96+
public virtual void Train(AtomicReader atomicReader,
97+
string[] textFieldNames,
98+
string classFieldName,
99+
Analyzer analyzer,
100+
Query query,
101+
CancellationToken cancellationToken = default)
84102
{
85103
this.atomicReader = atomicReader;
86104
indexSearcher = new IndexSearcher(this.atomicReader);
87105
this.textFieldNames = textFieldNames;
88106
this.classFieldName = classFieldName;
89107
this.analyzer = analyzer;
90108
this.query = query;
91-
docsWithClassSize = CountDocsWithClass();
109+
docsWithClassSize = CountDocsWithClass(cancellationToken);
92110
}
93111

94-
private int CountDocsWithClass()
112+
private int CountDocsWithClass(CancellationToken cancellationToken)
95113
{
96114
int docCount = MultiFields.GetTerms(atomicReader, classFieldName).DocCount;
97115
if (docCount == -1)
@@ -105,7 +123,7 @@ private int CountDocsWithClass()
105123
{
106124
q.Add(query, Occur.MUST);
107125
}
108-
indexSearcher.Search(q, totalHitCountCollector);
126+
indexSearcher.Search(q, totalHitCountCollector, cancellationToken);
109127
docCount = totalHitCountCollector.TotalHits;
110128
}
111129
return docCount;
@@ -141,8 +159,9 @@ private string[] TokenizeDoc(string doc)
141159
/// Assign a class (with score) to the given text string
142160
/// </summary>
143161
/// <param name="inputDocument">a string containing text to be classified</param>
162+
/// <param name="cancellationToken">a cancellation token to cancel the search. LUCENENET specific.</param>
144163
/// <returns>a <see cref="ClassificationResult{BytesRef}"/> holding assigned class of type <see cref="BytesRef"/> and score</returns>
145-
public virtual ClassificationResult<BytesRef> AssignClass(string inputDocument)
164+
public virtual ClassificationResult<BytesRef> AssignClass(string inputDocument, CancellationToken cancellationToken = default)
146165
{
147166
if (atomicReader is null)
148167
{
@@ -158,7 +177,7 @@ public virtual ClassificationResult<BytesRef> AssignClass(string inputDocument)
158177
while (termsEnum.MoveNext())
159178
{
160179
next = termsEnum.Term;
161-
double clVal = CalculateLogPrior(next) + CalculateLogLikelihood(tokenizedDoc, next);
180+
double clVal = CalculateLogPrior(next) + CalculateLogLikelihood(tokenizedDoc, next, cancellationToken);
162181
if (clVal > max)
163182
{
164183
max = clVal;
@@ -170,14 +189,14 @@ public virtual ClassificationResult<BytesRef> AssignClass(string inputDocument)
170189
}
171190

172191

173-
private double CalculateLogLikelihood(string[] tokenizedDoc, BytesRef c)
192+
private double CalculateLogLikelihood(string[] tokenizedDoc, BytesRef c, CancellationToken cancellationToken)
174193
{
175194
// for each word
176195
double result = 0d;
177196
foreach (string word in tokenizedDoc)
178197
{
179198
// search with text:word AND class:c
180-
int hits = GetWordFreqForClass(word, c);
199+
int hits = GetWordFreqForClass(word, c, cancellationToken);
181200

182201
// num : count the no of times the word appears in documents of class c (+1)
183202
double num = hits + 1; // +1 is added because of add 1 smoothing
@@ -207,7 +226,7 @@ private double GetTextTermFreqForClass(BytesRef c)
207226
return avgNumberOfUniqueTerms * docsWithC; // avg # of unique terms in text fields per doc * # docs with c
208227
}
209228

210-
private int GetWordFreqForClass(string word, BytesRef c)
229+
private int GetWordFreqForClass(string word, BytesRef c, CancellationToken cancellationToken)
211230
{
212231
BooleanQuery booleanQuery = new BooleanQuery();
213232
BooleanQuery subQuery = new BooleanQuery();
@@ -222,7 +241,7 @@ private int GetWordFreqForClass(string word, BytesRef c)
222241
booleanQuery.Add(query, Occur.MUST);
223242
}
224243
TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector();
225-
indexSearcher.Search(booleanQuery, totalHitCountCollector);
244+
indexSearcher.Search(booleanQuery, totalHitCountCollector, cancellationToken);
226245
return totalHitCountCollector.TotalHits;
227246
}
228247

0 commit comments

Comments
 (0)