diff --git a/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingInfixSuggester.cs b/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingInfixSuggester.cs index 0e8b4f43d9..7968b0b628 100644 --- a/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingInfixSuggester.cs +++ b/src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingInfixSuggester.cs @@ -65,6 +65,9 @@ namespace Lucene.Net.Search.Suggest.Analyzing public class AnalyzingInfixSuggester : Lookup, IDisposable { private readonly object syncLock = new object(); //uses syncLock as substitute for Java's synchronized (method) keyword + // LUCENENET specific - Support for LUCENE-7564. + // Forces single-threaded access to the SearcherManager when performing an acquire() or reassigning. + private readonly object searcherMgrLock = new object(); /// /// Field name used for the indexed text. @@ -92,12 +95,14 @@ public class AnalyzingInfixSuggester : Lookup, IDisposable private readonly Directory dir; internal readonly int minPrefixChars; private readonly bool commitOnBuild; + private readonly bool closeIndexWriterOnBuild; // LUCENENET specific - Support for LUCENE-7564. // LUCENENET specific - index writer config factory for extending classes private readonly IAnalyzingInfixSuggesterIndexWriterConfigFactory indexWriterConfigFactory; /// /// Used for ongoing NRT additions/updates. - private IndexWriter writer; + // LUCENENET specific - changed from private to protected internal for LUCENE-7564 test support. + protected internal IndexWriter m_writer; /// /// used for lookups. @@ -109,6 +114,12 @@ public class AnalyzingInfixSuggester : Lookup, IDisposable /// public const int DEFAULT_MIN_PREFIX_CHARS = 4; + /// + /// Default option to close the once the index has been built. + /// + // LUCENENET specific - Support for LUCENE-7564. + protected const bool DEFAULT_CLOSE_INDEXWRITER_ON_BUILD = true; + /// /// How we sort the postings and search results. private static readonly Sort SORT = new Sort(new SortField("weight", SortFieldType.INT64, true)); @@ -138,8 +149,9 @@ public AnalyzingInfixSuggester(LuceneVersion matchVersion, Directory dir, Analyz /// Prefixes shorter than this are indexed as character /// ngrams (increasing index size but making lookups /// faster). - // LUCENENET specific - LUCENE-5889, a 4.11.0 feature. calls new constructor with extra param. - // LUCENENET UPGRADE TODO: Remove method at version 4.11.0. Was retained for perfect 4.8 compatibility + // LUCENENET specific - backported from LUCENE-5889 (4.11.0), LUCENE-7564 (6.4.0), LUCENE-7670 (6.4.1). + // Calls new constructor with default values for commitOnBuild and closeIndexWriterOnBuild. + // Retained for backwards compatibility. public AnalyzingInfixSuggester(LuceneVersion matchVersion, Directory dir, Analyzer indexAnalyzer, Analyzer queryAnalyzer, int minPrefixChars) : this(matchVersion, dir, indexAnalyzer, queryAnalyzer, minPrefixChars, commitOnBuild: false) @@ -165,7 +177,33 @@ public AnalyzingInfixSuggester(LuceneVersion matchVersion, Directory dir, Analyz // LUCENENET specific - LUCENE-5889, a 4.11.0 feature. (Code moved from other constructor to here.) public AnalyzingInfixSuggester(LuceneVersion matchVersion, Directory dir, Analyzer indexAnalyzer, Analyzer queryAnalyzer, int minPrefixChars, bool commitOnBuild) - : this(new AnalyzingInfixSuggesterIndexWriterConfigFactory(SORT), matchVersion, dir, indexAnalyzer, queryAnalyzer, minPrefixChars, commitOnBuild) + : this(matchVersion, dir, indexAnalyzer, queryAnalyzer, minPrefixChars, commitOnBuild, DEFAULT_CLOSE_INDEXWRITER_ON_BUILD) + { + } + + /// + /// Create a new instance, loading from a previously built + /// directory, if it exists. This directory must be + /// private to the infix suggester (i.e., not an external + /// Lucene index). Note that + /// will also dispose the provided directory. + /// + /// Minimum number of leading characters + /// before is used (default 4). + /// Prefixes shorter than this are indexed as character + /// ngrams (increasing index size but making lookups + /// faster). + /// Call commit after the index has finished building. This + /// would persist the suggester index to disk and future instances of this suggester can + /// use this pre-built dictionary. + /// If true, the will be closed + /// after the index has finished building. + // LUCENENET specific - closeIndexWriterOnBuild backported from LUCENE-7564. + // Note: Java's equivalent constructor also has allTermsRequired and highlight parameters, which + // are not present here because those are method-level parameters in this version of Lucene.NET. + public AnalyzingInfixSuggester(LuceneVersion matchVersion, Directory dir, Analyzer indexAnalyzer, + Analyzer queryAnalyzer, int minPrefixChars, bool commitOnBuild, bool closeIndexWriterOnBuild) + : this(new AnalyzingInfixSuggesterIndexWriterConfigFactory(SORT), matchVersion, dir, indexAnalyzer, queryAnalyzer, minPrefixChars, commitOnBuild, closeIndexWriterOnBuild) { } @@ -186,14 +224,41 @@ public AnalyzingInfixSuggester(LuceneVersion matchVersion, Directory dir, Analyz /// use this pre-built dictionary. /// Factory for creating the . // LUCENENET specific - added indexWriterConfigFactory parameter to allow for customizing the index writer config. + // Retained for backwards compatibility. public AnalyzingInfixSuggester(IAnalyzingInfixSuggesterIndexWriterConfigFactory indexWriterConfigFactory, LuceneVersion matchVersion, Directory dir, Analyzer indexAnalyzer, Analyzer queryAnalyzer, int minPrefixChars, bool commitOnBuild) + : this(indexWriterConfigFactory, matchVersion, dir, indexAnalyzer, queryAnalyzer, minPrefixChars, commitOnBuild, DEFAULT_CLOSE_INDEXWRITER_ON_BUILD) + { + } + + /// + /// Create a new instance, loading from a previously built + /// directory, if it exists. This directory must be + /// private to the infix suggester (i.e., not an external + /// Lucene index). Note that + /// will also dispose the provided directory. + /// + /// Minimum number of leading characters + /// before is used (default 4). + /// Prefixes shorter than this are indexed as character + /// ngrams (increasing index size but making lookups + /// faster). + /// Call commit after the index has finished building. This + /// would persist the suggester index to disk and future instances of this suggester can + /// use this pre-built dictionary. + /// If true, the will be closed + /// after the index has finished building. + /// Factory for creating the . + // LUCENENET specific - added indexWriterConfigFactory and closeIndexWriterOnBuild parameters. + public AnalyzingInfixSuggester(IAnalyzingInfixSuggesterIndexWriterConfigFactory indexWriterConfigFactory, LuceneVersion matchVersion, + Directory dir, Analyzer indexAnalyzer, Analyzer queryAnalyzer, int minPrefixChars, bool commitOnBuild, bool closeIndexWriterOnBuild) { if (minPrefixChars < 0) { throw new ArgumentOutOfRangeException(nameof(minPrefixChars), "minPrefixChars must be >= 0; got: " + minPrefixChars);// LUCENENET specific - changed from IllegalArgumentException to ArgumentOutOfRangeException (.NET convention) } + // LUCENENET specific - moved IndexWriterConfig to AnalyzingInfixSuggesterIndexWriterConfigFactory if (indexWriterConfigFactory is null) throw new ArgumentNullException(nameof(indexWriterConfigFactory)); this.m_queryAnalyzer = queryAnalyzer; @@ -202,14 +267,19 @@ public AnalyzingInfixSuggester(IAnalyzingInfixSuggesterIndexWriterConfigFactory this.dir = dir; this.minPrefixChars = minPrefixChars; this.commitOnBuild = commitOnBuild; + this.closeIndexWriterOnBuild = closeIndexWriterOnBuild; this.indexWriterConfigFactory = indexWriterConfigFactory; if (DirectoryReader.IndexExists(dir)) { // Already built; open it: - var config = indexWriterConfigFactory.Get(matchVersion, GetGramAnalyzer(), OpenMode.APPEND); - writer = new IndexWriter(dir, config); - m_searcherMgr = new SearcherManager(writer, true, null); + + // LUCENENET specific - backported fix from Lucene 6.4.1 to fix #1242. previously was: + // var config = indexWriterConfigFactory.Get(matchVersion, GetGramAnalyzer(), OpenMode.APPEND); + // writer = new IndexWriter(dir, config); + // m_searcherMgr = new SearcherManager(writer, true, null); + + m_searcherMgr = new SearcherManager(dir, null); } } @@ -228,75 +298,96 @@ protected internal virtual Directory GetDirectory(DirectoryInfo path) public override void Build(IInputEnumerator enumerator) { - if (m_searcherMgr != null) - { - m_searcherMgr.Dispose(); - m_searcherMgr = null; - } - - if (writer != null) - { - writer.Dispose(); - writer = null; - } - - AtomicReader r = null; - bool success = false; + UninterruptableMonitor.Enter(searcherMgrLock); try { - // First pass: build a temporary normal Lucene index, - // just indexing the suggestions as they iterate: - writer = new IndexWriter(dir, indexWriterConfigFactory.Get(matchVersion, GetGramAnalyzer(), OpenMode.CREATE)); - //long t0 = System.nanoTime(); + if (m_searcherMgr != null) + { + m_searcherMgr.Dispose(); + m_searcherMgr = null; + } - // TODO: use threads? - BytesRef text; - while (enumerator.MoveNext()) + if (m_writer != null) { - text = enumerator.Current; - BytesRef payload; - if (enumerator.HasPayloads) + m_writer.Dispose(); + m_writer = null; + } + + bool success = false; + try + { + // First pass: build a temporary normal Lucene index, + // just indexing the suggestions as they iterate: + m_writer = new IndexWriter(dir, indexWriterConfigFactory.Get(matchVersion, GetGramAnalyzer(), OpenMode.CREATE)); + //long t0 = System.nanoTime(); + + // TODO: use threads? + BytesRef text; + while (enumerator.MoveNext()) { - payload = enumerator.Payload; + text = enumerator.Current; + BytesRef payload; + if (enumerator.HasPayloads) + { + payload = enumerator.Payload; + } + else + { + payload = null; + } + + Add(text, enumerator.Contexts, enumerator.Weight, payload); } - else + + //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec"); + if (commitOnBuild || closeIndexWriterOnBuild) // LUCENENET specific - Support for LUCENE-5889, LUCENE-7564. { - payload = null; + Commit(); } - - Add(text, enumerator.Contexts, enumerator.Weight, payload); + m_searcherMgr = new SearcherManager(m_writer, true, null); + success = true; } - - //System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec"); - if (commitOnBuild) //LUCENENET specific -Support for LUCENE - 5889. + finally { - Commit(); + if (success) + { + if (closeIndexWriterOnBuild) // LUCENENET specific - Support for LUCENE-7564. + { + m_writer.Dispose(); + m_writer = null; + } + } + else + { + if (m_writer != null) + { + m_writer.Rollback(); + m_writer = null; + } + } } - m_searcherMgr = new SearcherManager(writer, true, null); - success = true; } finally { - if (success) - { - IOUtils.Dispose(r); - } - else - { - IOUtils.DisposeWhileHandlingException(writer, r); - writer = null; - } + UninterruptableMonitor.Exit(searcherMgrLock); } } - // LUCENENET specific -Support for LUCENE-5889. + // LUCENENET specific - Support for LUCENE-5889, LUCENE-7564. public void Commit() { - if (writer is null) + if (m_writer is null) { - throw IllegalStateException.Create("Cannot commit on an closed writer. Add documents first"); + if (m_searcherMgr is null || closeIndexWriterOnBuild == false) + { + throw IllegalStateException.Create("Cannot commit on a closed writer. Add documents first"); + } + // else no-op: writer was committed and closed after the index was built, so commit is unnecessary + } + else + { + m_writer.Commit(); } - writer.Commit(); } private Analyzer GetGramAnalyzer() @@ -335,24 +426,40 @@ protected override TokenStreamComponents WrapComponents(string fieldName, TokenS } } - //LUCENENET specific -Support for LUCENE - 5889. + // LUCENENET specific - Support for LUCENE-5889, LUCENE-7564. private void EnsureOpen() { - if (writer != null) + if (m_writer != null) return; UninterruptableMonitor.Enter(syncLock); try { - if (writer is null) + if (m_writer is null) { - if (m_searcherMgr != null) + if (DirectoryReader.IndexExists(dir)) + { + // Already built; open it: + m_writer = new IndexWriter(dir, indexWriterConfigFactory.Get(matchVersion, GetGramAnalyzer(), OpenMode.APPEND)); + } + else { - m_searcherMgr.Dispose(); - m_searcherMgr = null; + m_writer = new IndexWriter(dir, indexWriterConfigFactory.Get(matchVersion, GetGramAnalyzer(), OpenMode.CREATE)); + } + UninterruptableMonitor.Enter(searcherMgrLock); + try + { + SearcherManager oldSearcherMgr = m_searcherMgr; + m_searcherMgr = new SearcherManager(m_writer, true, null); + if (oldSearcherMgr != null) + { + oldSearcherMgr.Dispose(); + } + } + finally + { + UninterruptableMonitor.Exit(searcherMgrLock); } - writer = new IndexWriter(dir, indexWriterConfigFactory.Get(matchVersion, GetGramAnalyzer(), OpenMode.CREATE)); - m_searcherMgr = new SearcherManager(writer, true, null); } } finally @@ -370,14 +477,14 @@ private void EnsureOpen() /// public virtual void Add(BytesRef text, IEnumerable contexts, long weight, BytesRef payload) { - EnsureOpen(); //LUCENENET specific -Support for LUCENE - 5889. - writer.AddDocument(BuildDocument(text, contexts, weight, payload)); + EnsureOpen(); // LUCENENET specific - Support for LUCENE-5889. + m_writer.AddDocument(BuildDocument(text, contexts, weight, payload)); } /// /// Updates a previous suggestion, matching the exact same /// text as before. Use this to change the weight or - /// payload of an already added suggstion. If you know + /// payload of an already added suggestion. If you know /// this text is not already present you can use /// instead. After adding or updating a batch of /// new suggestions, you must call in the @@ -385,7 +492,8 @@ public virtual void Add(BytesRef text, IEnumerable contexts, long weig /// public virtual void Update(BytesRef text, IEnumerable contexts, long weight, BytesRef payload) { - writer.UpdateDocument(new Term(EXACT_TEXT_FIELD_NAME, text.Utf8ToString()), BuildDocument(text, contexts, weight, payload)); + EnsureOpen(); // LUCENENET specific - Support for LUCENE-5889. + m_writer.UpdateDocument(new Term(EXACT_TEXT_FIELD_NAME, text.Utf8ToString()), BuildDocument(text, contexts, weight, payload)); } private Document BuildDocument(BytesRef text, IEnumerable contexts, long weight, BytesRef payload) @@ -424,11 +532,16 @@ private Document BuildDocument(BytesRef text, IEnumerable contexts, lo /// public virtual void Refresh() { - if (m_searcherMgr is null) // LUCENENET specific -Support for LUCENE-5889. + if (m_searcherMgr is null) // LUCENENET specific - Support for LUCENE-5889. { throw IllegalStateException.Create("suggester was not built"); } - m_searcherMgr.MaybeRefreshBlocking(); + if (m_writer != null) // LUCENENET specific - Support for LUCENE-7564. + { + m_searcherMgr.MaybeRefreshBlocking(); + } + // else no-op: writer was committed and closed after the index was built + // and before searcherMgr was constructed, so refresh is unnecessary } /// @@ -607,8 +720,19 @@ public virtual IList DoLookup(string key, // We sorted postings by weight during indexing, so we // only retrieve the first num hits now: ICollector c2 = new EarlyTerminatingSortingCollector(c, SORT, num); - IndexSearcher searcher = m_searcherMgr.Acquire(); IList results = null; + SearcherManager mgr; // LUCENENET specific - Support for LUCENE-7564: acquire & release on same SearcherManager, via local reference + IndexSearcher searcher; + UninterruptableMonitor.Enter(searcherMgrLock); + try + { + mgr = m_searcherMgr; + searcher = mgr.Acquire(); + } + finally + { + UninterruptableMonitor.Exit(searcherMgrLock); + } try { //System.out.println("got searcher=" + searcher); @@ -622,7 +746,7 @@ public virtual IList DoLookup(string key, } finally { - m_searcherMgr.Release(searcher); + mgr.Release(searcher); } //System.out.println(((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - t0) + " msec for infix suggest"); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results @@ -848,11 +972,14 @@ protected virtual void Dispose(bool disposing) // LUCENENET specific - implement m_searcherMgr.Dispose(); m_searcherMgr = null; } - if (writer != null) + if (m_writer != null) + { + m_writer.Dispose(); + m_writer = null; + } + if (dir != null) // LUCENENET specific - Support for LUCENE-7564. Close dir even when writer is null. { - writer.Dispose(); dir.Dispose(); - writer = null; } } } @@ -864,7 +991,18 @@ public override long GetSizeInBytes() { if (m_searcherMgr != null) { - IndexSearcher searcher = m_searcherMgr.Acquire(); + SearcherManager mgr; // LUCENENET specific - Support for LUCENE-7564: acquire & release on same SearcherManager, via local reference + IndexSearcher searcher; + UninterruptableMonitor.Enter(searcherMgrLock); + try + { + mgr = m_searcherMgr; + searcher = mgr.Acquire(); + } + finally + { + UninterruptableMonitor.Exit(searcherMgrLock); + } try { foreach (AtomicReaderContext context in searcher.IndexReader.Leaves) @@ -878,7 +1016,7 @@ public override long GetSizeInBytes() } finally { - m_searcherMgr.Release(searcher); + mgr.Release(searcher); } } return mem; @@ -897,14 +1035,25 @@ public override long Count { return 0; } - IndexSearcher searcher = m_searcherMgr.Acquire(); + SearcherManager mgr; // LUCENENET specific - Support for LUCENE-7564: acquire & release on same SearcherManager, via local reference + IndexSearcher searcher; + UninterruptableMonitor.Enter(searcherMgrLock); + try + { + mgr = m_searcherMgr; + searcher = mgr.Acquire(); + } + finally + { + UninterruptableMonitor.Exit(searcherMgrLock); + } try { return searcher.IndexReader.NumDocs; } finally { - m_searcherMgr.Release(searcher); + mgr.Release(searcher); } } } diff --git a/src/Lucene.Net.Suggest/Suggest/Analyzing/BlendedInfixSuggester.cs b/src/Lucene.Net.Suggest/Suggest/Analyzing/BlendedInfixSuggester.cs index 643965f20c..5d7e351634 100644 --- a/src/Lucene.Net.Suggest/Suggest/Analyzing/BlendedInfixSuggester.cs +++ b/src/Lucene.Net.Suggest/Suggest/Analyzing/BlendedInfixSuggester.cs @@ -105,8 +105,8 @@ public BlendedInfixSuggester(LuceneVersion matchVersion, Directory dir, Analyzer /// Type of blending strategy, see BlenderType for more precisions /// Factor to multiply the number of searched elements before ponderate /// If there are problems opening the underlying Lucene index. - // LUCENENET specific - LUCENE-5889, a 4.11.0 feature. calls new constructor with extra param. - // LUCENENET UPGRADE TODO: Remove method at version 4.11.0. Was retained for perfect 4.8 compatibility + // LUCENENET specific - retained for backwards compatibility. Calls new constructor with default + // commitOnBuild value. Java does not have this overload without commitOnBuild post-LUCENE-5889. public BlendedInfixSuggester(LuceneVersion matchVersion, Directory dir, Analyzer indexAnalyzer, Analyzer queryAnalyzer, int minPrefixChars, BlenderType blenderType, int numFactor) : this(matchVersion, dir, indexAnalyzer, queryAnalyzer, minPrefixChars, blenderType, numFactor, commitOnBuild: false) diff --git a/src/Lucene.Net.Tests.Suggest/Suggest/Analyzing/AnalyzingInfixSuggesterTest.cs b/src/Lucene.Net.Tests.Suggest/Suggest/Analyzing/AnalyzingInfixSuggesterTest.cs index 0e4a835941..e7e761eff9 100644 --- a/src/Lucene.Net.Tests.Suggest/Suggest/Analyzing/AnalyzingInfixSuggesterTest.cs +++ b/src/Lucene.Net.Tests.Suggest/Suggest/Analyzing/AnalyzingInfixSuggesterTest.cs @@ -4,6 +4,7 @@ using J2N.Threading.Atomic; using Lucene.Net.Analysis; using Lucene.Net.Analysis.Core; +using Lucene.Net.Index; using Lucene.Net.Analysis.TokenAttributes; using Lucene.Net.Analysis.Util; using Lucene.Net.Support.Threading; @@ -52,7 +53,7 @@ public void TestBasic() }; Analyzer a = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, false); - using AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + using AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, false); suggester.Build(new InputArrayEnumerator(keys)); IList results = suggester.DoLookup(TestUtil.StringToCharSequence("ear", Random).ToString(), 10, true, true); @@ -95,14 +96,14 @@ public void TestAfterLoad() DirectoryInfo tempDir = CreateTempDir("AnalyzingInfixSuggesterTest"); Analyzer a = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, false); - AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, 3); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, 3, false); try { suggester.Build(new InputArrayEnumerator(keys)); assertEquals(2, suggester.Count); suggester.Dispose(); - suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, 3); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, 3, false); IList results = suggester.DoLookup(TestUtil.StringToCharSequence("ear", Random).ToString(), 10, true, true); assertEquals(2, results.size()); assertEquals("a penny saved is a penny earned", results[0].Key); @@ -145,7 +146,7 @@ public override string ToString() internal class TestHighlightAnalyzingInfixSuggester : AnalyzingInfixSuggester { public TestHighlightAnalyzingInfixSuggester(AnalyzingInfixSuggesterTest outerInstance, Analyzer a) - : base(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3) //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + : base(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, false) { } @@ -254,7 +255,7 @@ public void TestRandomMinPrefixLength() Analyzer a = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, false); int minPrefixLength = Random.nextInt(10); - AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, minPrefixLength); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, minPrefixLength, false); try { suggester.Build(new InputArrayEnumerator(keys)); @@ -330,7 +331,7 @@ public void TestRandomMinPrefixLength() // Make sure things still work after close and reopen: suggester.Dispose(); - suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, minPrefixLength); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, minPrefixLength, false); } } finally @@ -348,7 +349,7 @@ public void TestHighlight() }; Analyzer a = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, false); - using AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + using AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, false); suggester.Build(new InputArrayEnumerator(keys)); IList results = suggester.DoLookup(TestUtil.StringToCharSequence("penn", Random).ToString(), 10, true, true); assertEquals(1, results.size()); @@ -359,7 +360,7 @@ internal class TestHighlightChangeCaseAnalyzingInfixSuggester : AnalyzingInfixSu { private readonly AnalyzingInfixSuggesterTest outerInstance; public TestHighlightChangeCaseAnalyzingInfixSuggester(AnalyzingInfixSuggesterTest outerInstance, Analyzer a) - : base(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3) //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + : base(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, false) { this.outerInstance = outerInstance; } @@ -381,7 +382,7 @@ public void TestHighlightCaseChange() Analyzer a = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, true); IList results; - using (AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3)) //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + using (AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, false)) { suggester.Build(new InputArrayEnumerator(keys)); results = suggester.DoLookup(TestUtil.StringToCharSequence("penn", Random).ToString(), 10, true, true); @@ -445,7 +446,7 @@ public void TestSuggestStopFilter() public void TestEmptyAtStart() { Analyzer a = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, false); - using AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + using AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, false); suggester.Build(new InputArrayEnumerator(new Input[0])); suggester.Add(new BytesRef("a penny saved is a penny earned"), null, 10, new BytesRef("foobaz")); suggester.Add(new BytesRef("lend me your ear"), null, 8, new BytesRef("foobar")); @@ -483,7 +484,7 @@ public void TestEmptyAtStart() public void TestBothExactAndPrefix() { Analyzer a = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, false); - using AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + using AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, false); suggester.Build(new InputArrayEnumerator(new Input[0])); suggester.Add(new BytesRef("the pen is pretty"), null, 10, new BytesRef("foobaz")); suggester.Refresh(); @@ -595,7 +596,7 @@ public void TestRandomNRT() Console.WriteLine(" minPrefixChars=" + minPrefixChars); } - AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, minPrefixChars); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, minPrefixChars, false); try { @@ -692,7 +693,7 @@ public void TestRandomNRT() } lookupThread.Finish(); suggester.Dispose(); - suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, minPrefixChars); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, minPrefixChars, false); lookupThread = new LookupThread(this, suggester); lookupThread.Start(); @@ -896,7 +897,7 @@ public void TestBasicNRT() }; Analyzer a = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, false); - using AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + using AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, false); suggester.Build(new InputArrayEnumerator(keys)); IList results = suggester.DoLookup(TestUtil.StringToCharSequence("ear", Random).ToString(), 10, true, true); @@ -1034,6 +1035,145 @@ private ISet AsSet(params string[] values) return result; } + // LUCENENET specific - helper subclass for LUCENE-7564 tests. + // Exposes writer and searcherMgr for assertions. + internal class MyAnalyzingInfixSuggester : AnalyzingInfixSuggester + { + public MyAnalyzingInfixSuggester(Store.Directory dir, Analyzer indexAnalyzer, Analyzer queryAnalyzer, + int minPrefixChars, bool commitOnBuild, bool closeIndexWriterOnBuild) + : base(TEST_VERSION_CURRENT, dir, indexAnalyzer, queryAnalyzer, minPrefixChars, commitOnBuild, closeIndexWriterOnBuild) + { + } + + public IndexWriter GetIndexWriter() + { + return m_writer; + } + + public SearcherManager GetSearcherManager() + { + return m_searcherMgr; + } + } + + // LUCENENET specific - backported from LUCENE-7564, with additions from LUCENE-7670. + [Test] + public void TestCloseIndexWriterOnBuild() + { + Input[] sharedInputs = new Input[] + { + new Input("lend me your ear", 8, new BytesRef("foobar")), + new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")), + }; + + // After Build(), when closeIndexWriterOnBuild = true: + // * The IndexWriter should be null + // * The SearcherManager should be non-null + // * SearcherManager's IndexWriter reference should be closed + // (as evidenced by MaybeRefreshBlocking() throwing AlreadyClosedException/ObjectDisposedException) + Analyzer a = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, false); + DirectoryInfo tempDir = CreateTempDir("analyzingInfixContext"); + MyAnalyzingInfixSuggester suggester = new MyAnalyzingInfixSuggester(NewFSDirectory(tempDir), a, a, 3, false, true); + suggester.Build(new InputArrayEnumerator(sharedInputs)); + assertNull(suggester.GetIndexWriter()); + assertNotNull(suggester.GetSearcherManager()); + Assert.Throws(() => suggester.GetSearcherManager().MaybeRefreshBlocking()); + + suggester.Dispose(); + + // LUCENENET specific - backported from LUCENE-7670. + // After instantiating from an already-built suggester dir: + // * The IndexWriter should be null + // * The SearcherManager should be non-null + MyAnalyzingInfixSuggester suggester2 = new MyAnalyzingInfixSuggester(NewFSDirectory(tempDir), a, a, 3, false, true); + assertNull(suggester2.GetIndexWriter()); + assertNotNull(suggester2.GetSearcherManager()); + + suggester2.Dispose(); + } + + // LUCENENET specific - backported from LUCENE-7564. + [Test] + public void TestCommitAfterBuild() + { + PerformOperationWithAllOptionCombinations(suggester => + { + suggester.Build(new InputArrayEnumerator(SharedInputs)); + suggester.Commit(); + }); + } + + // LUCENENET specific - backported from LUCENE-7564. + [Test] + public void TestRefreshAfterBuild() + { + PerformOperationWithAllOptionCombinations(suggester => + { + suggester.Build(new InputArrayEnumerator(SharedInputs)); + suggester.Refresh(); + }); + } + + // LUCENENET specific - backported from LUCENE-7564. + [Test] + public void TestDisallowCommitBeforeBuild() + { + PerformOperationWithAllOptionCombinations(suggester => + Assert.Throws(() => suggester.Commit())); + } + + // LUCENENET specific - backported from LUCENE-7564. + [Test] + public void TestDisallowRefreshBeforeBuild() + { + PerformOperationWithAllOptionCombinations(suggester => + Assert.Throws(() => suggester.Refresh())); + } + + private static readonly Input[] SharedInputs = new Input[] + { + new Input("lend me your ear", 8, new BytesRef("foobar")), + new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")), + }; + + /// + /// Perform the given operation on suggesters constructed with all combinations of options + /// commitOnBuild and closeIndexWriterOnBuild, including defaults. + /// + // LUCENENET specific - backported from LUCENE-7564. + private static void PerformOperationWithAllOptionCombinations(Action operation) + { + Analyzer a = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, false); + + AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a); + operation(suggester); + suggester.Dispose(); + + suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, false); + operation(suggester); + suggester.Dispose(); + + suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, true); + operation(suggester); + suggester.Dispose(); + + suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, true, true); + operation(suggester); + suggester.Dispose(); + + suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, true, false); + operation(suggester); + suggester.Dispose(); + + suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, false, true); + operation(suggester); + suggester.Dispose(); + + suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewDirectory(), a, a, 3, false, false); + operation(suggester); + suggester.Dispose(); + } + // LUCENE-5528 [Test] public void TestBasicContext() @@ -1053,13 +1193,13 @@ public void TestBasicContext() Analyzer a = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, false); if (iter == 0) { - suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, 3); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, 3, false); suggester.Build(new InputArrayEnumerator(keys)); } else { // Test again, after close/reopen: - suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, 3); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, 3, false); } // No context provided, all results returned diff --git a/src/Lucene.Net.Tests.Suggest/Suggest/Analyzing/BlendedInfixSuggesterTest.cs b/src/Lucene.Net.Tests.Suggest/Suggest/Analyzing/BlendedInfixSuggesterTest.cs index bd7828a735..e69a2e544a 100644 --- a/src/Lucene.Net.Tests.Suggest/Suggest/Analyzing/BlendedInfixSuggesterTest.cs +++ b/src/Lucene.Net.Tests.Suggest/Suggest/Analyzing/BlendedInfixSuggesterTest.cs @@ -48,7 +48,7 @@ public void TestBlendedSort() BlendedInfixSuggester suggester = new BlendedInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS, BlendedInfixSuggester.BlenderType.POSITION_LINEAR, - BlendedInfixSuggester.DEFAULT_NUM_FACTOR); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + BlendedInfixSuggester.DEFAULT_NUM_FACTOR, false); suggester.Build(new InputArrayEnumerator(keys)); // we query for star wars and check that the weight @@ -99,7 +99,7 @@ public void TestBlendingType() // BlenderType.RECIPROCAL is using 1/(1+p) * w where w is weight and p the position of the word suggester = new BlendedInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS, - BlendedInfixSuggester.BlenderType.POSITION_RECIPROCAL, 1); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + BlendedInfixSuggester.BlenderType.POSITION_RECIPROCAL, 1, false); suggester.Build(new InputArrayEnumerator(keys)); assertEquals(w, GetInResults(suggester, "top", pl, 1)); @@ -133,7 +133,7 @@ public void TestRequiresMore() // if factor is small, we don't get the expected element BlendedInfixSuggester suggester = new BlendedInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS, - BlendedInfixSuggester.BlenderType.POSITION_RECIPROCAL, 1); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + BlendedInfixSuggester.BlenderType.POSITION_RECIPROCAL, 1, false); suggester.Build(new InputArrayEnumerator(keys)); @@ -153,7 +153,7 @@ public void TestRequiresMore() // if we increase the factor we have it suggester = new BlendedInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS, - BlendedInfixSuggester.BlenderType.POSITION_RECIPROCAL, 2); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + BlendedInfixSuggester.BlenderType.POSITION_RECIPROCAL, 2, false); suggester.Build(new InputArrayEnumerator(keys)); // we have it @@ -185,7 +185,7 @@ public void TestNullPrefixToken() BlendedInfixSuggester suggester = new BlendedInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS, BlendedInfixSuggester.BlenderType.POSITION_LINEAR, - BlendedInfixSuggester.DEFAULT_NUM_FACTOR); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + BlendedInfixSuggester.DEFAULT_NUM_FACTOR, false); suggester.Build(new InputArrayEnumerator(keys)); GetInResults(suggester, "of ", payload, 1); @@ -216,7 +216,7 @@ public void TestTrying() BlendedInfixSuggester suggester = new BlendedInfixSuggester(TEST_VERSION_CURRENT, NewFSDirectory(tempDir), a, a, AnalyzingInfixSuggester.DEFAULT_MIN_PREFIX_CHARS, BlendedInfixSuggester.BlenderType.POSITION_RECIPROCAL, - BlendedInfixSuggester.DEFAULT_NUM_FACTOR); //LUCENENET UPGRADE TODO: add extra false param at version 4.11.0 + BlendedInfixSuggester.DEFAULT_NUM_FACTOR, false); suggester.Build(new InputArrayEnumerator(keys));