Fix IndexWriter.Dispose breaking the standard Dispose contract, #1399

src/Lucene.Net.Benchmark/ByTask/Tasks/CloseIndexTask.cs

@@ -47,7 +47,9 @@ public override int DoLogic()
                 {
                     infoStream.Dispose();
                 }
-                iw.Dispose(doWait);
+#pragma warning disable 612, 618
+                iw.Close(doWait);
+#pragma warning restore 612, 618
                 RunData.IndexWriter = null;
             }
             return 1;

src/Lucene.Net.TestFramework/Analysis/BaseTokenStreamTestCase.cs

@@ -629,15 +629,15 @@ internal class AnalysisThread : ThreadJob
             internal readonly bool simple;
             internal readonly bool offsetsAreCorrect;
             internal readonly RandomIndexWriter iw;
-            private readonly CountdownEvent latch;
+            private readonly CountdownLatch latch;
 
             // NOTE: not volatile because we don't want the tests to
             // add memory barriers (ie alter how threads
             // interact)... so this is just "best effort":
             public bool Failed { get; set; }
             public Exception FirstException { get; set; } = null;
 
-            internal AnalysisThread(long seed, CountdownEvent latch, Analyzer a, int iterations, int maxWordLength,
+            internal AnalysisThread(long seed, CountdownLatch latch, Analyzer a, int iterations, int maxWordLength,
                 bool useCharFilter, bool simple, bool offsetsAreCorrect, RandomIndexWriter iw)
             {
                 this.seed = seed;
@@ -710,7 +710,7 @@ public static void CheckRandomData(Random random, Analyzer a, int iterations, in
                 // now test with multiple threads: note we do the EXACT same thing we did before in each thread,
                 // so this should only really fail from another thread if its an actual thread problem
                 int numThreads = TestUtil.NextInt32(random, 2, 4);
-                var startingGun = new CountdownEvent(1);
+                using var startingGun = new CountdownLatch(1); // LUCENENET: CountdownLatch is disposable in .NET
                 var threads = new AnalysisThread[numThreads];
                 for (int i = 0; i < threads.Length; i++)
                 {

src/Lucene.Net.TestFramework/Codecs/MockRandom/MockRandomPostingsFormat.cs

@@ -1,3 +1,4 @@
+using J2N.Text;
 using Lucene.Net.Codecs.BlockTerms;
 using Lucene.Net.Codecs.Lucene41;
 using Lucene.Net.Codecs.Memory;
@@ -97,7 +98,12 @@ public override Int32IndexInput OpenInput(Directory dir, string fileName, IOCont
             {
                 // Must only use extension, because IW.addIndexes can
                 // rename segment!
-                Int32StreamFactory f = delegates[(Math.Abs(salt ^ GetExtension(fileName).GetHashCode())) % delegates.Count];
+                // LUCENENET specific: use J2N's CharSequenceComparer.Ordinal.GetHashCode(), which produces
+                // the same value as Java's String.hashCode(). The .NET string.GetHashCode() is randomized
+                // per-process, so the factory chosen when writing (e.g. in a forked child process, as in
+                // TestIndexWriterOnJRECrash) would differ from the one chosen when reading in another process,
+                // causing "codec header mismatch" failures. A deterministic hash keeps writer and reader in sync.
+                Int32StreamFactory f = delegates[(Math.Abs(salt ^ CharSequenceComparer.Ordinal.GetHashCode(GetExtension(fileName)))) % delegates.Count];
                 if (LuceneTestCase.Verbose)
                 {
                     Console.WriteLine("MockRandomCodec: read using int factory " + f + " from fileName=" + fileName);
@@ -107,7 +113,8 @@ public override Int32IndexInput OpenInput(Directory dir, string fileName, IOCont
 
             public override Int32IndexOutput CreateOutput(Directory dir, string fileName, IOContext context)
             {
-                Int32StreamFactory f = delegates[(Math.Abs(salt ^ GetExtension(fileName).GetHashCode())) % delegates.Count];
+                // LUCENENET specific: deterministic (Java-parity) hash; see the note in OpenInput().
+                Int32StreamFactory f = delegates[(Math.Abs(salt ^ CharSequenceComparer.Ordinal.GetHashCode(GetExtension(fileName)))) % delegates.Count];
                 if (LuceneTestCase.Verbose)
                 {
                     Console.WriteLine("MockRandomCodec: write using int factory " + f + " to fileName=" + fileName);

src/Lucene.Net.TestFramework/Index/BaseDocValuesFormatTestCase.cs

@@ -7,13 +7,11 @@
 using Lucene.Net.Index.Extensions;
 using Lucene.Net.Search;
 using Lucene.Net.Store;
-using Lucene.Net.Support;
 using Lucene.Net.Util;
 using RandomizedTesting.Generators;
 using System;
 using System.Collections.Generic;
 using System.Globalization;
-using System.Threading;
 using static Lucene.Net.Index.TermsEnum;
 using Assert = Lucene.Net.TestFramework.Assert;
 using JCG = J2N.Collections.Generic;
@@ -3243,7 +3241,7 @@ public virtual void TestThreads()
             using DirectoryReader ir = DirectoryReader.Open(dir);
             int numThreads = TestUtil.NextInt32(Random, 2, 7);
             ThreadJob[] threads = new ThreadJob[numThreads];
-            using CountdownEvent startingGun = new CountdownEvent(1);
+            using CountdownLatch startingGun = new CountdownLatch(1); // LUCENENET: CountdownLatch is disposable in .NET
             for (int i = 0; i < threads.Length; i++)
             {
                 threads[i] = new ThreadAnonymousClass(ir, startingGun);
@@ -3259,9 +3257,9 @@ public virtual void TestThreads()
         private sealed class ThreadAnonymousClass : ThreadJob
         {
             private readonly DirectoryReader ir;
-            private readonly CountdownEvent startingGun;
+            private readonly CountdownLatch startingGun;
 
-            public ThreadAnonymousClass(DirectoryReader ir, CountdownEvent startingGun)
+            public ThreadAnonymousClass(DirectoryReader ir, CountdownLatch startingGun)
             {
                 this.ir = ir;
                 this.startingGun = startingGun;
@@ -3379,7 +3377,7 @@ public virtual void TestThreads2()
             using DirectoryReader ir = DirectoryReader.Open(dir);
             int numThreads = TestUtil.NextInt32(Random, 2, 7);
             ThreadJob[] threads = new ThreadJob[numThreads];
-            using CountdownEvent startingGun = new CountdownEvent(1);
+            using CountdownLatch startingGun = new CountdownLatch(1); // LUCENENET: CountdownLatch is disposable in .NET
             for (int i = 0; i < threads.Length; i++)
             {
                 threads[i] = new ThreadAnonymousClass2(ir, startingGun);
@@ -3395,9 +3393,9 @@ public virtual void TestThreads2()
         private sealed class ThreadAnonymousClass2 : ThreadJob
         {
             private readonly DirectoryReader ir;
-            private readonly CountdownEvent startingGun;
+            private readonly CountdownLatch startingGun;
 
-            public ThreadAnonymousClass2(DirectoryReader ir, CountdownEvent startingGun)
+            public ThreadAnonymousClass2(DirectoryReader ir, CountdownLatch startingGun)
             {
                 this.ir = ir;
                 this.startingGun = startingGun;

src/Lucene.Net.TestFramework/Index/RandomCodec.cs

@@ -1,3 +1,4 @@
+using J2N.Text;
 using Lucene.Net.Codecs;
 using Lucene.Net.Codecs.Asserting;
 using Lucene.Net.Codecs.Bloom;
@@ -84,11 +85,16 @@ public override PostingsFormat GetPostingsFormatForField(string name)
         {
             if (!previousMappings.TryGetValue(name, out PostingsFormat codec) || codec is null)
             {
-                codec = formats[Math.Abs(perFieldSeed ^ name.GetHashCode()) % formats.Count];
+                // LUCENENET specific: use J2N's CharSequenceComparer.Ordinal.GetHashCode(), which produces
+                // the same value as Java's String.hashCode(). The .NET string.GetHashCode() is randomized
+                // per-process, so a field's format chosen when writing (e.g. in a forked child process, as
+                // in TestIndexWriterOnJRECrash) would differ from the one chosen when reading in another
+                // process, causing "codec header mismatch" failures. A deterministic hash keeps them in sync.
+                codec = formats[Math.Abs(perFieldSeed ^ CharSequenceComparer.Ordinal.GetHashCode(name)) % formats.Count];
                 if (codec is SimpleTextPostingsFormat && perFieldSeed % 5 != 0)
                 {
                     // make simpletext rarer, choose again
-                    codec = formats[Math.Abs(perFieldSeed ^ name.ToUpperInvariant().GetHashCode()) % formats.Count];
+                    codec = formats[Math.Abs(perFieldSeed ^ CharSequenceComparer.Ordinal.GetHashCode(name.ToUpperInvariant())) % formats.Count];
                 }
                 previousMappings[name] = codec;
                 // Safety:
@@ -107,11 +113,12 @@ public override DocValuesFormat GetDocValuesFormatForField(string name)
         {
             if (!previousDVMappings.TryGetValue(name, out DocValuesFormat codec) || codec is null)
             {
-                codec = dvFormats[Math.Abs(perFieldSeed ^ name.GetHashCode()) % dvFormats.Count];
+                // LUCENENET specific: deterministic (Java-parity) hash; see the note in GetPostingsFormatForField().
+                codec = dvFormats[Math.Abs(perFieldSeed ^ CharSequenceComparer.Ordinal.GetHashCode(name)) % dvFormats.Count];
                 if (codec is SimpleTextDocValuesFormat && perFieldSeed % 5 != 0)
                 {
                     // make simpletext rarer, choose again
-                    codec = dvFormats[Math.Abs(perFieldSeed ^ name.ToUpperInvariant().GetHashCode()) % dvFormats.Count];
+                    codec = dvFormats[Math.Abs(perFieldSeed ^ CharSequenceComparer.Ordinal.GetHashCode(name.ToUpperInvariant())) % dvFormats.Count];
                 }
                 previousDVMappings[name] = codec;
                 // Safety:

src/Lucene.Net.TestFramework/Index/RandomIndexWriter.cs

@@ -449,7 +449,13 @@ protected virtual void Dispose(bool disposing)
             {
                 // if someone isn't using getReader() API, we want to be sure to
                 // forceMerge since presumably they might open a reader on the dir.
-                if (getReaderCalled == false && r.Next(8) == 2)
+                // LUCENENET: the `IndexWriter.IsClosed == false` guard is ported from upstream
+                // LUCENE-5871 (commit 2cfcdcc, first released in 4.10.0), pulled in alongside
+                // the LUCENE-5871 IndexWriter close fix (#1284). Without it the new Shutdown
+                // contract can leave the writer closed even when the test's user-visible call
+                // threw, and the random force-merge here would then throw
+                // AlreadyClosedException during teardown.
+                if (getReaderCalled == false && r.Next(8) == 2 && IndexWriter.IsClosed == false)
                 {
                     _DoRandomForceMerge();
                 }

src/Lucene.Net.TestFramework/Index/ThreadedIndexingAndSearchingTestCase.cs

@@ -772,7 +772,9 @@ public virtual void RunTest(string testName)
             assertEquals("index=" + m_writer.SegString() + " addCount=" + m_addCount + " delCount=" + m_delCount, m_addCount - m_delCount, m_writer.NumDocs);
 
             DoClose();
-            m_writer.Dispose(false);
+#pragma warning disable 612, 618
+            m_writer.Close(false);
+#pragma warning restore 612, 618
 
             // Cannot shutdown until after writer is closed because
             // writer has merged segment warmer that uses IS to run

src/Lucene.Net.TestFramework/Search/RandomSimilarityProvider.cs

@@ -1,4 +1,5 @@
 using J2N.Collections.Generic.Extensions;
+using J2N.Text;
 using Lucene.Net.Diagnostics;
 using Lucene.Net.Search.Similarities;
 using Lucene.Net.Support.Threading;
@@ -78,7 +79,12 @@ public override Similarity Get(string field)
                 if (Debugging.AssertsEnabled) Debugging.Assert(field != null);
                 if (!previousMappings.TryGetValue(field, out Similarity sim) || sim is null)
                 {
-                    sim = knownSims[Math.Max(0, Math.Abs(perFieldSeed ^ field.GetHashCode())) % knownSims.Count];
+                    // LUCENENET specific: use J2N's CharSequenceComparer.Ordinal.GetHashCode(), which produces
+                    // the same value as Java's String.hashCode(). The .NET string.GetHashCode() is randomized
+                    // per-process, so the per-field similarity chosen in one process would differ from another,
+                    // breaking reproducibility across processes (e.g. the forked TestIndexWriterOnJRECrash) and
+                    // diverging from Java. A deterministic hash keeps the choice stable.
+                    sim = knownSims[Math.Max(0, Math.Abs(perFieldSeed ^ CharSequenceComparer.Ordinal.GetHashCode(field))) % knownSims.Count];
                     previousMappings[field] = sim;
                 }
                 return sim;

src/Lucene.Net.TestFramework/Store/MockDirectoryWrapper.cs

@@ -616,6 +616,15 @@ private void DeleteFile(string name, bool forced)
                     }
                 }
                 m_input.DeleteFile(name);
+                // LUCENENET: backport from upstream commit 6369012d ("fix a few nocommits",
+                // Mike McCandless), which landed on the LUCENE-5438 (NRT replication) branch and
+                // first shipped in Lucene 6.0.0. Removing a deleted file's name from createdFiles
+                // lets the same segment name be re-created later without preventDoubleWrite falsely
+                // reporting "already written to". Without this, a benign upstream segment-name reuse
+                // on reopen after Dispose(false) (the committed counter can trail a name issued by a
+                // background merge that was then aborted and its files deleted) surfaced as a flaky
+                // TestNoWaitClose failure. See #1284.
+                createdFiles.Remove(name);
             }
             finally
             {

src/Lucene.Net.Tests/Index/TestAddIndexes.cs

@@ -804,7 +804,9 @@ internal virtual void JoinThreads()
             internal virtual void Close(bool doWait)
             {
                 didClose = true;
-                writer2.Dispose(doWait);
+#pragma warning disable 612, 618
+                writer2.Close(doWait);
+#pragma warning restore 612, 618
             }
 
             internal virtual void CloseDir()

src/Lucene.Net.Tests/Index/TestBackwardsCompatibility.cs

@@ -313,7 +313,9 @@ public virtual void TestUnsupportedOldIndexes()
                     // above, so close without waiting for merges.
                     if (writer != null)
                     {
-                        writer.Dispose(false);
+#pragma warning disable 612, 618
+                        writer.Close(false);
+#pragma warning restore 612, 618
                     }
                     writer = null;
                 }
@@ -1050,7 +1052,9 @@ public virtual void TestUpgradeOldSingleSegmentIndexWithAdditions()
                     {
                         AddDoc(w, id++);
                     }
-                    w.Dispose(false);
+#pragma warning disable 612, 618
+                    w.Close(false);
+#pragma warning restore 612, 618
                 }
 
                 // add dummy segments (which are all in current
@@ -1061,7 +1065,9 @@ public virtual void TestUpgradeOldSingleSegmentIndexWithAdditions()
                     .SetMergePolicy(mp_);
                 IndexWriter iw = new IndexWriter(dir, iwc_);
                 iw.AddIndexes(ramDir);
-                iw.Dispose(false);
+#pragma warning disable 612, 618
+                iw.Close(false);
+#pragma warning restore 612, 618
 
                 // determine count of segments in modified index
                 int origSegCount = GetNumberOfSegments(dir);

src/Lucene.Net.Tests/Index/TestBackwardsCompatibility3x.cs

@@ -244,7 +244,9 @@ public virtual void TestUnsupportedOldIndexes()
                     // above, so close without waiting for merges.
                     if (writer != null)
                     {
-                        writer.Dispose(false);
+#pragma warning disable 612, 618
+                        writer.Close(false);
+#pragma warning restore 612, 618
                     }
                     writer = null;
                 }
@@ -955,7 +957,9 @@ public virtual void TestUpgradeOldSingleSegmentIndexWithAdditions()
                     {
                         AddDoc(w, id++);
                     }
-                    w.Dispose(false);
+#pragma warning disable 612, 618
+                    w.Close(false);
+#pragma warning restore 612, 618
                 }
 
                 // add dummy segments (which are all in current
@@ -965,7 +969,9 @@ public virtual void TestUpgradeOldSingleSegmentIndexWithAdditions()
                     .SetMergePolicy(mp_);
                 IndexWriter w_ = new IndexWriter(dir, iwc_);
                 w_.AddIndexes(ramDir);
-                w_.Dispose(false);
+#pragma warning disable 612, 618
+                w_.Close(false);
+#pragma warning restore 612, 618
 
                 // determine count of segments in modified index
                 int origSegCount = GetNumberOfSegments(dir);

src/Lucene.Net.Tests/Index/TestBagOfPositions.cs

@@ -1,17 +1,15 @@
 using J2N.Collections.Generic.Extensions;
 using J2N.Threading;
 using Lucene.Net.Documents;
-using Lucene.Net.Support.Threading;
 using NUnit.Framework;
 using RandomizedTesting.Generators;
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Globalization;
 using System.Text;
-using System.Threading;
-using JCG = J2N.Collections.Generic;
 using Assert = Lucene.Net.TestFramework.Assert;
+using JCG = J2N.Collections.Generic;
 
 namespace Lucene.Net.Index
 {
@@ -117,7 +115,7 @@ public virtual void Test()
             // else just positions
 
             ThreadJob[] threads = new ThreadJob[threadCount];
-            CountdownEvent startingGun = new CountdownEvent(1);
+            using CountdownLatch startingGun = new CountdownLatch(1); // LUCENENET: CountdownLatch is disposable in .NET
 
             for (int threadID = 0; threadID < threadCount; threadID++)
             {
@@ -160,12 +158,12 @@ private sealed class ThreadAnonymousClass : ThreadJob
             private readonly int maxTermsPerDoc;
             private readonly ConcurrentQueue<string> postings;
             private readonly RandomIndexWriter iw;
-            private readonly CountdownEvent startingGun;
+            private readonly CountdownLatch startingGun;
             private readonly Random threadRandom;
             private readonly Document document;
             private readonly Field field;
 
-            public ThreadAnonymousClass(int maxTermsPerDoc, ConcurrentQueue<string> postings, RandomIndexWriter iw, CountdownEvent startingGun, Random threadRandom, Document document, Field field)
+            public ThreadAnonymousClass(int maxTermsPerDoc, ConcurrentQueue<string> postings, RandomIndexWriter iw, CountdownLatch startingGun, Random threadRandom, Document document, Field field)
             {
                 this.maxTermsPerDoc = maxTermsPerDoc;
                 this.postings = postings;

src/Lucene.Net.Tests/Index/TestBagOfPostings.cs

@@ -1,14 +1,12 @@
 using J2N.Collections.Generic.Extensions;
 using J2N.Threading;
 using Lucene.Net.Documents;
-using Lucene.Net.Support.Threading;
 using NUnit.Framework;
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Globalization;
 using System.Text;
-using System.Threading;
 using Assert = Lucene.Net.TestFramework.Assert;
 using JCG = J2N.Collections.Generic;
 
@@ -96,7 +94,7 @@ public virtual void Test()
             }
 
             ThreadJob[] threads = new ThreadJob[threadCount];
-            CountdownEvent startingGun = new CountdownEvent(1);
+            using CountdownLatch startingGun = new CountdownLatch(1); // LUCENENET: CountdownLatch is disposable in .NET
 
             for (int threadID = 0; threadID < threadCount; threadID++)
             {
@@ -141,9 +139,9 @@ private sealed class ThreadAnonymousClass : ThreadJob
             private readonly int maxTermsPerDoc;
             private readonly ConcurrentQueue<string> postings;
             private readonly RandomIndexWriter iw;
-            private readonly CountdownEvent startingGun;
+            private readonly CountdownLatch startingGun;
 
-            public ThreadAnonymousClass(int maxTermsPerDoc, ConcurrentQueue<string> postings, RandomIndexWriter iw, CountdownEvent startingGun)
+            public ThreadAnonymousClass(int maxTermsPerDoc, ConcurrentQueue<string> postings, RandomIndexWriter iw, CountdownLatch startingGun)
             {
                 this.maxTermsPerDoc = maxTermsPerDoc;
                 this.postings = postings;