/// Tests MMapDirectory's MultiMMapIndexInput
- ///
- /// Because Java's ByteBuffer uses an int to address the
- /// values, it's necessary to access a file >
- /// Integer.MAX_VALUE in size using multiple byte buffers.
+ ///
+ /// Because .NET's and use an int to address the
+ /// values, and because we use a similar chunking approach to Lucene, it's necessary to access a file >
+ /// in size using multiple byte buffers.
///
[TestFixture]
public class TestMultiMMap : LuceneTestCase
@@ -225,6 +230,79 @@ public virtual void TestCloneSliceClose()
mmapDir.Dispose();
}
+ // LUCENENET specific: exercises the shared MemoryMappedFile refactor
+ // where OpenInput, CreateSlicer, its slices, and clones all piggyback
+ // on a single MemoryMappedFile per file (per directory instance).
+ // Verifies that (a) concurrent IndexInputs all see correct bytes,
+ // (b) disposing in arbitrary order keeps siblings functional, and
+ // (c) once the last referrer is disposed the OS handle is released
+ // (on Windows a still-open mapping would prevent the file delete).
+ [Test, LuceneNetSpecific]
+ public virtual void TestSharedMappingLifecycle()
+ {
+ var tempDir = CreateTempDir("testSharedMappingLifecycle");
+ MMapDirectory mmapDir = new MMapDirectory(tempDir);
+ const string name = "bytes";
+ using (IndexOutput io = mmapDir.CreateOutput(name, NewIOContext(Random)))
+ {
+ // 4 ints at offsets 0, 4, 8, 12 — each slice reads a known value.
+ io.WriteInt32(10);
+ io.WriteInt32(20);
+ io.WriteInt32(30);
+ io.WriteInt32(40);
+ }
+
+ // Open several IndexInputs for the same file through both
+ // OpenInput and CreateSlicer. All should share one mapping.
+ IndexInput root = mmapDir.OpenInput(name, IOContext.DEFAULT);
+ IndexInput rootClone = (IndexInput)root.Clone();
+
+ IndexInputSlicer slicer = mmapDir.CreateSlicer(name, NewIOContext(Random));
+ IndexInput sliceA = slicer.OpenSlice("a", 0, 4);
+ IndexInput sliceB = slicer.OpenSlice("b", 8, 4);
+ IndexInput sliceAClone = (IndexInput)sliceA.Clone();
+
+ // Reads across all instances must be independent and correct.
+ Assert.AreEqual(10, root.ReadInt32());
+ Assert.AreEqual(10, rootClone.ReadInt32());
+ Assert.AreEqual(10, sliceA.ReadInt32());
+ Assert.AreEqual(30, sliceB.ReadInt32());
+ Assert.AreEqual(10, sliceAClone.ReadInt32());
+
+ // Dispose a clone first; the root and siblings must keep working.
+ rootClone.Dispose();
+ root.Seek(4);
+ Assert.AreEqual(20, root.ReadInt32());
+ sliceB.Seek(0);
+ Assert.AreEqual(30, sliceB.ReadInt32());
+
+ // Dispose a slice; its siblings from the same slicer must keep working.
+ sliceAClone.Dispose();
+ sliceA.Seek(0);
+ Assert.AreEqual(10, sliceA.ReadInt32());
+
+ // Dispose the remaining slice-side instances. The root IndexInput
+ // owns its own mapping, so it must stay alive and readable.
+ sliceA.Dispose();
+ sliceB.Dispose();
+ slicer.Dispose();
+
+ root.Seek(12);
+ Assert.AreEqual(40, root.ReadInt32());
+
+ // Disposing the root tears down its MemoryMappedFile and backing
+ // FileStream.
+ root.Dispose();
+
+ // If any OS file handle is still open, this delete will fail on
+ // Windows. On Unix it silently unlinks but the test still proves
+ // the read-phase invariants above.
+ mmapDir.DeleteFile(name);
+ Assert.IsFalse(File.Exists(Path.Combine(tempDir.FullName, name)));
+
+ mmapDir.Dispose();
+ }
+
[Test]
public virtual void TestSeekZero()
{
@@ -419,16 +497,34 @@ private void AssertChunking(Random random, int chunkSize)
// LUCENENET: Regression test for GitHub #1090. A background thread
// extends a file on disk while the foreground thread repeatedly
- // opens it with MMapDirectory.OpenInput. Before the fix, the
- // file's length could grow between the caller capturing fc.Length
- // and MemoryMappedFile.CreateFromFile performing its internal
- // stat, causing ArgumentOutOfRangeException (paramName="capacity")
- // with the message "The capacity may not be smaller than the
- // file size."
- // NonParallelizable: the retry-path assertion reads static counters on
- // MMapDirectory, so any other test exercising MMapDirectory in parallel
- // could skew the observed retry count.
- [Test, LuceneNetSpecific, Slow, NonParallelizable]
+ // opens it with MMapDirectory.OpenInput. The original failure
+ // mode was ArgumentOutOfRangeException(paramName="capacity")
+ // from MemoryMappedFile.CreateFromFile, because the on-disk file
+ // size could exceed our caller-computed capacity by the time the
+ // framework did its internal stat. .NET Framework's
+ // CreateFromFile reads fileStream.Length multiple times
+ // non-atomically (referencesource MemoryMappedFile.cs L192-L243);
+ // modern .NET snapshots it into a single local
+ // (dotnet/runtime MemoryMappedFile.cs L237-L268). Even when we
+ // pass capacity: 0 the .NET Framework path still races because
+ // the length is re-read for both the defaulting step and the
+ // capacity-vs-size guard. SharedMapping.Create handles the
+ // residual race with a retry loop. This test asserts that
+ // OpenInput continues to succeed under concurrent file
+ // extension.
+ //
+ // Test design notes:
+ // - The writer extends only (never truncates). Truncating a
+ // user-mapped file on Windows fails with ERROR_USER_MAPPED_FILE
+ // and is unrelated to what we're verifying here.
+ // - The reader runs a bounded number of iterations rather than a
+ // wall-clock loop. Sustained mmap churn (thousands of
+ // map/unmap pairs per second) can transiently exhaust Windows
+ // kernel resources (ERROR_NO_SYSTEM_RESOURCES,
+ // ERROR_ACCESS_DENIED on view creation), which is also
+ // unrelated to the capacity race. A few hundred iterations
+ // are plenty to repeatedly hit the race window.
+ [Test, LuceneNetSpecific, Nightly]
public void TestOpenInputConcurrentFileExtension_Issue1090()
{
var dir = CreateTempDir("testOpenInputConcurrentFileExtension");
@@ -440,8 +536,8 @@ public void TestOpenInputConcurrentFileExtension_Issue1090()
using var mmapDir = new MMapDirectory(dir);
- const long maxFileSize = 1L * 1024 * 1024; // 1 MiB cap
- var stop = new ManualResetEventSlim(false);
+ const long maxFileSize = 64L * 1024 * 1024; // 64 MiB safety cap
+ using var stop = new ManualResetEventSlim(false);
Exception writerError = null;
var writer = new Thread(() =>
@@ -449,19 +545,19 @@ public void TestOpenInputConcurrentFileExtension_Issue1090()
var chunk = new byte[64];
try
{
+ // ReSharper disable once AccessToDisposedClosure - thread joined below
while (!stop.IsSet)
{
using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Write, FileShare.ReadWrite);
fs.Seek(0, SeekOrigin.End);
- if (fs.Length < maxFileSize)
- {
- fs.Write(chunk, 0, chunk.Length);
- }
- else
+ if (fs.Length >= maxFileSize)
{
- // Keep the file bounded: truncate back and grow again.
- fs.SetLength(64);
+ // Stop extending if we somehow reach the cap. The
+ // reader's bounded iteration count guarantees this
+ // is far above what we'll hit in a normal run.
+ break;
}
+ fs.Write(chunk, 0, chunk.Length);
}
}
catch (Exception e)
@@ -472,80 +568,2135 @@ public void TestOpenInputConcurrentFileExtension_Issue1090()
{ IsBackground = true, Name = "mmap-issue1090-extender" };
writer.Start();
- // Snapshot counters so this test's assertion is not affected by
- // any earlier test's activity on MMapDirectory.
- long baselineRetries = Interlocked.Read(ref MMapDirectory.s_capacityRetryCount);
-
try
{
- var sw = Stopwatch.StartNew();
- int iterations = 0;
- // Keep stretching the window until either the race fires or we
- // hit a hard deadline. On most machines this takes < 1 second.
- const int maxSeconds = 15;
- while (sw.Elapsed < TimeSpan.FromSeconds(maxSeconds))
+ // Bounded iteration count keeps mmap churn well below the
+ // Windows kernel-resource threshold while still exercising
+ // the capacity race many times over.
+ const int iterations = 500;
+ for (int i = 0; i < iterations; i++)
{
using (var _ = mmapDir.OpenInput(name, NewIOContext(Random)))
{
// Just open and dispose; the race occurs during construction.
}
- iterations++;
+ }
+ }
+ finally
+ {
+ stop.Set();
+ writer.Join();
+ }
+
+ if (writerError != null)
+ {
+ throw new Exception("Writer thread failed", writerError);
+ }
+ }
+
+ // Regression test for issue #1013: sporadic AccessViolationException
+ // during concurrent search with SearcherManager on MMapDirectory.
+ //
+ // Strategy: spin many reader threads cloning + reading a shared
+ // IndexInput while another thread disposes it mid-flight. The
+ // invariant under test: concurrent Clone/read against a Dispose
+ // must only ever surface AlreadyClosed-style exceptions — never an
+ // AVE (which crashes the test host), never an NRE, never an IOE
+ // from a half-torn-down mapping.
+ //
+ // Under the chunked, reclaimer-backed design, clones observe the closed
+ // mapping (a chunk crossing after close throws) and throw AlreadyClosed
+ // promptly after the root is disposed. A successful pass here is
+ // therefore a *positive* result — not Inconclusive — because the
+ // expected behavior is that the invariant holds throughout.
+ // [Nightly]: wall-clock stress loop (up to ~30s). Kept out of the
+ // default run so CI isn't lengthened, but exercised in nightly runs
+ // where catching regressions in the #1013 race path is worth the time.
+ [Test, LuceneNetSpecific, Slow, Nightly, NonParallelizable]
+ public void TestConcurrentCloneReadVsDispose_Issue1013()
+ {
+ var dirPath = CreateTempDir("testIssue1013");
+ using var mmapDir = new MMapDirectory(dirPath);
+
+ const string name = "bytes";
+ const int fileSize = 1 << 20; // 1 MiB
+ var random = Random;
+
+ using (var io = mmapDir.CreateOutput(name, NewIOContext(random)))
+ {
+ var buf = new byte[4096];
+ random.NextBytes(buf);
+ for (int written = 0; written < fileSize; written += buf.Length)
+ {
+ io.WriteBytes(buf, 0, buf.Length);
+ }
+ }
+
+ const int readerThreads = 8;
+ const int maxSeconds = 30;
+
+ var sw = Stopwatch.StartNew();
+ int iteration = 0;
+ int raceObserved = 0;
+ var unexpectedExceptions = new ConcurrentBag