diff --git a/src/Lucene.Net.Tests/Store/TestMultiMMap.cs b/src/Lucene.Net.Tests/Store/TestMultiMMap.cs index db0934f9fd..8a2fbc7e66 100644 --- a/src/Lucene.Net.Tests/Store/TestMultiMMap.cs +++ b/src/Lucene.Net.Tests/Store/TestMultiMMap.cs @@ -3,8 +3,10 @@ using Lucene.Net.Index.Extensions; using NUnit.Framework; using System; +using System.Diagnostics; using System.IO; using System.Text; +using System.Threading; using Assert = Lucene.Net.TestFramework.Assert; namespace Lucene.Net.Store @@ -415,6 +417,118 @@ private void AssertChunking(Random random, int chunkSize) } + // LUCENENET: Regression test for GitHub #1090. A background thread + // extends a file on disk while the foreground thread repeatedly + // opens it with MMapDirectory.OpenInput. Before the fix, the + // file's length could grow between the caller capturing fc.Length + // and MemoryMappedFile.CreateFromFile performing its internal + // stat, causing ArgumentOutOfRangeException (paramName="capacity") + // with the message "The capacity may not be smaller than the + // file size." + // NonParallelizable: the retry-path assertion reads static counters on + // MMapDirectory, so any other test exercising MMapDirectory in parallel + // could skew the observed retry count. + [Test, LuceneNetSpecific, Slow, NonParallelizable] + public void TestOpenInputConcurrentFileExtension_Issue1090() + { + var dir = CreateTempDir("testOpenInputConcurrentFileExtension"); + const string name = "data.bin"; + string filePath = Path.Combine(dir.FullName, name); + + // Seed with a small initial payload. + File.WriteAllBytes(filePath, new byte[64]); + + using var mmapDir = new MMapDirectory(dir); + + const long maxFileSize = 1L * 1024 * 1024; // 1 MiB cap + var stop = new ManualResetEventSlim(false); + Exception writerError = null; + + var writer = new Thread(() => + { + var chunk = new byte[64]; + try + { + while (!stop.IsSet) + { + using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Write, FileShare.ReadWrite); + fs.Seek(0, SeekOrigin.End); + if (fs.Length < maxFileSize) + { + fs.Write(chunk, 0, chunk.Length); + } + else + { + // Keep the file bounded: truncate back and grow again. + fs.SetLength(64); + } + } + } + catch (Exception e) + { + writerError = e; + } + }) + { IsBackground = true, Name = "mmap-issue1090-extender" }; + writer.Start(); + + // Snapshot counters so this test's assertion is not affected by + // any earlier test's activity on MMapDirectory. + long baselineRetries = Interlocked.Read(ref MMapDirectory.s_capacityRetryCount); + + try + { + var sw = Stopwatch.StartNew(); + int iterations = 0; + // Keep stretching the window until either the race fires or we + // hit a hard deadline. On most machines this takes < 1 second. + const int maxSeconds = 15; + while (sw.Elapsed < TimeSpan.FromSeconds(maxSeconds)) + { + using (var _ = mmapDir.OpenInput(name, NewIOContext(Random))) + { + // Just open and dispose; the race occurs during construction. + } + iterations++; + + if (Interlocked.Read(ref MMapDirectory.s_capacityRetryCount) > baselineRetries) + { + break; // race reproduced and handled by the retry loop + } + } + + long retries = Interlocked.Read(ref MMapDirectory.s_capacityRetryCount) - baselineRetries; + int maxAttempts = Volatile.Read(ref MMapDirectory.s_maxCapacityAttemptsObserved); + + // Surface what was observed for diagnostics when run with -v normal. + TestContext.Progress.WriteLine( + $"TestOpenInputConcurrentFileExtension: iterations={iterations}, retries={retries}, maxAttemptsObserved={maxAttempts}"); + + // The real check: the race must have fired and our retry loop + // must have swallowed it. Without the fix, the exception + // escapes OpenInput and the test fails with ArgumentOutOfRangeException + // (as seen in #1090). If the race never fires during this run + // (timing-dependent), mark the test inconclusive rather than + // silently passing — we haven't actually exercised the fix. + if (retries == 0) + { + NUnit.Framework.Assert.Inconclusive( + $"The concurrent-extension race was not reproduced within {maxSeconds}s " + + $"({iterations} OpenInput iterations). The fix was therefore not exercised on this run."); + } + } + finally + { + stop.Set(); + writer.Join(); + } + + if (writerError != null) + { + throw new Exception("Writer thread failed", writerError); + } + } + [Test, LuceneNetSpecific] public void TestDisposeIndexInput() { diff --git a/src/Lucene.Net/Store/MMapDirectory.cs b/src/Lucene.Net/Store/MMapDirectory.cs index d8e92c7713..0309a5d845 100644 --- a/src/Lucene.Net/Store/MMapDirectory.cs +++ b/src/Lucene.Net/Store/MMapDirectory.cs @@ -63,6 +63,15 @@ public class MMapDirectory : FSDirectory private readonly int chunkSizePower; + // LUCENENET specific BEGIN: test-only counters for the capacity-retry + // path in Map() — see #1090. Internal (exposed via InternalsVisibleTo + // to the test assemblies) so regression tests can assert that the + // race was actually exercised during a run, and to gather data on how + // many retries are typically needed. Not intended for production use. + internal static long s_capacityRetryCount; + internal static int s_maxCapacityAttemptsObserved; + // LUCENENET specific END + /// /// Create a new for the named location. /// @@ -311,16 +320,52 @@ internal virtual ByteBuffer[] Map(MMapIndexInput input, FileStream fc, long offs if (input.memoryMappedFile is null) { - input.memoryMappedFile = MemoryMappedFile.CreateFromFile( - fileStream: fc, - mapName: null, - capacity: length, - access: MemoryMappedFileAccess.Read, + // LUCENENET specific BEGIN: MemoryMappedFile.CreateFromFile + // performs an internal stat and throws + // ArgumentOutOfRangeException("capacity") if the on-disk file + // size exceeds the requested capacity. When another + // process/thread is appending to this file (e.g. an + // IndexWriter that still holds a write handle), the file can + // grow between when we capture fc.Length and when + // CreateFromFile reads the size. Retry with the latest + // observed length on that specific failure; we only map the + // bytes the caller requested via the buffer-sizing loop + // below, so an oversized capacity is harmless. See #1090. + long capacity = Math.Max(length, fc.Length); + const int maxAttempts = 5; + int attempt = 0; + while (true) + { + try + { + input.memoryMappedFile = MemoryMappedFile.CreateFromFile( + fileStream: fc, + mapName: null, + capacity: capacity, + access: MemoryMappedFileAccess.Read, #if FEATURE_MEMORYMAPPEDFILESECURITY - memoryMappedFileSecurity: null, + memoryMappedFileSecurity: null, #endif - inheritability: HandleInheritability.Inheritable, - leaveOpen: true); // LUCENENET: We explicitly dispose the FileStream separately. + inheritability: HandleInheritability.Inheritable, + leaveOpen: true); // LUCENENET: We explicitly dispose the FileStream separately. + break; + } + catch (ArgumentOutOfRangeException e) when (e.ParamName == "capacity" && attempt < maxAttempts - 1) + { + Interlocked.Increment(ref s_capacityRetryCount); + capacity = Math.Max(capacity, fc.Length); + attempt++; + } + } + // Record the highest total attempts observed (1 = first try succeeded). + int attemptsTaken = attempt + 1; + int prior; + do + { + prior = Volatile.Read(ref s_maxCapacityAttemptsObserved); + if (attemptsTaken <= prior) break; + } while (Interlocked.CompareExchange(ref s_maxCapacityAttemptsObserved, attemptsTaken, prior) != prior); + // LUCENENET specific END } long bufferStart = 0L;