Fix concurrent-extension race in MMapDirectory.Map, #1090 (#1263)

* Fix concurrent-extension race in MMapDirectory.Map, #1090

When a file is being appended to concurrently (e.g. by an IndexWriter
that still holds a write handle), MemoryMappedFile.CreateFromFile can
throw ArgumentOutOfRangeException("capacity") because its internal stat
observes a file size greater than the capacity we computed from
fc.Length moments earlier. Take the max of the caller-supplied length
and a fresh fc.Length read as the capacity, and retry with an updated
length on that specific failure. Adds test-only counters and a
stress-based regression test in TestMultiMMap.

* Address Copilot review on #1090 MMapDirectory.Map fix

Narrow the ArgumentOutOfRangeException retry filter to
ParamName == "capacity" so unrelated argument errors aren't masked,
and mark the regression test NonParallelizable since it relies on
static counters on MMapDirectory.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: paulirwin

src/Lucene.Net.Tests/Store/TestMultiMMap.cs

@@ -3,8 +3,10 @@
 using Lucene.Net.Index.Extensions;
 using NUnit.Framework;
 using System;
+using System.Diagnostics;
 using System.IO;
 using System.Text;
+using System.Threading;
 using Assert = Lucene.Net.TestFramework.Assert;
 
 namespace Lucene.Net.Store
@@ -415,6 +417,118 @@ private void AssertChunking(Random random, int chunkSize)
         }
 
 
+        // LUCENENET: Regression test for GitHub #1090. A background thread
+        // extends a file on disk while the foreground thread repeatedly
+        // opens it with MMapDirectory.OpenInput. Before the fix, the
+        // file's length could grow between the caller capturing fc.Length
+        // and MemoryMappedFile.CreateFromFile performing its internal
+        // stat, causing ArgumentOutOfRangeException (paramName="capacity")
+        // with the message "The capacity may not be smaller than the
+        // file size."
+        // NonParallelizable: the retry-path assertion reads static counters on
+        // MMapDirectory, so any other test exercising MMapDirectory in parallel
+        // could skew the observed retry count.
+        [Test, LuceneNetSpecific, Slow, NonParallelizable]
+        public void TestOpenInputConcurrentFileExtension_Issue1090()
+        {
+            var dir = CreateTempDir("testOpenInputConcurrentFileExtension");
+            const string name = "data.bin";
+            string filePath = Path.Combine(dir.FullName, name);
+
+            // Seed with a small initial payload.
+            File.WriteAllBytes(filePath, new byte[64]);
+
+            using var mmapDir = new MMapDirectory(dir);
+
+            const long maxFileSize = 1L * 1024 * 1024; // 1 MiB cap
+            var stop = new ManualResetEventSlim(false);
+            Exception writerError = null;
+
+            var writer = new Thread(() =>
+            {
+                var chunk = new byte[64];
+                try
+                {
+                    while (!stop.IsSet)
+                    {
+                        using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Write, FileShare.ReadWrite);
+                        fs.Seek(0, SeekOrigin.End);
+                        if (fs.Length < maxFileSize)
+                        {
+                            fs.Write(chunk, 0, chunk.Length);
+                        }
+                        else
+                        {
+                            // Keep the file bounded: truncate back and grow again.
+                            fs.SetLength(64);
+                        }
+                    }
+                }
+                catch (Exception e)
+                {
+                    writerError = e;
+                }
+            })
+            { IsBackground = true, Name = "mmap-issue1090-extender" };
+            writer.Start();
+
+            // Snapshot counters so this test's assertion is not affected by
+            // any earlier test's activity on MMapDirectory.
+            long baselineRetries = Interlocked.Read(ref MMapDirectory.s_capacityRetryCount);
+
+            try
+            {
+                var sw = Stopwatch.StartNew();
+                int iterations = 0;
+                // Keep stretching the window until either the race fires or we
+                // hit a hard deadline. On most machines this takes < 1 second.
+                const int maxSeconds = 15;
+                while (sw.Elapsed < TimeSpan.FromSeconds(maxSeconds))
+                {
+                    using (var _ = mmapDir.OpenInput(name, NewIOContext(Random)))
+                    {
+                        // Just open and dispose; the race occurs during construction.
+                    }
+                    iterations++;
+
+                    if (Interlocked.Read(ref MMapDirectory.s_capacityRetryCount) > baselineRetries)
+                    {
+                        break; // race reproduced and handled by the retry loop
+                    }
+                }
+
+                long retries = Interlocked.Read(ref MMapDirectory.s_capacityRetryCount) - baselineRetries;
+                int maxAttempts = Volatile.Read(ref MMapDirectory.s_maxCapacityAttemptsObserved);
+
+                // Surface what was observed for diagnostics when run with -v normal.
+                TestContext.Progress.WriteLine(
+                    $"TestOpenInputConcurrentFileExtension: iterations={iterations}, retries={retries}, maxAttemptsObserved={maxAttempts}");
+
+                // The real check: the race must have fired and our retry loop
+                // must have swallowed it. Without the fix, the exception
+                // escapes OpenInput and the test fails with ArgumentOutOfRangeException
+                // (as seen in #1090). If the race never fires during this run
+                // (timing-dependent), mark the test inconclusive rather than
+                // silently passing — we haven't actually exercised the fix.
+                if (retries == 0)
+                {
+                    NUnit.Framework.Assert.Inconclusive(
+                        $"The concurrent-extension race was not reproduced within {maxSeconds}s " +
+                        $"({iterations} OpenInput iterations). The fix was therefore not exercised on this run.");
+                }
+            }
+            finally
+            {
+                stop.Set();
+                writer.Join();
+            }
+
+            if (writerError != null)
+            {
+                throw new Exception("Writer thread failed", writerError);
+            }
+        }
+
         [Test, LuceneNetSpecific]
         public void TestDisposeIndexInput()
         {

src/Lucene.Net/Store/MMapDirectory.cs

@@ -63,6 +63,15 @@ public class MMapDirectory : FSDirectory
 
         private readonly int chunkSizePower;
 
+        // LUCENENET specific BEGIN: test-only counters for the capacity-retry
+        // path in Map() — see #1090. Internal (exposed via InternalsVisibleTo
+        // to the test assemblies) so regression tests can assert that the
+        // race was actually exercised during a run, and to gather data on how
+        // many retries are typically needed. Not intended for production use.
+        internal static long s_capacityRetryCount;
+        internal static int s_maxCapacityAttemptsObserved;
+        // LUCENENET specific END
+
         /// <summary>
         /// Create a new <see cref="MMapDirectory"/> for the named location.
         /// </summary>
@@ -311,16 +320,52 @@ internal virtual ByteBuffer[] Map(MMapIndexInput input, FileStream fc, long offs
 
             if (input.memoryMappedFile is null)
             {
-                input.memoryMappedFile = MemoryMappedFile.CreateFromFile(
-                    fileStream: fc,
-                    mapName: null,
-                    capacity: length,
-                    access: MemoryMappedFileAccess.Read,
+                // LUCENENET specific BEGIN: MemoryMappedFile.CreateFromFile
+                // performs an internal stat and throws
+                // ArgumentOutOfRangeException("capacity") if the on-disk file
+                // size exceeds the requested capacity. When another
+                // process/thread is appending to this file (e.g. an
+                // IndexWriter that still holds a write handle), the file can
+                // grow between when we capture fc.Length and when
+                // CreateFromFile reads the size. Retry with the latest
+                // observed length on that specific failure; we only map the
+                // bytes the caller requested via the buffer-sizing loop
+                // below, so an oversized capacity is harmless. See #1090.
+                long capacity = Math.Max(length, fc.Length);
+                const int maxAttempts = 5;
+                int attempt = 0;
+                while (true)
+                {
+                    try
+                    {
+                        input.memoryMappedFile = MemoryMappedFile.CreateFromFile(
+                            fileStream: fc,
+                            mapName: null,
+                            capacity: capacity,
+                            access: MemoryMappedFileAccess.Read,
 #if FEATURE_MEMORYMAPPEDFILESECURITY
-                    memoryMappedFileSecurity: null,
+                            memoryMappedFileSecurity: null,
 #endif
-                    inheritability: HandleInheritability.Inheritable,
-                    leaveOpen: true); // LUCENENET: We explicitly dispose the FileStream separately.
+                            inheritability: HandleInheritability.Inheritable,
+                            leaveOpen: true); // LUCENENET: We explicitly dispose the FileStream separately.
+                        break;
+                    }
+                    catch (ArgumentOutOfRangeException e) when (e.ParamName == "capacity" && attempt < maxAttempts - 1)
+                    {
+                        Interlocked.Increment(ref s_capacityRetryCount);
+                        capacity = Math.Max(capacity, fc.Length);
+                        attempt++;
+                    }
+                }
+                // Record the highest total attempts observed (1 = first try succeeded).
+                int attemptsTaken = attempt + 1;
+                int prior;
+                do
+                {
+                    prior = Volatile.Read(ref s_maxCapacityAttemptsObserved);
+                    if (attemptsTaken <= prior) break;
+                } while (Interlocked.CompareExchange(ref s_maxCapacityAttemptsObserved, attemptsTaken, prior) != prior);
+                // LUCENENET specific END
             }
 
             long bufferStart = 0L;