Skip to content

Commit 13398f9

Browse files
paulirwinclaude
andcommitted
Address PR feedback: nest MMapIndexInput, share MemoryMappedFile across slices
- Move MemoryMappedViewAccessorIndexInput into MMapDirectory.cs as the internal nested MMapIndexInput class, per the repo convention of not adding new files in Lucene-derived directories. - Replace the slicer's manual dispose loop with IOUtils.DisposeWhileHandlingException. - Share one MemoryMappedFile + chunk array per file per MMapDirectory instance via a refcounted SharedMapping tracked in a ConcurrentDictionary<string, Lazy<SharedMapping>>. OpenInput, CreateSlicer, its slices, and clones all piggyback on the same underlying mapping; the mapping is torn down when the last referrer disposes. - Use bufferSize: 1 + RandomAccess | Asynchronous on all FileStream creations since MemoryMappedFile bypasses the FileStream buffer. - Add a FEATURE_CONCURRENTDICTIONARY_TRYREMOVE_KEYVALUEPAIR gate and a polyfill in Support/ConcurrentDictionaryExtensions.cs for target frameworks predating .NET 5, where the TryRemove(KeyValuePair) overload is unavailable. - Add TestSharedMappingLifecycle covering the end-to-end lifecycle of the shared mapping across OpenInput, slicer, slices, and clones. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 681991d commit 13398f9

5 files changed

Lines changed: 793 additions & 499 deletions

File tree

Directory.Build.targets

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
<!-- Features in .NET 5.x+ only -->
6666
<PropertyGroup Condition=" $(TargetFramework.StartsWith('net5.')) Or $(TargetFramework.StartsWith('net6.')) Or $(TargetFramework.StartsWith('net7.')) Or $(TargetFramework.StartsWith('net8.')) Or $(TargetFramework.StartsWith('net9.')) Or $(TargetFramework.StartsWith('net10.')) ">
6767

68+
<DefineConstants>$(DefineConstants);FEATURE_CONCURRENTDICTIONARY_TRYREMOVE_KEYVALUEPAIR</DefineConstants>
6869
<DefineConstants>$(DefineConstants);FEATURE_MEMORYMARSHAL_GETARRAYDATAREFERENCE</DefineConstants>
6970
<DefineConstants>$(DefineConstants);FEATURE_READONLYSET</DefineConstants>
7071

src/Lucene.Net.Tests/Store/TestMultiMMap.cs

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,79 @@ public virtual void TestCloneSliceClose()
229229
mmapDir.Dispose();
230230
}
231231

232+
// LUCENENET specific: exercises the shared MemoryMappedFile refactor
233+
// where OpenInput, CreateSlicer, its slices, and clones all piggyback
234+
// on a single MemoryMappedFile per file (per directory instance).
235+
// Verifies that (a) concurrent IndexInputs all see correct bytes,
236+
// (b) disposing in arbitrary order keeps siblings functional, and
237+
// (c) once the last referrer is disposed the OS handle is released
238+
// (on Windows a still-open mapping would prevent the file delete).
239+
[Test]
240+
public virtual void TestSharedMappingLifecycle()
241+
{
242+
var tempDir = CreateTempDir("testSharedMappingLifecycle");
243+
MMapDirectory mmapDir = new MMapDirectory(tempDir);
244+
const string name = "bytes";
245+
using (IndexOutput io = mmapDir.CreateOutput(name, NewIOContext(Random)))
246+
{
247+
// 4 ints at offsets 0, 4, 8, 12 — each slice reads a known value.
248+
io.WriteInt32(10);
249+
io.WriteInt32(20);
250+
io.WriteInt32(30);
251+
io.WriteInt32(40);
252+
}
253+
254+
// Open several IndexInputs for the same file through both
255+
// OpenInput and CreateSlicer. All should share one mapping.
256+
IndexInput root = mmapDir.OpenInput(name, IOContext.DEFAULT);
257+
IndexInput rootClone = (IndexInput)root.Clone();
258+
259+
IndexInputSlicer slicer = mmapDir.CreateSlicer(name, NewIOContext(Random));
260+
IndexInput sliceA = slicer.OpenSlice("a", 0, 4);
261+
IndexInput sliceB = slicer.OpenSlice("b", 8, 4);
262+
IndexInput sliceAClone = (IndexInput)sliceA.Clone();
263+
264+
// Reads across all instances must be independent and correct.
265+
Assert.AreEqual(10, root.ReadInt32());
266+
Assert.AreEqual(10, rootClone.ReadInt32());
267+
Assert.AreEqual(10, sliceA.ReadInt32());
268+
Assert.AreEqual(30, sliceB.ReadInt32());
269+
Assert.AreEqual(10, sliceAClone.ReadInt32());
270+
271+
// Dispose a clone first; the root and siblings must keep working.
272+
rootClone.Dispose();
273+
root.Seek(4);
274+
Assert.AreEqual(20, root.ReadInt32());
275+
sliceB.Seek(0);
276+
Assert.AreEqual(30, sliceB.ReadInt32());
277+
278+
// Dispose a slice; its siblings from the same slicer must keep working.
279+
sliceAClone.Dispose();
280+
sliceA.Seek(0);
281+
Assert.AreEqual(10, sliceA.ReadInt32());
282+
283+
// Dispose the remaining slice-side instances. The root IndexInput
284+
// still holds a refcount, so the underlying mapping must stay alive.
285+
sliceA.Dispose();
286+
sliceB.Dispose();
287+
slicer.Dispose();
288+
289+
root.Seek(12);
290+
Assert.AreEqual(40, root.ReadInt32());
291+
292+
// Final referrer — this should bring the refcount to zero, tear
293+
// down the MemoryMappedFile, and remove the dictionary entry.
294+
root.Dispose();
295+
296+
// If any OS file handle is still open, this delete will fail on
297+
// Windows. On Unix it silently unlinks but the test still proves
298+
// the read-phase invariants above.
299+
mmapDir.DeleteFile(name);
300+
Assert.IsFalse(File.Exists(Path.Combine(tempDir.FullName, name)));
301+
302+
mmapDir.Dispose();
303+
}
304+
232305
[Test]
233306
public virtual void TestSeekZero()
234307
{

0 commit comments

Comments
 (0)