diff --git a/src/Lucene.Net.Misc/Lucene.Net.Misc.csproj b/src/Lucene.Net.Misc/Lucene.Net.Misc.csproj index 688a05ae46..41794edb7c 100644 --- a/src/Lucene.Net.Misc/Lucene.Net.Misc.csproj +++ b/src/Lucene.Net.Misc/Lucene.Net.Misc.csproj @@ -36,14 +36,9 @@ $(PackageTags);miscellaneous bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml $(NoWarn);1591;1573 + true - - - - - - diff --git a/src/Lucene.Net.Misc/Store/NativePosixUtil.cs b/src/Lucene.Net.Misc/Store/NativePosixUtil.cs index a1d0a8ed86..272f6ed808 100644 --- a/src/Lucene.Net.Misc/Store/NativePosixUtil.cs +++ b/src/Lucene.Net.Misc/Store/NativePosixUtil.cs @@ -1,8 +1,15 @@ +using Lucene.Net.Util; +using Microsoft.Win32.SafeHandles; using System; +using System.ComponentModel; +using System.IO; +using System.Runtime.InteropServices; +#if FEATURE_SUPPORTEDOSPLATFORMATTRIBUTE +using System.Runtime.Versioning; +#endif -namespace org.apache.lucene.store +namespace Lucene.Net.Store { - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -20,45 +27,227 @@ namespace org.apache.lucene.store * limitations under the License. */ - ignore - /// - /// Provides JNI access to native methods such as madvise() for - /// + /// Provides access to native POSIX methods such as madvise() for + /// . + /// + /// LUCENENET specific: the original Lucene implementation called these through a JNI + /// shim compiled from NativePosixUtil.cpp. This port replaces that native build + /// step with direct P/Invoke into libc, so it can only be used on Unix-like + /// platforms (Linux and macOS); it is not supported on Microsoft Windows. /// - public final class NativePosixUtil +#if FEATURE_SUPPORTEDOSPLATFORMATTRIBUTE + [UnsupportedOSPlatform("windows")] +#endif + public static class NativePosixUtil { - public final static int NORMAL = 0; - public final static int SEQUENTIAL = 1; - public final static int RANDOM = 2; - public final static int WILLNEED = 3; - public final static int DONTNEED = 4; - public final static int NOREUSE = 5; - -//JAVA TO C# CONVERTER NOTE: This static initializer block is converted to a static constructor, but there is no current class: - static ImpliedClass() - { -//JAVA TO C# CONVERTER TODO TASK: The library is specified in the 'DllImport' attribute for .NET: -// System.loadLibrary("NativePosixUtil"); - } - - private static native int posix_fadvise(FileDescriptor fd, long offset, long len, int advise) throws IOException; - public static native int posix_madvise(ByteBuffer buf, int advise) throws IOException; - public static native int madvise(ByteBuffer buf, int advise) throws IOException; - public static native FileDescriptor open_direct(string filename, bool read) throws IOException; - public static native long pread(FileDescriptor fd, long pos, ByteBuffer byteBuf) throws IOException; - - public static void advise(FileDescriptor fd, long offset, long len, int advise) throws IOException - { -//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': -//ORIGINAL LINE: final int code = posix_fadvise(fd, offset, len, advise); - int code = posix_fadvise(fd, offset, len, advise); - if (code != 0) + // These constants mirror the Java NativePosixUtil ordering. Note this is NOT the same + // ordering as the OS POSIX_FADV_*/POSIX_MADV_* values (SEQUENTIAL/RANDOM are swapped); + // MapAdvice() translates to the OS values. + public const int NORMAL = 0; + public const int SEQUENTIAL = 1; + public const int RANDOM = 2; + public const int WILLNEED = 3; + public const int DONTNEED = 4; + public const int NOREUSE = 5; + + /// + /// Opens a file for direct (un-cached) I/O. + /// + /// On Linux this uses O_DIRECT | O_NOATIME; on macOS it opens normally and then + /// applies fcntl(F_NOCACHE), matching the original native implementation. + /// + /// the file to open + /// true to open read-only; false to open read-write (creating if needed) + /// a wrapping the open file descriptor + /// If the file could not be opened + /// If running on Microsoft Windows + public static SafeFileHandle OpenDirect(string filename, bool read) { - throw new Exception("posix_fadvise failed code=" + code); + EnsureUnix(); + + // Upstream's C++ called open(fname, O_RDWR | O_CREAT | DIRECT_FLAG, 0666) for the write path. + // We cannot create-with-mode through P/Invoke: open() is variadic (int open(const char*, int, ...)) + // and the mode argument rides the varargs ABI. On platforms where varargs are not passed in the + // same registers as fixed args (notably macOS, including Apple Silicon arm64), a fixed-signature + // P/Invoke delivers garbage for mode, producing a file without owner-read permission, so the + // subsequent O_RDONLY open of the same file fails with EACCES. To stay correct and ABI-agnostic + // we never pass mode: the file is created up front by the BCL (which applies the normal mode and + // umask), and we then open it with the plain two-argument open() - no O_CREAT, no varargs. + if (!read) + { + // mirrors O_CREAT: ensure the file exists with a sane mode before opening it + using (System.IO.File.Open(filename, System.IO.FileMode.OpenOrCreate, System.IO.FileAccess.ReadWrite)) { } + } + + int fd; + if (Constants.MAC_OS_X) + { + fd = read + ? NativeMethods.open(filename, NativeMethods.O_RDONLY) + : NativeMethods.open(filename, NativeMethods.O_RDWR); + if (fd < 0) + { + throw NewIOException("open", filename); + } + + // macOS has no O_DIRECT; disable the page cache for this descriptor instead. + if (NativeMethods.fcntl(fd, NativeMethods.F_NOCACHE, 1) < 0) + { + int err = Marshal.GetLastWin32Error(); + NativeMethods.close(fd); + throw NewIOException("fcntl(F_NOCACHE)", filename, err); + } + } + else // Linux (and other Unixes, best-effort) + { + fd = read + ? NativeMethods.open(filename, NativeMethods.O_RDONLY | NativeMethods.O_DIRECT | NativeMethods.O_NOATIME) + : NativeMethods.open(filename, NativeMethods.O_RDWR | NativeMethods.O_DIRECT | NativeMethods.O_NOATIME); + if (fd < 0) + { + throw NewIOException("open", filename); + } + } + + return new SafeFileHandle((IntPtr)fd, ownsHandle: true); } - } - } + /// + /// Positioned read of up to bytes from at + /// absolute offset into the native buffer at , + /// without changing the file's current offset. + /// + /// the number of bytes read (may be less than near EOF) + /// If the read failed + public static long Pread(SafeFileHandle fd, long pos, IntPtr buffer, int length) + { + EnsureUnix(); + long n = (long)NativeMethods.pread(Fd(fd), buffer, (nuint)length, pos); + if (n < 0) + { + throw NewIOException("pread", null); + } + return n; + } + + /// + /// Issues a posix_madvise() hint over the native buffer at . + public static int PosixMAdvise(IntPtr buffer, int length, int advise) + { + EnsureUnix(); + return NativeMethods.posix_madvise(buffer, (nuint)length, MapAdvice(advise)); + } + + /// + /// Issues a madvise() hint over the native buffer at . + public static int MAdvise(IntPtr buffer, int length, int advise) + { + EnsureUnix(); + return NativeMethods.madvise(buffer, (nuint)length, MapAdvice(advise)); + } + + /// + /// Issues a posix_fadvise() hint for the given range of , + /// throwing if the call reports a non-zero error code. + /// + /// If posix_fadvise returned a non-zero code + public static void Advise(SafeFileHandle fd, long offset, long len, int advise) + { + EnsureUnix(); + int code = NativeMethods.posix_fadvise(Fd(fd), offset, len, MapAdvice(advise)); + if (code != 0) + { + // LUCENENET: upstream throws RuntimeException; we use IOException as this is an I/O failure. + // posix_fadvise returns the error number directly (it does not set errno); surface it as + // the HResult to match the errno-as-HResult convention used elsewhere here. + throw new IOException("posix_fadvise failed code=" + code, code); + } + } + + /// + /// Translates the Java-style advice ordinal (see the public constants) to the OS + /// POSIX_FADV_*/POSIX_MADV_* value. Only SEQUENTIAL and RANDOM + /// differ in ordering between the two. + /// + private static int MapAdvice(int advise) + { + switch (advise) + { + case SEQUENTIAL: return 2; // POSIX_*_SEQUENTIAL + case RANDOM: return 1; // POSIX_*_RANDOM + default: return advise; // NORMAL=0, WILLNEED=3, DONTNEED=4, NOREUSE=5 are identical + } + } + + private static int Fd(SafeFileHandle handle) => (int)handle.DangerousGetHandle(); + + private static IOException NewIOException(string operation, string filename) + => NewIOException(operation, filename, Marshal.GetLastWin32Error()); + + private static IOException NewIOException(string operation, string filename, int errno) + { + string where = filename is null ? operation : $"{operation} {filename}"; + // On Unix with SetLastError, GetLastWin32Error() returns errno. Win32Exception's message + // is not meaningful on Unix, so include the raw errno for diagnosis. + // LUCENENET: surface the errno as the IOException's HResult. On Unix the BCL sets HResult to + // the raw errno (e.g. a FileStream share violation surfaces HResult == EWOULDBLOCK), and + // NativeFSLock inspects IOException.HResult, so we match that convention here. + return new IOException($"{where} failed (errno {errno})", errno); + } + + internal static void EnsureUnix() + { + if (Constants.WINDOWS) + { + throw new PlatformNotSupportedException( + $"{nameof(NativePosixUtil)} requires Linux or macOS direct I/O and is not supported on Microsoft Windows."); + } + } + + /// + /// P/Invoke declarations for the libc functions used here. These replace the + /// JNI/C++ native methods of the original implementation. + /// + private static class NativeMethods + { + private const string LIBC = "libc"; + // open() flags. O_DIRECT/O_NOATIME are Linux-only. We never pass O_CREAT/mode: see OpenDirect. + internal const int O_RDONLY = 0x0; + internal const int O_RDWR = 0x2; + + // O_DIRECT is architecture-dependent on Linux: most arches (x86/x86-64) use the asm-generic + // value 0x4000, but Arm/Arm64 (and a few others) swap it with O_DIRECTORY and use 0x10000. + // Using the wrong value silently means "O_DIRECTORY", which makes open() of a regular file + // fail with ENOTDIR. O_NOATIME (0x40000) is the same across these arches. + internal static readonly int O_DIRECT = + RuntimeInformation.OSArchitecture is Architecture.Arm or Architecture.Arm64 + ? 0x10000 + : 0x4000; + internal const int O_NOATIME = 0x40000; // Linux + internal const int F_NOCACHE = 48; // macOS + + [DllImport(LIBC, SetLastError = true, EntryPoint = "open")] + internal static extern int open([MarshalAs(UnmanagedType.LPStr)] string pathname, int flags); + + [DllImport(LIBC, SetLastError = true)] + internal static extern int close(int fd); + + [DllImport(LIBC, SetLastError = true)] + internal static extern int fcntl(int fd, int cmd, int arg); + + [DllImport(LIBC, SetLastError = true)] + internal static extern nint pread(int fd, IntPtr buf, nuint count, long offset); + + [DllImport(LIBC, SetLastError = true)] + internal static extern int posix_fadvise(int fd, long offset, long len, int advice); + + [DllImport(LIBC, SetLastError = true)] + internal static extern int posix_madvise(IntPtr addr, nuint length, int advice); + + [DllImport(LIBC, SetLastError = true)] + internal static extern int madvise(IntPtr addr, nuint length, int advice); + } + } } diff --git a/src/Lucene.Net.Misc/Store/NativeUnixDirectory.cs b/src/Lucene.Net.Misc/Store/NativeUnixDirectory.cs index 3cb38e6939..5d7c1f479a 100644 --- a/src/Lucene.Net.Misc/Store/NativeUnixDirectory.cs +++ b/src/Lucene.Net.Misc/Store/NativeUnixDirectory.cs @@ -1,32 +1,32 @@ +using Lucene.Net.Support; +using Lucene.Net.Util; +using Microsoft.Win32.SafeHandles; using System; -using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; +#if FEATURE_SUPPORTEDOSPLATFORMATTRIBUTE +using System.Runtime.Versioning; +#endif -namespace org.apache.lucene.store +namespace Lucene.Net.Store { - /* * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - ignore - import java.nio.channels.FileChannel; - - import org.apache.lucene.store.Directory; // javadoc - import org.apache.lucene.store.IOContext.Context; - // TODO // - newer Linux kernel versions (after 2.6.29) have // improved MADV_SEQUENTIAL (and hopefully also @@ -35,493 +35,606 @@ namespace org.apache.lucene.store // IO when context is merge /// - /// A implementation for all Unixes that uses + /// A implementation for all Unixes that uses /// DIRECT I/O to bypass OS level IO caching during /// merging. For all other cases (searching, writing) we delegate - /// to the provided Directory instance. + /// to the provided instance. /// - /// See Overview - /// for more details. + /// LUCENENET specific: the original Lucene implementation reached the OS through a JNI + /// shim (NativePosixUtil.cpp) plus Java NIO FileChannels. This port replaces the + /// native build step with direct P/Invoke into libc (see ), + /// so no native compilation is required, but it can only be used on Unix-like platforms + /// (Linux and macOS); it is not supported on Microsoft Windows. /// - /// - /// To use this you must compile - /// NativePosixUtil.cpp (exposes Linux-specific APIs through - /// JNI) for your platform, by running ant - /// build-native-unix, and then putting the resulting - /// libNativePosixUtil.so (from - /// lucene/build/native) onto your dynamic - /// linker search path. - /// - /// /// WARNING: this code is very new and quite easily /// could contain horrible bugs. For example, here's one - /// known issue: if you use seek in IndexOutput, and then + /// known issue: if you use seek in , and then /// write more than one buffer's worth of bytes, then the /// file will be wrong. Lucene does not do this today (only writes - /// small number of bytes after seek), but that may change. + /// small number of bytes after seek), but that may change. /// - /// - /// This directory passes Solr and Lucene tests on Linux - /// and OS X; other Unixes should work but have not been - /// tested! Use at your own risk. + /// Direct I/O requires that reads and writes are aligned to the device block size + /// (here 512 bytes) and that the underlying filesystem supports it; some filesystems + /// (e.g. tmpfs) and overlay/virtual mounts do not. /// /// @lucene.experimental - /// /// - public class NativeUnixDirectory extends FSDirectory +#if FEATURE_SUPPORTEDOSPLATFORMATTRIBUTE + [UnsupportedOSPlatform("windows")] +#endif + public class NativeUnixDirectory : FSDirectory { - - // TODO: this is OS dependent, but likely 512 is the LCD - private final static long ALIGN = 512; - private final static long ALIGN_NOT_MASK = ~(ALIGN - 1); - - /// - /// Default buffer size before writing to disk (256 KB); - /// larger means less IO load but more RAM and direct - /// buffer storage space consumed during merging. - /// - - public final static int DEFAULT_MERGE_BUFFER_SIZE = 262144; - - /// - /// Default min expected merge size before direct IO is - /// used (10 MB): - /// - public final static long DEFAULT_MIN_BYTES_DIRECT = 10 * 1024 * 1024; - - private final int mergeBufferSize; - private final long minBytesDirect; - private final Directory @delegate; - - /// - /// Create a new NIOFSDirectory for the named location. - /// - /// the path of the directory - /// Size of buffer to use for - /// merging. See . - /// Merges, or files to be opened for - /// reading, smaller than this will - /// not use direct IO. See {@link - /// #DEFAULT_MIN_BYTES_DIRECT} - /// fallback Directory for non-merges - /// If there is a low-level I/O error - public NativeUnixDirectory(File path, int mergeBufferSize, long minBytesDirect, Directory @delegate) throws IOException - { - base(path, @delegate.LockFactory); - if ((mergeBufferSize & ALIGN) != 0) + // TODO: this is OS dependent, but likely 512 is the LCD + private const long ALIGN = 512; + private const long ALIGN_NOT_MASK = ~(ALIGN - 1); + + /// + /// Default buffer size before writing to disk (256 KB); + /// larger means less IO load but more RAM and direct + /// buffer storage space consumed during merging. + /// + public const int DEFAULT_MERGE_BUFFER_SIZE = 262144; + + /// + /// Default min expected merge size before direct IO is + /// used (10 MB): + /// + public const long DEFAULT_MIN_BYTES_DIRECT = 10 * 1024 * 1024; + + private readonly int mergeBufferSize; + private readonly long minBytesDirect; + private readonly Directory m_delegate; + + /// + /// Create a new for the named location. + /// + /// the path of the directory + /// Size of buffer to use for + /// merging. See . + /// Merges, or files to be opened for + /// reading, smaller than this will + /// not use direct IO. See + /// fallback for non-merges + /// If there is a low-level I/O error + /// If running on Microsoft Windows + public NativeUnixDirectory(DirectoryInfo path, int mergeBufferSize, long minBytesDirect, Directory @delegate) + : base(path, @delegate.LockFactory) { - throw new System.ArgumentException("mergeBufferSize must be 0 mod " + ALIGN + " (got: " + mergeBufferSize + ")"); - } - this.mergeBufferSize = mergeBufferSize; - this.minBytesDirect = minBytesDirect; - this.@delegate = @delegate; - } - - /// - /// Create a new NIOFSDirectory for the named location. - /// - /// the path of the directory - /// fallback Directory for non-merges - /// If there is a low-level I/O error - public NativeUnixDirectory(File path, Directory @delegate) throws IOException - { - this(path, DEFAULT_MERGE_BUFFER_SIZE, DEFAULT_MIN_BYTES_DIRECT, @delegate); - } - - public IndexInput openInput(string name, IOContext context) throws IOException - { - ensureOpen(); - if (context.context != Context.MERGE || context.mergeInfo.estimatedMergeBytes < minBytesDirect || fileLength(name) < minBytesDirect) - { - return @delegate.openInput(name, context); + EnsureUnix(); + if ((mergeBufferSize & ALIGN) != 0) + { + throw new ArgumentException("mergeBufferSize must be 0 mod " + ALIGN + " (got: " + mergeBufferSize + ")"); + } + this.mergeBufferSize = mergeBufferSize; + this.minBytesDirect = minBytesDirect; + this.m_delegate = @delegate; } - else + + /// + /// Create a new for the named location. + /// + /// the path of the directory + /// fallback for non-merges + /// If there is a low-level I/O error + /// If running on Microsoft Windows + public NativeUnixDirectory(DirectoryInfo path, Directory @delegate) + : this(path, DEFAULT_MERGE_BUFFER_SIZE, DEFAULT_MIN_BYTES_DIRECT, @delegate) { - return new NativeUnixIndexInput(new File(Directory, name), mergeBufferSize); } - } - public IndexOutput createOutput(string name, IOContext context) throws IOException - { - ensureOpen(); - if (context.context != Context.MERGE || context.mergeInfo.estimatedMergeBytes < minBytesDirect) + /// + /// Create a new for the named location. + /// + /// LUCENENET specific overload for convenience using string instead of . + /// + public NativeUnixDirectory(string path, int mergeBufferSize, long minBytesDirect, Directory @delegate) + : this(new DirectoryInfo(path), mergeBufferSize, minBytesDirect, @delegate) { - return @delegate.createOutput(name, context); } - else + + /// + /// Create a new for the named location. + /// + /// LUCENENET specific overload for convenience using string instead of . + /// + public NativeUnixDirectory(string path, Directory @delegate) + : this(new DirectoryInfo(path), @delegate) { - ensureCanWrite(name); - return new NativeUnixIndexOutput(new File(Directory, name), mergeBufferSize); } - } - - private final static class NativeUnixIndexOutput extends IndexOutput - { - private final ByteBuffer buffer; - private final FileOutputStream fos; - private final FileChannel channel; - private final int bufferSize; - - //private final File path; - - private int bufferPos; - private long filePos; - private long fileLength; - private bool isOpen; - public NativeUnixIndexOutput(File path, int bufferSize) throws IOException + private static void EnsureUnix() { - //this.path = path; -//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': -//ORIGINAL LINE: final java.io.FileDescriptor fd = NativePosixUtil.open_direct(path.toString(), false); - FileDescriptor fd = NativePosixUtil.open_direct(path.ToString(), false); - fos = new FileOutputStream(fd); - //fos = new FileOutputStream(path); - channel = fos.Channel; - buffer = ByteBuffer.allocateDirect(bufferSize); - this.bufferSize = bufferSize; - isOpen = true; + if (Constants.WINDOWS) + { + throw new PlatformNotSupportedException( + $"{nameof(NativeUnixDirectory)} requires Linux or macOS direct I/O and is not supported on Microsoft Windows."); + } } - public void writeByte(sbyte b) throws IOException + public override IndexInput OpenInput(string name, IOContext context) { - Debug.Assert(bufferPos == buffer.position(), "bufferPos=" + bufferPos + " vs buffer.position()=" + buffer.position()); - buffer.put(b); - if (++bufferPos == bufferSize) - { - dump(); - } + EnsureOpen(); + if (context.Context != IOContext.UsageContext.MERGE || context.MergeInfo.EstimatedMergeBytes < minBytesDirect || FileLength(name) < minBytesDirect) + { + return m_delegate.OpenInput(name, context); + } + else + { + return new NativeUnixIndexInput(Path.Combine(Directory.FullName, name), mergeBufferSize); + } } - public void writeBytes(sbyte[] src, int offset, int len) throws IOException + public override IndexOutput CreateOutput(string name, IOContext context) { - int toWrite = len; - while (true) - { -//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': -//ORIGINAL LINE: final int left = bufferSize - bufferPos; - int left = bufferSize - bufferPos; - if (left <= toWrite) - { - buffer.put(src, offset, left); - toWrite -= left; - offset += left; - bufferPos = bufferSize; - dump(); + EnsureOpen(); + if (context.Context != IOContext.UsageContext.MERGE || context.MergeInfo.EstimatedMergeBytes < minBytesDirect) + { + return m_delegate.CreateOutput(name, context); } else { - buffer.put(src, offset, toWrite); - bufferPos += toWrite; - break; + EnsureCanWrite(name); + return new NativeUnixIndexOutput(Path.Combine(Directory.FullName, name), mergeBufferSize); } - } } - //@Override - //public void setLength() throws IOException { - // TODO -- how to impl this? neither FOS nor - // FileChannel provides an API? - //} + // ----- libc helpers for the operations the original code performed via FileChannel ----- - public void flush() - { - // TODO -- I don't think this method is necessary? - } + private static int Fd(SafeFileHandle handle) => (int)handle.DangerousGetHandle(); - private void dump() throws IOException + private static void PWrite(SafeFileHandle fd, IntPtr buffer, int length, long pos) { - buffer.flip(); -//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': -//ORIGINAL LINE: final long limit = filePos + buffer.limit(); - long limit = filePos + buffer.limit(); - if (limit > fileLength) - { - // this dump extends the file - fileLength = limit; - } - else - { - // we had seek'd back & wrote some changes - } - - // must always round to next block - buffer.limit((int)((buffer.limit() + ALIGN - 1) & ALIGN_NOT_MASK)); - - assert(buffer.limit() & ALIGN_NOT_MASK) == buffer.limit() : "limit=" + buffer.limit() + " vs " + (buffer.limit() & ALIGN_NOT_MASK); - assert(filePos & ALIGN_NOT_MASK) == filePos; - //System.out.println(Thread.currentThread().getName() + ": dump to " + filePos + " limit=" + buffer.limit() + " fos=" + fos); - channel.write(buffer, filePos); - filePos += bufferPos; - bufferPos = 0; - buffer.clear(); - //System.out.println("dump: done"); - - // TODO: the case where we'd seek'd back, wrote an - // entire buffer, we must here read the next buffer; - // likely Lucene won't trip on this since we only - // write smallish amounts on seeking back + long written = (long)NativeMethods.pwrite(Fd(fd), buffer, (nuint)length, pos); + if (written < 0) + { + int errno = Marshal.GetLastWin32Error(); + // LUCENENET: surface errno as the HResult (see NativePosixUtil.NewIOException) so callers + // such as NativeFSLock, which inspect IOException.HResult, still see the underlying error. + throw new IOException($"pwrite failed (errno {errno})", errno); + } } - public long FilePointer + private static void FTruncate(SafeFileHandle fd, long length) { - return filePos + bufferPos; + if (NativeMethods.ftruncate(Fd(fd), length) < 0) + { + int errno = Marshal.GetLastWin32Error(); + // LUCENENET: surface errno as the HResult (see PWrite). + throw new IOException($"ftruncate failed (errno {errno})", errno); + } } - // TODO: seek is fragile at best; it can only properly - // handle seek & then change bytes that fit entirely - // within one buffer - public void seek(long pos) throws IOException + private static long FileSize(SafeFileHandle fd) { - if (pos != FilePointer) - { - dump(); -//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': -//ORIGINAL LINE: final long alignedPos = pos & ALIGN_NOT_MASK; - long alignedPos = pos & ALIGN_NOT_MASK; - filePos = alignedPos; - int n = (int) NativePosixUtil.pread(fos.FD, filePos, buffer); - if (n < bufferSize) - { - buffer.limit(n); - } - //System.out.println("seek refill=" + n); -//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': -//ORIGINAL LINE: final int delta = (int)(pos - alignedPos); - int delta = (int)(pos - alignedPos); - buffer.position(delta); - bufferPos = delta; - } + // The original used FileChannel.size(). All reads here are positioned (pread), so + // moving the file offset with lseek(SEEK_END) to learn the size is harmless. + long size = NativeMethods.lseek(Fd(fd), 0, NativeMethods.SEEK_END); + if (size < 0) + { + int errno = Marshal.GetLastWin32Error(); + // LUCENENET: surface errno as the HResult (see PWrite). + throw new IOException($"lseek failed (errno {errno})", errno); + } + return size; } - public long length() + /// + /// A small native, block-aligned buffer that mimics the subset of Java's direct + /// ByteBuffer behavior used by the index input/output (position/limit, relative + /// get/put, flip/clear/rewind). Direct I/O requires the buffer to be aligned to the + /// device block size, which does not guarantee, so + /// we over-allocate and align manually. + /// + private sealed unsafe class AlignedByteBuffer : IDisposable { - return fileLength + bufferPos; - } + private IntPtr basePtr; + private readonly byte* alignedPtr; + private readonly int capacity; + private int position; + private int limit; - public long Checksum throws IOException - { - throw new System.NotSupportedException("this directory currently does not work at all!"); + public AlignedByteBuffer(int capacity, int alignment) + { + this.capacity = capacity; + basePtr = Marshal.AllocHGlobal(capacity + alignment - 1); + long addr = basePtr.ToInt64(); + long aligned = (addr + alignment - 1) & ~((long)alignment - 1); + alignedPtr = (byte*)aligned; + Clear(); + } + + ~AlignedByteBuffer() + { + // Clones are never disposed by Lucene, so free native memory as a backstop. + Free(); + } + + public int Capacity => capacity; + public int Position { get => position; set => position = value; } + public int Limit { get => limit; set => limit = value; } + public IntPtr Pointer => (IntPtr)alignedPtr; + + public void Clear() { position = 0; limit = capacity; } + public void Flip() { limit = position; position = 0; } + public void Rewind() { position = 0; } + + public void Put(byte b) => alignedPtr[position++] = b; + public byte Get() => alignedPtr[position++]; + + public void Put(ReadOnlySpan src) + { + src.CopyTo(new Span(alignedPtr + position, src.Length)); + position += src.Length; + } + + public void Get(Span dst) + { + new ReadOnlySpan(alignedPtr + position, dst.Length).CopyTo(dst); + position += dst.Length; + } + + private void Free() + { + if (basePtr != IntPtr.Zero) + { + Marshal.FreeHGlobal(basePtr); + basePtr = IntPtr.Zero; + } + } + + public void Dispose() + { + Free(); + GC.SuppressFinalize(this); + } } - public void close() throws IOException + private sealed class NativeUnixIndexOutput : IndexOutput { - if (isOpen) - { - isOpen = false; - try - { - dump(); - } - finally - { - try - { - //System.out.println("direct close set len=" + fileLength + " vs " + channel.size() + " path=" + path); - channel.truncate(fileLength); - //System.out.println(" now: " + channel.size()); - } - finally - { - try + private readonly AlignedByteBuffer buffer; + private readonly SafeFileHandle fd; + private readonly int bufferSize; + // LUCENENET specific: track a running CRC32 over the bytes written (in logical order), so + // Checksum can return it like FSIndexOutput does. Upstream left getChecksum() unsupported. + private readonly CRC32 crc = new CRC32(); + + private int bufferPos; + private long filePos; + private long fileLength; + private bool isOpen; + + public NativeUnixIndexOutput(string path, int bufferSize) + { + fd = NativePosixUtil.OpenDirect(path, read: false); + buffer = new AlignedByteBuffer(bufferSize, (int)ALIGN); + this.bufferSize = bufferSize; + isOpen = true; + } + + public override void WriteByte(byte b) + { + crc.Update(b); + buffer.Put(b); + if (++bufferPos == bufferSize) { - channel.close(); + Dump(); } - finally + } + + public override void WriteBytes(ReadOnlySpan source) + { + crc.Update(source); + int offset = 0; + int toWrite = source.Length; + while (true) { - fos.close(); - //System.out.println(" final len=" + path.length()); + int left = bufferSize - bufferPos; + if (left <= toWrite) + { + buffer.Put(source.Slice(offset, left)); + toWrite -= left; + offset += left; + bufferPos = bufferSize; + Dump(); + } + else + { + buffer.Put(source.Slice(offset, toWrite)); + bufferPos += toWrite; + break; + } } - } } - } - } - } - private final static class NativeUnixIndexInput extends IndexInput - { - private final ByteBuffer buffer; - private final FileInputStream fis; - private final FileChannel channel; - private final int bufferSize; + public override void Flush() + { + // TODO -- I don't think this method is necessary? + } - private bool isOpen; - private bool isClone; - private long filePos; - private int bufferPos; + private void Dump() + { + buffer.Flip(); + long limit = filePos + buffer.Limit; + if (limit > fileLength) + { + // this dump extends the file + fileLength = limit; + } + // else: we had seek'd back & wrote some changes - public NativeUnixIndexInput(File path, int bufferSize) throws IOException - { - base("NativeUnixIndexInput(path=\"" + path.Path + "\")"); -//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': -//ORIGINAL LINE: final java.io.FileDescriptor fd = NativePosixUtil.open_direct(path.toString(), true); - FileDescriptor fd = NativePosixUtil.open_direct(path.ToString(), true); - fis = new FileInputStream(fd); - channel = fis.Channel; - this.bufferSize = bufferSize; - buffer = ByteBuffer.allocateDirect(bufferSize); - isOpen = true; - isClone = false; - filePos = -bufferSize; - bufferPos = bufferSize; - //System.out.println("D open " + path + " this=" + this); - } + // must always round to next block + buffer.Limit = (int)((buffer.Limit + ALIGN - 1) & ALIGN_NOT_MASK); - // for clone - public NativeUnixIndexInput(NativeUnixIndexInput other) throws IOException - { - base(other.ToString()); - this.fis = null; - channel = other.channel; - this.bufferSize = other.bufferSize; - buffer = ByteBuffer.allocateDirect(bufferSize); - filePos = -bufferSize; - bufferPos = bufferSize; - isOpen = true; - isClone = true; - //System.out.println("D clone this=" + this); - seek(other.FilePointer); - } + PWrite(fd, buffer.Pointer, buffer.Limit, filePos); + filePos += bufferPos; + bufferPos = 0; + buffer.Clear(); - public void close() throws IOException - { - if (isOpen && !isClone) - { - try + // TODO: the case where we'd seek'd back, wrote an + // entire buffer, we must here read the next buffer; + // likely Lucene won't trip on this since we only + // write smallish amounts on seeking back + } + + public override long Position => filePos + bufferPos; + + // TODO: seek is fragile at best; it can only properly + // handle seek & then change bytes that fit entirely + // within one buffer + [Obsolete("(4.1) this method will be removed in Lucene 5.0")] + public override void Seek(long pos) { - channel.close(); + if (pos != Position) + { + Dump(); + long alignedPos = pos & ALIGN_NOT_MASK; + filePos = alignedPos; + int n = (int)NativePosixUtil.Pread(fd, filePos, buffer.Pointer, bufferSize); + if (n < bufferSize) + { + buffer.Limit = n; + } + int delta = (int)(pos - alignedPos); + buffer.Position = delta; + bufferPos = delta; + } } - finally + + public override long Length { - if (!isClone) - { - fis.close(); - } + get => fileLength + bufferPos; + set { /* not supported: length is managed internally */ } } - } - } - public long FilePointer - { - return filePos + bufferPos; + // LUCENENET specific: upstream left getChecksum() unsupported; we maintain a running CRC32 + // over the written bytes (like FSIndexOutput) so the checksum footer can be written. + public override long Checksum => crc.Value; + + protected override void Dispose(bool disposing) + { + if (isOpen) + { + isOpen = false; + try + { + Dump(); + } + finally + { + try + { + FTruncate(fd, fileLength); + } + finally + { + fd.Dispose(); + buffer.Dispose(); + } + } + } + } } - public void seek(long pos) throws IOException + private sealed class NativeUnixIndexInput : IndexInput { - if (pos != FilePointer) - { -//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': -//ORIGINAL LINE: final long alignedPos = pos & ALIGN_NOT_MASK; - long alignedPos = pos & ALIGN_NOT_MASK; - filePos = alignedPos - bufferSize; - -//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': -//ORIGINAL LINE: final int delta = (int)(pos - alignedPos); - int delta = (int)(pos - alignedPos); - if (delta != 0) - { - refill(); - buffer.position(delta); - bufferPos = delta; + private readonly AlignedByteBuffer buffer; + private readonly SafeFileHandle fd; // owned descriptor; null for clones + private readonly SafeFileHandle sharedFd; // descriptor used for reads (own, or the original's) + private readonly int bufferSize; + + private bool isOpen; + private readonly bool isClone; + private long filePos; + private int bufferPos; + + public NativeUnixIndexInput(string path, int bufferSize) + : base("NativeUnixIndexInput(path=\"" + path + "\")") + { + fd = NativePosixUtil.OpenDirect(path, read: true); + sharedFd = fd; + this.bufferSize = bufferSize; + buffer = new AlignedByteBuffer(bufferSize, (int)ALIGN); + isOpen = true; + isClone = false; + filePos = -bufferSize; + bufferPos = bufferSize; } - else + + // for clone + private NativeUnixIndexInput(NativeUnixIndexInput other) + : base(other.ToString()) { - // force refill on next read - bufferPos = bufferSize; + fd = null; + sharedFd = other.sharedFd; + bufferSize = other.bufferSize; + buffer = new AlignedByteBuffer(bufferSize, (int)ALIGN); + filePos = -bufferSize; + bufferPos = bufferSize; + isOpen = true; + isClone = true; + Seek(other.Position); } - } - } - public long length() - { - try - { - return channel.size(); - } - catch (IOException ioe) - { - throw new Exception("IOException during length(): " + this, ioe); - } - } + public override long Position => filePos + bufferPos; - public sbyte readByte() throws IOException - { - // NOTE: we don't guard against EOF here... ie the - // "final" buffer will typically be filled to less - // than bufferSize - if (bufferPos == bufferSize) - { - refill(); - } - Debug.Assert(bufferPos == buffer.position(), "bufferPos=" + bufferPos + " vs buffer.position()=" + buffer.position()); - bufferPos++; - return buffer.get(); - } + public override void Seek(long pos) + { + if (pos != Position) + { + long alignedPos = pos & ALIGN_NOT_MASK; + filePos = alignedPos - bufferSize; + + int delta = (int)(pos - alignedPos); + if (delta != 0) + { + Refill(); + buffer.Position = delta; + bufferPos = delta; + } + else + { + // force refill on next read + bufferPos = bufferSize; + } + } + } - private void refill() throws IOException - { - buffer.clear(); - filePos += bufferSize; - bufferPos = 0; - assert(filePos & ALIGN_NOT_MASK) == filePos : "filePos=" + filePos + " anded=" + (filePos & ALIGN_NOT_MASK); - //System.out.println("X refill filePos=" + filePos); - int n; - try - { - n = channel.read(buffer, filePos); - } - catch (IOException ioe) - { - throw new IOException(ioe.Message + ": " + this, ioe); - } - if (n < 0) - { - throw new EOFException("read past EOF: " + this); - } - buffer.rewind(); - } + public override long Length + { + get + { + try + { + return FileSize(sharedFd); + } + catch (Exception ioe) when (ioe.IsIOException()) + { + // LUCENENET: upstream wraps in a RuntimeException here (IndexInput.Length cannot + // throw a checked IOException in Java). Preserve the original HResult so callers + // that inspect it (e.g. NativeFSLock) still see the underlying error. The HResult + // setter is only public on modern TFMs; on net462/netstandard2.0 it is protected, + // so the detail is retained in the message there instead. + Exception wrapped = RuntimeException.Create("IOException during Length: " + this, ioe); +#if !(NETSTANDARD2_0 || NET462) + wrapped.HResult = ioe.HResult; +#endif + throw wrapped; + } + } + } - public void readBytes(sbyte[] dst, int offset, int len) throws IOException - { - int toRead = len; - //System.out.println("\nX readBytes len=" + len + " fp=" + getFilePointer() + " size=" + length() + " this=" + this); - while (true) - { -//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': -//ORIGINAL LINE: final int left = bufferSize - bufferPos; - int left = bufferSize - bufferPos; - if (left < toRead) - { - //System.out.println(" copy " + left); - buffer.get(dst, offset, left); - toRead -= left; - offset += left; - refill(); + public override byte ReadByte() + { + // NOTE: we don't guard against EOF here... ie the + // "final" buffer will typically be filled to less + // than bufferSize + if (bufferPos == bufferSize) + { + Refill(); + } + bufferPos++; + return buffer.Get(); } - else + + private void Refill() + { + buffer.Clear(); + filePos += bufferSize; + bufferPos = 0; + long n; + try + { + n = NativePosixUtil.Pread(sharedFd, filePos, buffer.Pointer, bufferSize); + } + catch (Exception ioe) when (ioe.IsIOException()) + { + // LUCENENET: surface the original HResult (via the ctor that accepts it, which is + // public on all target frameworks) so callers such as NativeFSLock still see the + // underlying error. The inner detail is retained in the message. + throw new IOException(ioe.Message + ": " + this, ioe.HResult); + } + // Upstream used FileChannel.read(), which returns -1 at EOF. Here Pread() wraps libc + // pread(), which returns 0 at EOF (a genuine error already threw above), so a refill that + // reads nothing means we have stepped entirely past the end of the file. + if (n <= 0) + { + throw EOFException.Create("read past EOF: " + this); + } + buffer.Rewind(); + } + + public override void ReadBytes(Span destination) { - //System.out.println(" copy " + toRead); - buffer.get(dst, offset, toRead); - bufferPos += toRead; - //System.out.println(" readBytes done"); - break; + int offset = 0; + int toRead = destination.Length; + while (true) + { + int left = bufferSize - bufferPos; + if (left < toRead) + { + buffer.Get(destination.Slice(offset, left)); + toRead -= left; + offset += left; + Refill(); + } + else + { + buffer.Get(destination.Slice(offset, toRead)); + bufferPos += toRead; + break; + } + } + } + + public override object Clone() + { + try + { + return new NativeUnixIndexInput(this); + } + catch (Exception ioe) when (ioe.IsIOException()) + { + // LUCENENET: as in Length above, upstream wraps in a RuntimeException (Clone cannot + // throw a checked IOException in Java). Preserve the original HResult where the setter + // is public (modern TFMs); on net462/netstandard2.0 the detail stays in the message. + Exception wrapped = RuntimeException.Create("IOException during clone: " + this, ioe); +#if !(NETSTANDARD2_0 || NET462) + wrapped.HResult = ioe.HResult; +#endif + throw wrapped; + } + } + + protected override void Dispose(bool disposing) + { + if (isOpen && !isClone) + { + isOpen = false; + fd.Dispose(); + } + // each instance (including clones) owns its own native buffer + buffer.Dispose(); } - } } - public NativeUnixIndexInput MemberwiseClone() + /// + /// P/Invoke declarations for the libc functions that the original implementation + /// reached through Java NIO FileChannels (positioned write, truncate, size). + /// + private static class NativeMethods { - try - { - return new NativeUnixIndexInput(this); - } - catch (IOException ioe) - { - throw new Exception("IOException during clone: " + this, ioe); - } + private const string LIBC = "libc"; + + internal const int SEEK_END = 2; + + [DllImport(LIBC, SetLastError = true)] + internal static extern nint pwrite(int fd, IntPtr buf, nuint count, long offset); + + [DllImport(LIBC, SetLastError = true)] + internal static extern int ftruncate(int fd, long length); + + [DllImport(LIBC, SetLastError = true)] + internal static extern long lseek(int fd, long offset, int whence); } - } } - } diff --git a/src/Lucene.Net.Misc/Store/WindowsDirectory.cs b/src/Lucene.Net.Misc/Store/WindowsDirectory.cs index 60a8698c4e..adccc3173a 100644 --- a/src/Lucene.Net.Misc/Store/WindowsDirectory.cs +++ b/src/Lucene.Net.Misc/Store/WindowsDirectory.cs @@ -1,9 +1,16 @@ +using Lucene.Net.Util; +using Microsoft.Win32.SafeHandles; using System; +using System.ComponentModel; +using System.IO; using System.Runtime.InteropServices; +#if FEATURE_SUPPORTEDOSPLATFORMATTRIBUTE +using System.Runtime.Versioning; +#endif +using System.Threading; -namespace org.apache.lucene.store +namespace Lucene.Net.Store { - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this @@ -21,161 +28,305 @@ namespace org.apache.lucene.store * the License. */ - - /// - /// Native implementation for Microsoft Windows. - /// - /// Steps: - ///
    - ///
  1. Compile the source code to create WindowsDirectory.dll: - ///
    - /// c:\mingw\bin\g++ -Wall -D_JNI_IMPLEMENTATION_ -Wl,--kill-at - /// -I"%JAVA_HOME%\include" -I"%JAVA_HOME%\include\win32" -static-libgcc - /// -static-libstdc++ -shared WindowsDirectory.cpp -o WindowsDirectory.dll - ///
    - /// For 64-bit JREs, use mingw64, with the -m64 option. - ///
  2. Put WindowsDirectory.dll into some directory in your windows PATH - ///
  3. Open indexes with WindowsDirectory and use it. - ///
- /// + /// Native implementation for Microsoft Windows. + /// + /// This uses CreateFile with the FILE_FLAG_RANDOM_ACCESS cache hint via + /// P/Invoke, so that the operating system can optimize its caching for the random-access + /// pattern that searching produces. Each read supplies its own file offset through the + /// OVERLAPPED structure; because the handle is synchronous (not opened with + /// FILE_FLAG_OVERLAPPED), these positioned reads are serialized by the OS on the + /// shared handle rather than running in parallel, but no explicit seek state is needed and + /// clones can read independently. + /// + /// NOTE: Unlike the original Lucene implementation, which + /// required compiling a native WindowsDirectory.dll with the JNI sources, + /// this implementation calls the Win32 APIs directly through P/Invoke. No native + /// build step is required, but it can only be used on Microsoft Windows. + /// /// @lucene.experimental /// +#if FEATURE_SUPPORTEDOSPLATFORMATTRIBUTE + [SupportedOSPlatform("windows")] +#endif public class WindowsDirectory : FSDirectory { - private const int DEFAULT_BUFFERSIZE = 4096; // default pgsize on ia32/amd64 - - static WindowsDirectory() - { -//JAVA TO C# CONVERTER TODO TASK: The library is specified in the 'DllImport' attribute for .NET: -// System.loadLibrary("WindowsDirectory"); - } - - /// - /// Create a new WindowsDirectory for the named location. - /// - /// the path of the directory - /// the lock factory to use, or null for the default - /// (); - /// If there is a low-level I/O error -//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: -//ORIGINAL LINE: public WindowsDirectory(java.io.File path, LockFactory lockFactory) throws java.io.IOException - public WindowsDirectory(File path, LockFactory lockFactory) : base(path, lockFactory) - { - } - - /// - /// Create a new WindowsDirectory for the named location and . - /// - /// the path of the directory - /// If there is a low-level I/O error -//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: -//ORIGINAL LINE: public WindowsDirectory(java.io.File path) throws java.io.IOException - public WindowsDirectory(File path) : base(path, null) - { - } - -//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: -//ORIGINAL LINE: @Override public IndexInput openInput(String name, IOContext context) throws java.io.IOException - public override IndexInput openInput(string name, IOContext context) - { - ensureOpen(); - return new WindowsIndexInput(new File(Directory, name), Math.Max(BufferedIndexInput.bufferSize(context), DEFAULT_BUFFERSIZE)); - } - - internal class WindowsIndexInput : BufferedIndexInput - { - internal readonly long fd; - internal readonly long length_Renamed; - internal bool isClone; - internal bool isOpen; - -//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: -//ORIGINAL LINE: public WindowsIndexInput(java.io.File file, int bufferSize) throws java.io.IOException - public WindowsIndexInput(File file, int bufferSize) : base("WindowsIndexInput(path=\"" + file.Path + "\")", bufferSize) + private const int DEFAULT_BUFFERSIZE = 4096; // default pgsize on ia32/amd64 + + /// + /// Create a new for the named location. + /// + /// the path of the directory + /// the lock factory to use, or null for the default + /// (); + /// If there is a low-level I/O error + /// If not running on Microsoft Windows + public WindowsDirectory(DirectoryInfo path, LockFactory lockFactory) + : base(path, lockFactory) { - fd = WindowsDirectory.open(file.Path); - length_Renamed = WindowsDirectory.length(fd); - isOpen = true; + EnsureWindows(); } -//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: -//ORIGINAL LINE: @Override protected void readInternal(byte[] b, int offset, int length) throws java.io.IOException - protected internal override void readInternal(sbyte[] b, int offset, int length) + /// + /// Create a new for the named location and . + /// + /// the path of the directory + /// If there is a low-level I/O error + /// If not running on Microsoft Windows + public WindowsDirectory(DirectoryInfo path) + : base(path, null) { - int bytesRead; - try - { - bytesRead = WindowsDirectory.read(fd, b, offset, length, FilePointer); - } - catch (IOException ioe) - { - throw new IOException(ioe.Message + ": " + this, ioe); - } - - if (bytesRead != length) - { - throw new EOFException("read past EOF: " + this); - } + EnsureWindows(); } -//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: -//ORIGINAL LINE: @Override protected void seekInternal(long pos) throws java.io.IOException - protected internal override void seekInternal(long pos) + /// + /// Create a new for the named location. + /// + /// LUCENENET specific overload for convenience using string instead of . + /// + /// the path of the directory + /// the lock factory to use, or null for the default + /// (); + /// If there is a low-level I/O error + /// If not running on Microsoft Windows + public WindowsDirectory(string path, LockFactory lockFactory) + : this(new DirectoryInfo(path), lockFactory) { } -//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: -//ORIGINAL LINE: @Override public synchronized void close() throws java.io.IOException - public override void close() + /// + /// Create a new for the named location and . + /// + /// LUCENENET specific overload for convenience using string instead of . + /// + /// the path of the directory + /// If there is a low-level I/O error + /// If not running on Microsoft Windows + public WindowsDirectory(string path) + : this(path, null) { - lock (this) + } + + private static void EnsureWindows() + { + if (!Constants.WINDOWS) { - // NOTE: we synchronize and track "isOpen" because Lucene sometimes closes IIs twice! - if (!isClone && isOpen) - { - WindowsDirectory.close(fd); - isOpen = false; - } + throw new PlatformNotSupportedException($"{nameof(WindowsDirectory)} is only supported on Microsoft Windows."); } } - public override long length() + public override IndexInput OpenInput(string name, IOContext context) { - return length_Renamed; + EnsureOpen(); + var path = Path.Combine(Directory.FullName, name); + return new WindowsIndexInput(path, Math.Max(BufferedIndexInput.GetBufferSize(context), DEFAULT_BUFFERSIZE)); } - public override WindowsIndexInput clone() + internal class WindowsIndexInput : BufferedIndexInput { - WindowsIndexInput clone = (WindowsIndexInput)base.clone(); - clone.isClone = true; - return clone; + private readonly SafeFileHandle fd; + private readonly long length; + private bool isClone; + private bool isOpen; + private int disposed = 0; // LUCENENET specific - allow double-dispose + + public WindowsIndexInput(string path, int bufferSize) + : base("WindowsIndexInput(path=\"" + path + "\")", bufferSize) + { + fd = WindowsDirectory.OpenFile(path); + try + { + length = WindowsDirectory.Length(fd); + } + catch + { + // LUCENENET specific: avoid leaking the handle if reading the length fails; + // the ctor throws so Dispose() will never be called on this instance. + fd.Dispose(); + throw; + } + isOpen = true; + } + + protected override void ReadInternal(Span b) + { + int bytesRead; + try + { + bytesRead = WindowsDirectory.Read(fd, b, Position); // LUCENENET: Position is the file pointer (renamed from getFilePointer()) + } + catch (Exception ioe) when (ioe.IsIOException()) + { + // LUCENENET: surface the original HResult (via the ctor that accepts it, which is + // public on all target frameworks) so callers such as NativeFSLock, which inspect + // IOException.HResult, still see the underlying error. The inner detail is retained + // in the message. + throw new IOException(ioe.Message + ": " + this, ioe.HResult); + } + + if (bytesRead != b.Length) + { + throw EOFException.Create("read past EOF: " + this); + } + } + + protected override void SeekInternal(long pos) + { + } + + protected override void Dispose(bool disposing) + { + // NOTE: we track "isOpen" because Lucene sometimes closes IndexInputs twice! + if (0 != Interlocked.CompareExchange(ref this.disposed, 1, 0)) return; // LUCENENET specific - allow double-dispose + + if (disposing && !isClone && isOpen) + { + fd.Dispose(); // closes the underlying Win32 handle (CloseHandle) + isOpen = false; + } + } + + public override sealed long Length => length; + + public override object Clone() + { + WindowsIndexInput clone = (WindowsIndexInput)base.Clone(); + clone.isClone = true; + return clone; + } } - } - - /// - /// Opens a handle to a file. -//JAVA TO C# CONVERTER TODO TASK: Replace 'unknown' with the appropriate dll name: - [DllImport("unknown")] - private static extern long open(string filename); - - /// - /// Reads data from a file at pos into bytes -//JAVA TO C# CONVERTER TODO TASK: Replace 'unknown' with the appropriate dll name: - [DllImport("unknown")] - private static extern int read(long fd, sbyte[] bytes, int offset, int length, long pos); - - /// - /// Closes a handle to a file -//JAVA TO C# CONVERTER TODO TASK: Replace 'unknown' with the appropriate dll name: - [DllImport("unknown")] - private static extern void close(long fd); - - /// - /// Returns the length of a file -//JAVA TO C# CONVERTER TODO TASK: Replace 'unknown' with the appropriate dll name: - [DllImport("unknown")] - private static extern long length(long fd); - } + /// + /// Opens a handle to a file. + private static SafeFileHandle OpenFile(string filename) + { + // LUCENENET specific: include FILE_SHARE_DELETE (upstream used only READ|WRITE) so that, + // like SimpleFSDirectory's read path (#1283), an open read handle does not block deletion + // of the underlying file on Windows. + IntPtr handle = NativeMethods.CreateFileW( + filename, + NativeMethods.GENERIC_READ, + NativeMethods.FILE_SHARE_READ | NativeMethods.FILE_SHARE_WRITE | NativeMethods.FILE_SHARE_DELETE, + IntPtr.Zero, + NativeMethods.OPEN_EXISTING, + NativeMethods.FILE_FLAG_RANDOM_ACCESS, + IntPtr.Zero); + int lastError = Marshal.GetLastWin32Error(); + // LUCENENET: surface the Win32 error as the IOException's HResult (matching what the BCL + // sets on a FileStream IOException) so callers such as NativeFSLock can recognize it. + int hresult = Marshal.GetHRForLastWin32Error(); + + if (handle == NativeMethods.INVALID_HANDLE_VALUE) + { + throw new IOException("Could not open file " + filename + ": " + new Win32Exception(lastError).Message, hresult); + } + + return new SafeFileHandle(handle, ownsHandle: true); + } + + /// + /// Reads data from a file at into . + /// + /// The position is supplied via the OVERLAPPED structure. The handle is + /// synchronous (no FILE_FLAG_OVERLAPPED), so the read completes synchronously and + /// is serialized by the OS on the handle; the explicit offset is what lets clones share + /// one handle without seeking. + private static unsafe int Read(SafeFileHandle fd, Span bytes, long pos) + { + NativeOverlapped overlapped = default; + overlapped.OffsetLow = (int)(pos & 0xFFFFFFFFL); + overlapped.OffsetHigh = (int)((pos >> 0x20) & 0x7FFFFFFFL); + + int numRead; + bool success; + fixed (byte* p = bytes) + { + success = NativeMethods.ReadFile(fd, p, bytes.Length, out numRead, &overlapped); + } + + if (!success) + { + int lastError = Marshal.GetLastWin32Error(); + // LUCENENET: surface the Win32 error as the IOException's HResult (see OpenFile). + throw new IOException(new Win32Exception(lastError).Message, Marshal.GetHRForLastWin32Error()); + } + + return numRead; + } + + /// + /// Returns the length of a file. + private static long Length(SafeFileHandle fd) + { + if (!NativeMethods.GetFileInformationByHandle(fd, out NativeMethods.BY_HANDLE_FILE_INFORMATION info)) + { + int lastError = Marshal.GetLastWin32Error(); + // LUCENENET: surface the Win32 error as the IOException's HResult (see OpenFile). + throw new IOException(new Win32Exception(lastError).Message, Marshal.GetHRForLastWin32Error()); + } + + return ((long)info.nFileSizeHigh << 0x20) | info.nFileSizeLow; + } + + /// + /// P/Invoke declarations for the Win32 APIs used by . + /// + /// These replace the JNI/C++ native methods (open, read, close, + /// length) in the original Lucene implementation. Closing is handled by + /// , which calls CloseHandle. + /// + private static class NativeMethods + { + internal const uint GENERIC_READ = 0x80000000; + internal const uint FILE_SHARE_READ = 0x00000001; + internal const uint FILE_SHARE_WRITE = 0x00000002; + internal const uint FILE_SHARE_DELETE = 0x00000004; + internal const uint OPEN_EXISTING = 3; + internal const uint FILE_FLAG_RANDOM_ACCESS = 0x10000000; + + internal static readonly IntPtr INVALID_HANDLE_VALUE = new IntPtr(-1); + + // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + internal static extern IntPtr CreateFileW( + string lpFileName, + uint dwDesiredAccess, + uint dwShareMode, + IntPtr lpSecurityAttributes, + uint dwCreationDisposition, + uint dwFlagsAndAttributes, + IntPtr hTemplateFile); + + // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-readfile + [DllImport("kernel32.dll", SetLastError = true)] + internal static extern unsafe bool ReadFile( + SafeFileHandle hFile, + byte* lpBuffer, + int nNumberOfBytesToRead, + out int lpNumberOfBytesRead, + NativeOverlapped* lpOverlapped); + + // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfileinformationbyhandle + [DllImport("kernel32.dll", SetLastError = true)] + internal static extern bool GetFileInformationByHandle( + SafeFileHandle hFile, + out BY_HANDLE_FILE_INFORMATION lpFileInformation); + + // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/ns-fileapi-by_handle_file_information + [StructLayout(LayoutKind.Sequential)] + internal struct BY_HANDLE_FILE_INFORMATION + { + public uint dwFileAttributes; + public System.Runtime.InteropServices.ComTypes.FILETIME ftCreationTime; + public System.Runtime.InteropServices.ComTypes.FILETIME ftLastAccessTime; + public System.Runtime.InteropServices.ComTypes.FILETIME ftLastWriteTime; + public uint dwVolumeSerialNumber; + public uint nFileSizeHigh; + public uint nFileSizeLow; + public uint nNumberOfLinks; + public uint nFileIndexHigh; + public uint nFileIndexLow; + } + } + } } diff --git a/src/Lucene.Net.Tests.Misc/Support/Store/TestNativeUnixDirectory.cs b/src/Lucene.Net.Tests.Misc/Support/Store/TestNativeUnixDirectory.cs new file mode 100644 index 0000000000..82aa3fcf94 --- /dev/null +++ b/src/Lucene.Net.Tests.Misc/Support/Store/TestNativeUnixDirectory.cs @@ -0,0 +1,334 @@ +using Lucene.Net.Attributes; +using Lucene.Net.Util; +using NUnit.Framework; +using RandomizedTesting.Generators; +using System; +using System.Collections.Generic; +using System.IO; +#if FEATURE_SUPPORTEDOSPLATFORMATTRIBUTE +using System.Runtime.Versioning; +#endif +using System.Threading; +using Assert = Lucene.Net.TestFramework.Assert; + +namespace Lucene.Net.Store +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// + /// Cross-platform tests for the platform guard on and + /// . This fixture runs on every platform; on Microsoft Windows it + /// verifies that the types refuse to operate (they require Linux/macOS direct I/O). + /// + /// LUCENENET specific: the original Lucene types had no test coverage (GH-1342). + /// + [TestFixture] + [LuceneNetSpecific] + public class TestNativeUnixDirectoryPlatformGuard : LuceneTestCase + { + [Test] + public virtual void TestDirectoryThrowsOnWindows() + { + DirectoryInfo path = CreateTempDir("nativeUnixGuard"); +#pragma warning disable CA1416 // Validate platform compatibility - intentionally exercising the guard on all platforms + using Directory @delegate = new RAMDirectory(); + if (Constants.WINDOWS) + { + Assert.Throws(() => new NativeUnixDirectory(path, @delegate)); + } + else + { + // On Unix the constructor must succeed (no direct files are opened until a merge-context + // CreateOutput/OpenInput); functional behavior is covered by TestRoundTrip below. + using Directory dir = new NativeUnixDirectory(path, @delegate); + Assert.IsNotNull(dir); + } +#pragma warning restore CA1416 // Validate platform compatibility + } + + [Test] + public virtual void TestOpenDirectThrowsOnWindows() + { + if (Constants.WINDOWS) + { +#pragma warning disable CA1416 // Validate platform compatibility - intentionally exercising the guard + Assert.Throws(() => NativePosixUtil.OpenDirect("anything", read: true)); +#pragma warning restore CA1416 // Validate platform compatibility + } + else + { + AssumeTrue("OpenDirect's open path is exercised by the Unix functional tests.", false); + } + } + } + + /// + /// Functional tests for 's direct-I/O read/write path. These + /// only run on Unix-like platforms and additionally require a filesystem that supports direct + /// I/O (O_DIRECT); they are skipped on Microsoft Windows. + /// + /// Unlike TestWindowsDirectory, this does not subclass : + /// only uses its direct-I/O input/output in a MERGE + /// for files larger than minBytesDirect (otherwise it delegates), + /// and the experimental direct-I/O input does not throw on read-past-EOF the way the generic + /// suite expects, so the randomized generic suite would be non-deterministic. These targeted + /// tests instead exercise the direct path in its designed usage (sequential write, full read, + /// clone, and concurrent clones). + /// + /// LUCENENET specific (GH-1342): the original Lucene type had no test coverage. + /// + [TestFixture] + [LuceneNetSpecific] +#if FEATURE_SUPPORTEDOSPLATFORMATTRIBUTE + [UnsupportedOSPlatform("windows")] +#endif + public class TestNativeUnixDirectory : LuceneTestCase + { + // A MERGE context with estimatedMergeBytes >= minBytesDirect(0) forces the direct path. + private static readonly IOContext MERGE_CONTEXT = new IOContext(new MergeInfo(1, 1024 * 1024, false, 1)); + + public override void SetUp() + { + base.SetUp(); + AssumeTrue("NativeUnixDirectory requires Linux or macOS direct I/O.", !Constants.WINDOWS); + } + + /// + /// Creates a (plus its delegate) configured so that the + /// direct-I/O path is always taken for MERGE-context I/O. The caller must dispose both the + /// returned directory and . + /// + private NativeUnixDirectory NewDirectDirectory(DirectoryInfo path, out Directory @delegate) + { + @delegate = new SimpleFSDirectory(path); + // mergeBufferSize must be a multiple of 512 (and, per the upstream check, have bit 9 clear). + return new NativeUnixDirectory(path, mergeBufferSize: 1024, minBytesDirect: 0, @delegate); + } + + /// + /// Writes a file through the direct-I/O merge path and reads it back (in bulk), verifying the + /// bytes round-trip across many buffer dumps/refills, plus a clone read. + /// + [Test] + public virtual void TestRoundTrip() + { + DirectoryInfo path = CreateTempDir("nativeUnixRoundTrip"); + using var dir = NewDirectDirectory(path, out Directory @delegate); + using (@delegate) + { + int len = TestUtil.NextInt32(Random, 2000, 20000); + byte[] expected = new byte[len]; + Random.NextBytes(expected); + + using (IndexOutput output = dir.CreateOutput("test", MERGE_CONTEXT)) + { + output.WriteBytes(expected, expected.Length); + } + + using IndexInput input = dir.OpenInput("test", MERGE_CONTEXT); + Assert.AreEqual(len, input.Length); + byte[] actual = new byte[len]; + input.ReadBytes(actual, 0, actual.Length); + Assert.AreEqual(expected, actual); + + // random positioned reads + for (int i = 0; i < 50; i++) + { + int pos = Random.Next(len); + int count = Math.Min(TestUtil.NextInt32(Random, 1, 500), len - pos); + input.Seek(pos); + byte[] buf = new byte[count]; + input.ReadBytes(buf, 0, count); + for (int j = 0; j < count; j++) + { + Assert.AreEqual(expected[pos + j], buf[j], "mismatch at " + (pos + j)); + } + } + + // a clone reads independently from the same shared descriptor + input.Seek(0); + using IndexInput clone = (IndexInput)input.Clone(); + int clonePos = Random.Next(len); + clone.Seek(clonePos); + Assert.AreEqual(expected[clonePos], clone.ReadByte()); + } + } + + /// + /// Disposing an input (and a clone) more than once must be a no-op; clones must not close the + /// shared descriptor. Mirrors the equivalent WindowsDirectory test. + /// + [Test] + public virtual void TestCloneDisposeDoesNotCloseSharedDescriptor() + { + DirectoryInfo path = CreateTempDir("nativeUnixDoubleDispose"); + using var dir = NewDirectDirectory(path, out Directory @delegate); + using (@delegate) + { + byte[] data = new byte[2048]; + Random.NextBytes(data); + IndexOutput output = dir.CreateOutput("data", MERGE_CONTEXT); + output.WriteBytes(data, data.Length); + Assert.DoesNotThrow(() => output.Dispose()); + Assert.DoesNotThrow(() => output.Dispose()); // double dispose is a no-op + + IndexInput input = dir.OpenInput("data", MERGE_CONTEXT); + IndexInput clone = (IndexInput)input.Clone(); + + // disposing the clone must not close the shared descriptor: the original still reads + Assert.DoesNotThrow(() => clone.Dispose()); + Assert.DoesNotThrow(() => clone.Dispose()); + input.Seek(0); + byte[] check = new byte[data.Length]; + input.ReadBytes(check, 0, check.Length); + Assert.AreEqual(data, check); + + Assert.DoesNotThrow(() => input.Dispose()); + Assert.DoesNotThrow(() => input.Dispose()); // double dispose is a no-op + } + } + + /// + /// Many threads reading random positions through independent clones of the same input (and + /// thus the same shared descriptor) must each get correct data. Positioned pread is + /// atomic and each clone has its own buffer, so this must be safe under concurrency. Mirrors + /// the equivalent WindowsDirectory test. + /// + [Test] + public virtual void TestConcurrentCloneReads() + { + DirectoryInfo path = CreateTempDir("nativeUnixConcurrent"); + using var dir = NewDirectDirectory(path, out Directory @delegate); + using (@delegate) + { + int len = TestUtil.NextInt32(Random, 10000, 50000); + byte[] expected = new byte[len]; + Random.NextBytes(expected); + using (IndexOutput output = dir.CreateOutput("data", MERGE_CONTEXT)) + { + output.WriteBytes(expected, expected.Length); + } + + using IndexInput input = dir.OpenInput("data", MERGE_CONTEXT); + + int numThreads = Math.Max(2, Environment.ProcessorCount); + var threads = new Thread[numThreads]; + Exception failure = null; + using (var start = new ManualResetEventSlim(false)) + { + for (int t = 0; t < numThreads; t++) + { + int seed = t; + threads[t] = new Thread(() => + { + try + { + var rnd = new J2N.Randomizer(seed + 1); + using IndexInput clone = (IndexInput)input.Clone(); + start.Wait(); + for (int i = 0; i < 1000; i++) + { + int pos = rnd.Next(len); + int count = Math.Min(rnd.Next(1, 256), len - pos); + clone.Seek(pos); + byte[] buf = new byte[count]; + clone.ReadBytes(buf, 0, count); + for (int j = 0; j < count; j++) + { + if (buf[j] != expected[pos + j]) + { + throw new Exception($"mismatch at {pos + j}: got {buf[j]}, expected {expected[pos + j]}"); + } + } + } + } + catch (Exception e) + { + Interlocked.CompareExchange(ref failure, e, null); + } + }); + threads[t].Start(); + } + + start.Set(); // release all threads at once to maximize contention + foreach (var thread in threads) + { + thread.Join(); + } + } + + Assert.IsNull(failure, "concurrent clone reads failed: " + failure); + } + } + } + + /// + /// Runs the full suite against + /// with minBytesDirect == 0, so the direct-I/O path is taken for all MERGE-context I/O. + /// Unix-gated and requires an O_DIRECT-capable filesystem. + /// + /// LUCENENET specific (GH-1342). + /// + [TestFixture] + [LuceneNetSpecific] +#if FEATURE_SUPPORTEDOSPLATFORMATTRIBUTE + [UnsupportedOSPlatform("windows")] +#endif + public class TestNativeUnixDirectoryBase : BaseDirectoryTestCase + { + private readonly List delegates = new List(); + + public override void SetUp() + { + base.SetUp(); + AssumeTrue("NativeUnixDirectory requires Linux or macOS direct I/O.", !Constants.WINDOWS); + } + + public override void TearDown() + { + foreach (var d in delegates) + { + try { d.Dispose(); } catch { /* ignore: best-effort cleanup of the delegate */ } + } + delegates.Clear(); + base.TearDown(); + } + + protected override Directory GetDirectory(DirectoryInfo path) + { + var del = new SimpleFSDirectory(path); + delegates.Add(del); + // minBytesDirect = 0 so the direct-I/O path is used for all MERGE-context I/O. + return new NativeUnixDirectory(path, mergeBufferSize: 1024, minBytesDirect: 0, del); + } + + /// + /// Not applicable to . It is a composite/delegating directory: + /// files written in a non-MERGE context go through the delegate and are tracked in the + /// delegate's staleFiles, not this directory's. Since only + /// fsyncs files in its own staleFiles (toSync.IntersectWith(m_staleFiles)), it does + /// not throw for a file written through the delegate, so this backdoor-the-filesystem test does + /// not hold here. The base test itself notes (TODO) that it does not handle composite/two-FSDir + /// directories. + /// + public override void TestFsyncDoesntCreateNewFiles() + { + AssumeTrue("TestFsyncDoesntCreateNewFiles does not apply to the composite/delegating NativeUnixDirectory; see override remarks.", false); + } + } +} diff --git a/src/Lucene.Net.Tests.Misc/Support/Store/TestWindowsDirectory.cs b/src/Lucene.Net.Tests.Misc/Support/Store/TestWindowsDirectory.cs new file mode 100644 index 0000000000..6dc5be95ba --- /dev/null +++ b/src/Lucene.Net.Tests.Misc/Support/Store/TestWindowsDirectory.cs @@ -0,0 +1,281 @@ +using Lucene.Net.Attributes; +using Lucene.Net.Util; +using NUnit.Framework; +using RandomizedTesting.Generators; +using System; +using System.IO; +#if FEATURE_SUPPORTEDOSPLATFORMATTRIBUTE +using System.Runtime.Versioning; +#endif +using System.Threading; +using Assert = Lucene.Net.TestFramework.Assert; + +namespace Lucene.Net.Store +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// + /// Tests for . + /// + /// This runs the full suite against + /// . Because uses the + /// Win32 file APIs via P/Invoke, these tests are only run on Microsoft Windows; on + /// other platforms the entire fixture is skipped (see ). + /// + /// LUCENENET specific: the original Lucene WindowsDirectory had no test coverage (GH-1342). + /// + [TestFixture] + [LuceneNetSpecific] +#if FEATURE_SUPPORTEDOSPLATFORMATTRIBUTE + [SupportedOSPlatform("windows")] +#endif + public class TestWindowsDirectory : BaseDirectoryTestCase + { + public override void SetUp() + { + base.SetUp(); + AssumeTrue("WindowsDirectory is only supported on Microsoft Windows.", Constants.WINDOWS); + } + + protected override Directory GetDirectory(DirectoryInfo path) + { + return new WindowsDirectory(path); + } + + /// + /// Exercises the positioned reads that back : random seeks + /// plus an independent clone reading from a different position, all sharing the same + /// underlying Win32 file handle. + /// + [Test] + public virtual void TestRandomAccessAndClones() + { + using Directory dir = GetDirectory(CreateTempDir("testWindowsRandomAccess")); + + int len = TestUtil.NextInt32(Random, 100, 100000); + byte[] expected = new byte[len]; + Random.NextBytes(expected); + + using (IndexOutput output = dir.CreateOutput("data", NewIOContext(Random))) + { + output.WriteBytes(expected, expected.Length); + } + + using IndexInput input = dir.OpenInput("data", NewIOContext(Random)); + Assert.AreEqual(len, input.Length); + + // sequential read of the whole file + byte[] actual = new byte[len]; + input.ReadBytes(actual, 0, actual.Length); + Assert.AreEqual(expected, actual); + + // random positioned reads + for (int i = 0; i < 100; i++) + { + int pos = Random.Next(len); + int count = Math.Min(TestUtil.NextInt32(Random, 1, 1000), len - pos); + input.Seek(pos); + byte[] buf = new byte[count]; + input.ReadBytes(buf, 0, count); + for (int j = 0; j < count; j++) + { + Assert.AreEqual(expected[pos + j], buf[j], "mismatch at " + (pos + j)); + } + } + + // a clone shares the same handle but must read independently of the original + input.Seek(0); + using (IndexInput clone = (IndexInput)input.Clone()) + { + int clonePos = Random.Next(len); + clone.Seek(clonePos); + Assert.AreEqual(expected[clonePos], clone.ReadByte()); + + // the original's position is unaffected by the clone's reads + Assert.AreEqual(0, input.Position); + Assert.AreEqual(expected[0], input.ReadByte()); + } + } + + /// + /// Reading past the end of the file must throw an EOF exception, not silently return + /// garbage or hang. + /// + [Test] + public virtual void TestReadPastEOF() + { + using Directory dir = GetDirectory(CreateTempDir("testWindowsEOF")); + + const int len = 100; + using (IndexOutput output = dir.CreateOutput("eof", NewIOContext(Random))) + { + output.WriteBytes(new byte[len], len); + } + + using IndexInput input = dir.OpenInput("eof", NewIOContext(Random)); + Assert.AreEqual(len, input.Length); + + // read right up to EOF: fine + input.Seek(len); + try + { + input.ReadByte(); + fail("did not hit expected EOF exception"); + } + catch (Exception e) when (e.IsEOFException()) + { + // expected + } + + // a read that straddles EOF must also throw + input.Seek(len - 4); + try + { + byte[] buf = new byte[8]; + input.ReadBytes(buf, 0, buf.Length); + fail("did not hit expected EOF exception"); + } + catch (Exception e) when (e.IsEOFException()) + { + // expected + } + } + + /// + /// Disposing an input (and a clone) more than once must be a no-op, never a double + /// CloseHandle. Clones must not close the shared handle. + /// + [Test] + public virtual void TestCloneDisposeDoesNotCloseSharedHandle() + { + using Directory dir = GetDirectory(CreateTempDir("testWindowsDoubleDispose")); + using (IndexOutput output = dir.CreateOutput("data", NewIOContext(Random))) + { + output.WriteInt64(42); + } + + IndexInput input = dir.OpenInput("data", NewIOContext(Random)); + IndexInput clone = (IndexInput)input.Clone(); + + // disposing the clone must not close the shared handle: original still reads + Assert.DoesNotThrow(() => clone.Dispose()); + Assert.DoesNotThrow(() => clone.Dispose()); + input.Seek(0); + Assert.AreEqual(42L, input.ReadInt64()); + + // original double-dispose is a no-op + Assert.DoesNotThrow(() => input.Dispose()); + Assert.DoesNotThrow(() => input.Dispose()); + } + + /// + /// Many threads reading random positions through independent clones of the same input + /// (and thus the same shared Win32 handle) must each get correct data. This proves the + /// positioned reads are safe under concurrency, not just single-threaded. + /// + [Test] + public virtual void TestConcurrentCloneReads() + { + using Directory dir = GetDirectory(CreateTempDir("testWindowsConcurrent")); + + int len = TestUtil.NextInt32(Random, 10000, 50000); + byte[] expected = new byte[len]; + Random.NextBytes(expected); + using (IndexOutput output = dir.CreateOutput("data", NewIOContext(Random))) + { + output.WriteBytes(expected, expected.Length); + } + + using IndexInput input = dir.OpenInput("data", NewIOContext(Random)); + + int numThreads = Math.Max(2, Environment.ProcessorCount); + var threads = new Thread[numThreads]; + Exception failure = null; + using (var start = new ManualResetEventSlim(false)) + { + for (int t = 0; t < numThreads; t++) + { + int seed = t; + threads[t] = new Thread(() => + { + try + { + var rnd = new J2N.Randomizer(seed + 1); + using IndexInput clone = (IndexInput)input.Clone(); + start.Wait(); + for (int i = 0; i < 2000; i++) + { + int pos = rnd.Next(len); + int count = Math.Min(rnd.Next(1, 256), len - pos); + clone.Seek(pos); + byte[] buf = new byte[count]; + clone.ReadBytes(buf, 0, count); + for (int j = 0; j < count; j++) + { + if (buf[j] != expected[pos + j]) + { + throw new Exception($"mismatch at {pos + j}: got {buf[j]}, expected {expected[pos + j]}"); + } + } + } + } + catch (Exception e) + { + Interlocked.CompareExchange(ref failure, e, null); + } + }); + threads[t].Start(); + } + + start.Set(); // release all threads at once to maximize contention + foreach (var thread in threads) + { + thread.Join(); + } + } + + Assert.IsNull(failure, "concurrent clone reads failed: " + failure); + } + } + + /// + /// Verifies the cross-platform guard on . Unlike + /// , this fixture runs on every platform. + /// + [TestFixture] + [LuceneNetSpecific] + public class TestWindowsDirectoryPlatformGuard : LuceneTestCase + { + [Test] + public virtual void TestThrowsOnNonWindows() + { + DirectoryInfo path = CreateTempDir("testWindowsGuard"); +#pragma warning disable CA1416 // Validate platform compatibility - intentionally exercising the guard on all platforms + if (Constants.WINDOWS) + { + using Directory dir = new WindowsDirectory(path); + Assert.IsNotNull(dir); + } + else + { + Assert.Throws(() => new WindowsDirectory(path)); + } +#pragma warning restore CA1416 // Validate platform compatibility + } + } +}