From a61acb7dae6723496833b45259fdf4e9f3de3e8f Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 17 Feb 2026 10:14:24 -0500 Subject: [PATCH 1/4] refactor: back deprecated methods by default implementation using the new methods --- .../n5/BufferedKvaLockedChannel.java | 47 +++++++++++++++++++ .../saalfeldlab/n5/KeyValueAccess.java | 8 +++- 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java new file mode 100644 index 000000000..70ed4ba60 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java @@ -0,0 +1,47 @@ +package org.janelia.saalfeldlab.n5; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +import java.io.*; + +class BufferedKvaLockedChannel implements LockedChannel { + + private final KeyValueAccess kva; + private final String key; + private ByteArrayOutputStream baos = null; + + BufferedKvaLockedChannel(final KeyValueAccess kva, final String key) { + this.kva = kva; + this.key = key; + } + + @Override + public Reader newReader() throws N5Exception.N5IOException { + + return new InputStreamReader(newInputStream()); + } + + @Override + public InputStream newInputStream() throws N5Exception.N5IOException { + return kva.createReadData(key).inputStream(); + } + + @Override + public Writer newWriter() throws N5Exception.N5IOException { + + return new BufferedWriter(new OutputStreamWriter(newOutputStream())); + } + + @Override + public OutputStream newOutputStream() throws N5Exception.N5IOException { + if (baos == null) + baos = new ByteArrayOutputStream(); + return baos; + } + + @Override + public void close() throws IOException { + if (baos != null && baos.size() > 0) + kva.write(key, ReadData.from(baos.toByteArray())); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index e6465c5b8..49c450ff7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -297,7 +297,9 @@ default URI uri(final String uriString) throws URISyntaxException { * @deprecated migrate to {@link KeyValueAccess#createReadData(String)} */ @Deprecated - LockedChannel lockForReading( final String normalPath ) throws N5IOException; + default LockedChannel lockForReading( final String normalPath ) throws N5IOException { + return new BufferedKvaLockedChannel(this, normalPath); + } /** * Create an exclusive lock on a path for writing. If the file doesn't exist @@ -318,7 +320,9 @@ default URI uri(final String uriString) throws URISyntaxException { * @deprecated migrate to {@link KeyValueAccess#write(String, ReadData)} */ @Deprecated - LockedChannel lockForWriting( final String normalPath ) throws N5IOException; + default LockedChannel lockForWriting( final String normalPath ) throws N5IOException { + return new BufferedKvaLockedChannel(this, normalPath); + } /** * List all 'directory'-like children of a path. From cb1da33b3db4b9bede798e92543cd5d3d97bdca7 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 17 Feb 2026 11:09:00 -0500 Subject: [PATCH 2/4] refactor: remove unused static lockForReading/Writing methods in favore of new default in KeyValueAccess. Move FileLazyRead to FsIoPolicy. Signed-off-by: Caleb Hulbert --- .../n5/FileSystemKeyValueAccess.java | 149 ++---------------- .../janelia/saalfeldlab/n5/FsIoPolicy.java | 91 ++++++++++- .../janelia/saalfeldlab/n5/N5Exception.java | 33 +++- 3 files changed, 132 insertions(+), 141 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 0bf0fbe06..ec58d6870 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -34,8 +34,6 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.attribute.FileAttribute; @@ -45,11 +43,8 @@ import java.util.stream.Stream; import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.readdata.LazyRead; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; -import static org.janelia.saalfeldlab.n5.FileKeyLockManager.FILE_LOCK_MANAGER; - /** * Filesystem {@link KeyValueAccess}. * @@ -100,44 +95,6 @@ public void write(final String normalPath, final ReadData data) throws N5IOExcep } } - @Override - @Deprecated - public LockedFileChannel lockForReading(final String normalPath) throws N5IOException { - - return lockForReading(Paths.get(normalPath)); - } - - @Override - @Deprecated - public LockedChannel lockForWriting(final String normalPath) throws N5IOException { - - return lockForWriting(Paths.get(normalPath)); - } - - @Deprecated - protected static LockedFileChannel lockForReading(final Path path) throws N5IOException { - - try { - return FILE_LOCK_MANAGER.lockForReading(path); - } catch (final NoSuchFileException e) { - throw new N5NoSuchKeyException("No such file", e); - } catch (IOException | UncheckedIOException e) { - throw new N5IOException("Failed to lock file for reading: " + path, e); - } - } - - @Deprecated - static LockedFileChannel lockForWriting(final Path path) throws N5IOException { - - try { - return FILE_LOCK_MANAGER.lockForWriting(path); - } catch (final NoSuchFileException e) { - throw new N5NoSuchKeyException("No such file", e); - } catch (IOException | UncheckedIOException e) { - throw new N5IOException("Failed to lock file for writing: " + path, e); - } - } - @Override public boolean isDirectory(final String normalPath) { @@ -165,17 +122,6 @@ public long size(final String normalPath) { return size(Paths.get(normalPath)); } - protected static long size(final Path path) { - - try { - return Files.size(path); - } catch (NoSuchFileException e) { - throw new N5NoSuchKeyException("No such file", e); - } catch (IOException | UncheckedIOException e) { - throw new N5IOException(e); - } - } - @Override public String[] listDirectories(final String normalPath) throws N5IOException { @@ -346,6 +292,17 @@ public void delete(final String normalPath) throws N5IOException { } } + protected static long size(final Path path) { + + try { + return Files.size(path); + } catch (NoSuchFileException e) { + throw new N5NoSuchKeyException("No such file", e); + } catch (IOException | UncheckedIOException e) { + throw new N5IOException(e); + } + } + protected static void tryDelete(final Path path) throws IOException { try { @@ -501,88 +458,4 @@ protected static void createAndCheckIsDirectory( throw x; } } - - static class FileLazyRead implements LazyRead { - - private static final Closeable NO_OP = () -> { }; - - private final Path path; - private Closeable lock; - - FileLazyRead(final Path path) throws IOException { - this(path, true); - } - - FileLazyRead(final Path path, final boolean requireLock ) throws IOException { - this.path = path; - if (requireLock) - lock = FILE_LOCK_MANAGER.lockForReading(path); - else - lock = NO_OP; - } - - @Override - public long size() throws N5IOException { - - if (lock == null) { - throw new N5IOException("FileLazyRead is already closed."); - } - return FileSystemKeyValueAccess.size(path); - } - - @Override - public ReadData materialize(final long offset, final long length) { - - if (lock == null) { - throw new N5IOException("FileLazyRead is already closed."); - } - - try (final FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { - - channel.position(offset); - - final long channelSize = channel.size(); - if (!validBounds(channelSize, offset, length)) { - throw new IndexOutOfBoundsException(); - } - - final long size = length < 0 ? (channelSize - offset) : length; - if (size > Integer.MAX_VALUE) { - throw new IndexOutOfBoundsException("Attempt to materialize too large data"); - } - - final byte[] data = new byte[(int) size]; - final ByteBuffer buf = ByteBuffer.wrap(data); - channel.read(buf); - return ReadData.from(data); - - } catch (final NoSuchFileException e) { - throw new N5NoSuchKeyException("No such file", e); - } catch (IOException | UncheckedIOException e) { - throw new N5IOException(e); - } - } - - @Override - public void close() throws IOException { - - if (lock != null) { - lock.close(); - lock = null; - } - } - } - - private static boolean validBounds(long channelSize, long offset, long length) { - - if (offset < 0) - return false; - else if (channelSize > 0 && offset >= channelSize) // offset == 0 and channelSize == 0 is okay - return false; - else if (length >= 0 && offset + length > channelSize) - return false; - - return true; - } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java index 457896acd..5d01dd919 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java @@ -1,9 +1,13 @@ package org.janelia.saalfeldlab.n5; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; +import java.io.Closeable; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.*; @@ -13,6 +17,18 @@ public class FsIoPolicy { static final IoPolicy atomicWithFallback = IoPolicy.withFallback(new Atomic(), new Unsafe()); + private static boolean validBounds(long channelSize, long offset, long length) { + + if (offset < 0) + return false; + else if (channelSize > 0 && offset >= channelSize) // offset == 0 and channelSize == 0 is okay + return false; + else if (length >= 0 && offset + length > channelSize) + return false; + + return true; + } + public static class Unsafe implements IoPolicy { @Override public void write(String key, ReadData readData) throws IOException { @@ -23,7 +39,7 @@ public void write(String key, ReadData readData) throws IOException { @Override public VolatileReadData read(final String key) throws IOException { final Path path = Paths.get(key); - FileSystemKeyValueAccess.FileLazyRead fileLazyRead = new FileSystemKeyValueAccess.FileLazyRead(path, false); + FileLazyRead fileLazyRead = new FileLazyRead(path, false); return VolatileReadData.from(fileLazyRead); } @@ -46,7 +62,7 @@ public void write(String key, ReadData readData) throws IOException { @Override public VolatileReadData read(String key) throws IOException { final Path path = Paths.get(key); - FileSystemKeyValueAccess.FileLazyRead fileLazyRead = new FileSystemKeyValueAccess.FileLazyRead(path, true); + FileLazyRead fileLazyRead = new FileLazyRead(path, true); return VolatileReadData.from(fileLazyRead); } @@ -58,4 +74,75 @@ public void delete(final String key) throws IOException { } } } + + static class FileLazyRead implements LazyRead { + + private static final Closeable NO_OP = () -> { }; + + private final Path path; + private Closeable lock; + + FileLazyRead(final Path path) throws IOException { + this(path, true); + } + + FileLazyRead(final Path path, final boolean requireLock ) throws IOException { + this.path = path; + if (requireLock) + lock = FILE_LOCK_MANAGER.lockForReading(path); + else + lock = NO_OP; + } + + @Override + public long size() throws N5Exception.N5IOException { + + if (lock == null) { + throw new N5Exception.N5IOException("FileLazyRead is already closed."); + } + return FileSystemKeyValueAccess.size(path); + } + + @Override + public ReadData materialize(final long offset, final long length) { + + if (lock == null) { + throw new N5Exception.N5IOException("FileLazyRead is already closed."); + } + + try (final FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { + + channel.position(offset); + + final long channelSize = channel.size(); + if (!validBounds(channelSize, offset, length)) { + throw new IndexOutOfBoundsException(); + } + + final long size = length < 0 ? (channelSize - offset) : length; + if (size > Integer.MAX_VALUE) { + throw new IndexOutOfBoundsException("Attempt to materialize too large data"); + } + + final byte[] data = new byte[(int) size]; + final ByteBuffer buf = ByteBuffer.wrap(data); + channel.read(buf); + return ReadData.from(data); + + } catch (final NoSuchFileException e) { + throw new N5Exception.N5NoSuchKeyException("No such file", e); + } catch (IOException | UncheckedIOException e) { + throw new N5Exception.N5IOException(e); + } + } + + @Override + public void close() throws IOException { + + if (lock != null) { + lock.close(); + lock = null; + } + } + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java index 49922748a..a97eeab51 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java @@ -28,6 +28,9 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.IOException; +import java.io.UncheckedIOException; + public class N5Exception extends RuntimeException { public N5Exception() { @@ -59,7 +62,7 @@ protected N5Exception( super(message, cause, enableSuppression, writableStackTrace); } - public static class N5IOException extends N5Exception { + public static class N5IOException extends N5Exception{ public N5IOException(final String message) { @@ -178,4 +181,32 @@ protected N5JsonParseException( super(message, cause, enableSuppression, writableStackTrace); } } + + + public static class N5ConcurrentModificationException extends N5IOException { + + public N5ConcurrentModificationException(final String message) { + + super(message); + } + + public N5ConcurrentModificationException(final String message, final Throwable cause) { + + super(message, cause); + } + + public N5ConcurrentModificationException(final Throwable cause) { + + super(cause); + } + + protected N5ConcurrentModificationException( + final String message, + final Throwable cause, + final boolean enableSuppression, + final boolean writableStackTrace) { + + super(message, cause, enableSuppression, writableStackTrace); + } + } } From eb5692e1fac59606dfca7b4589b13db15934b376 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 17 Feb 2026 11:09:49 -0500 Subject: [PATCH 3/4] refactor(test): migrate remaining uses of `KVA#lockForReading/Writing`. Extract and generify TrackingKeyValueAccess to be more reusable Signed-off-by: Caleb Hulbert --- .../saalfeldlab/n5/TrackingN5Writer.java | 49 +++++ .../n5/backward/CompatibilityTest.java | 21 +- .../n5/benchmarks/ReadDataBenchmarks.java | 16 +- .../n5/http/HttpKeyValueAccessTest.java | 18 +- .../n5/kva/DelegateKeyValueAccess.java | 102 +++++++++ .../n5/kva/TrackingKeyValueAccess.java | 59 ++++++ .../saalfeldlab/n5/shard/ShardTest.java | 196 +----------------- 7 files changed, 237 insertions(+), 224 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/TrackingN5Writer.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/kva/DelegateKeyValueAccess.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java diff --git a/src/test/java/org/janelia/saalfeldlab/n5/TrackingN5Writer.java b/src/test/java/org/janelia/saalfeldlab/n5/TrackingN5Writer.java new file mode 100644 index 000000000..19816c2b6 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/TrackingN5Writer.java @@ -0,0 +1,49 @@ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.GsonBuilder; +import org.janelia.saalfeldlab.n5.kva.TrackingKeyValueAccess; + +/** + * An N5Writer that tracks the number of materialize calls performed by + * its underlying key value access. + */ +public class TrackingN5Writer extends N5KeyValueWriter { + + public final TrackingKeyValueAccess tkva; + + public TrackingN5Writer(String basePath, KeyValueAccess kva) { + + super(new TrackingKeyValueAccess(kva), basePath, new GsonBuilder(), false); + this.tkva = (TrackingKeyValueAccess) getKeyValueAccess(); + } + + public void resetNumMaterializeCalls() { + tkva.numMaterializeCalls = 0; + } + + public int getNumMaterializeCalls() { + return tkva.numMaterializeCalls; + } + + public void resetNumIsFileCalls() { + tkva.numIsFileCalls = 0; + } + + public int getNumIsFileCalls() { + return tkva.numIsFileCalls; + } + + public void resetTotalBytesRead() { + tkva.totalBytesRead = 0; + } + + public long getTotalBytesRead() { + return tkva.totalBytesRead; + } + + public void resetAllTracking() { + tkva.numMaterializeCalls = 0; + tkva.numIsFileCalls = 0; + tkva.totalBytesRead = 0; + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java index 0cdb5cd66..bb579c813 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java @@ -40,14 +40,8 @@ import java.nio.file.Files; import java.util.Arrays; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader; -import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; -import org.janelia.saalfeldlab.n5.N5FSReader; -import org.janelia.saalfeldlab.n5.N5FSWriter; -import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.*; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.junit.Test; import com.google.gson.JsonElement; @@ -148,13 +142,10 @@ public void testBlockData() throws IOException { private byte[] read(KeyValueAccess kva, String path) { - int N = (int)kva.size(path); - byte[] data = new byte[N]; - try (LockedChannel ch = kva.lockForReading(path); - InputStream is = ch.newInputStream();) { - - is.read(data); - } catch (IOException e) { + byte[] data; + try (VolatileReadData readData = kva.createReadData(path)) { + data = readData.allBytes(); + } catch (N5Exception.N5IOException e) { return null; } return data; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java index 19254700d..4ce148c50 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java @@ -41,6 +41,8 @@ import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -125,14 +127,16 @@ protected void write(Path path, int numBytes) { final byte[] data = new byte[numBytes]; random.nextBytes(data); - System.out.println(path.toAbsolutePath().toString()); + System.out.println(path.toAbsolutePath()); System.out.println(numBytes); - try (final LockedChannel ch = kva.lockForWriting(path.toAbsolutePath().toString())) { - final OutputStream os = ch.newOutputStream(); + + ReadData readData = ReadData.from(os -> { os.write(data); - os.flush(); - os.close(); - } catch (final IOException e) { + }).materialize(); + + try { + kva.write(path.toAbsolutePath().toString(), readData); + } catch (N5Exception.N5IOException e) { e.printStackTrace(); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java index d8df4ab44..8572e1cee 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java @@ -32,12 +32,15 @@ import org.janelia.saalfeldlab.n5.HttpKeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.junit.Test; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.nio.charset.Charset; +import java.util.function.Function; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; @@ -58,16 +61,9 @@ public void testExistsRead() { final String absolutePath = kva.compose(baseUrl, key); assumeTrue(kva.exists(absolutePath)); - try (LockedChannel ch = kva.lockForReading(absolutePath)) { - - final InputStream is = ch.newInputStream(); - - final String attributes = IOUtils.toString(is, Charset.defaultCharset()); - assertEquals(expectedAttributes, attributes); - - is.close(); - - } catch (IOException e) { + try (VolatileReadData data = kva.createReadData(absolutePath)) { + IOUtils.toString(data.inputStream(), Charset.defaultCharset()); + } catch (IOException e) { // not correct to fail for an IO exception e.printStackTrace(); } @@ -78,7 +74,7 @@ public void testUnsupportedOperations() { final HttpKeyValueAccess kva = new HttpKeyValueAccess(); assertThrows(N5Exception.class, () -> kva.delete("foo")); - assertThrows(N5Exception.class, () -> kva.lockForWriting("bar")); + assertThrows(N5Exception.class, () -> kva.write("bar", ReadData.from(os -> {}))); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/DelegateKeyValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/DelegateKeyValueAccess.java new file mode 100644 index 000000000..b4ae4b738 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/DelegateKeyValueAccess.java @@ -0,0 +1,102 @@ +package org.janelia.saalfeldlab.n5.kva; + +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; + +import java.net.URI; +import java.net.URISyntaxException; + +public class DelegateKeyValueAccess implements KeyValueAccess { + + protected final KeyValueAccess kva; + + public DelegateKeyValueAccess(KeyValueAccess kva) { this.kva = kva; } + + @Override + public String[] components(String path) { + return kva.components(path); + } + + @Override + public String compose(URI uri, String... components) { + return kva.compose(uri, components); + } + + @Override + public String compose(String... components) { + return kva.compose(components); + } + + @Override + public String parent(String path) { + return kva.parent(path); + } + + @Override + public String relativize(String path, String base) { + return kva.relativize(path, base); + } + + @Override + public String normalize(String path) { + return kva.normalize(path); + } + + @Override + public URI uri(String uriString) throws URISyntaxException { + return kva.uri(uriString); + } + + @Override + public boolean exists(String normalPath) { + return kva.exists(normalPath); + } + + @Override + public long size(String normalPath) throws N5Exception.N5NoSuchKeyException { + return kva.size(normalPath); + } + + @Override + public boolean isDirectory(String normalPath) { + return kva.isDirectory(normalPath); + } + + @Override + public boolean isFile(String normalPath) { + return kva.isFile(normalPath); + } + + @Override + public VolatileReadData createReadData(String normalPath) throws N5Exception.N5IOException { + return kva.createReadData(normalPath); + } + + @Override + public void write(String normalPath, ReadData data) throws N5Exception.N5IOException { + kva.write( normalPath, data); + } + + @Override + public String[] listDirectories(String normalPath) throws N5Exception.N5IOException { + return kva.listDirectories(normalPath); + } + + @Override + public String[] list(String normalPath) throws N5Exception.N5IOException { + return kva.list(normalPath); + } + + @Override + public void createDirectories(String normalPath) throws N5Exception.N5IOException { + kva.createDirectories(normalPath); + + } + + @Override + public void delete(String normalPath) throws N5Exception.N5IOException { + kva.delete(normalPath); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java new file mode 100644 index 000000000..0c4f4a876 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java @@ -0,0 +1,59 @@ +package org.janelia.saalfeldlab.n5.kva; + +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; +import org.janelia.saalfeldlab.n5.shard.ShardTest; + +public class TrackingKeyValueAccess extends DelegateKeyValueAccess { + + public int numMaterializeCalls = 0; + public int numIsFileCalls = 0; + public long totalBytesRead = 0; + + public TrackingKeyValueAccess(final KeyValueAccess kva) { + super(kva); + } + + @Override + public boolean isFile(String normalPath) { + numIsFileCalls++; + return kva.isFile(normalPath); + } + + @Override + public VolatileReadData createReadData(final String normalPath) { +// throw new N5NoSuchKeyException("Test No Such Key"); + return VolatileReadData.from(new TrackingVolatileReadData(kva.createReadData(normalPath))); + } + + private class TrackingVolatileReadData implements LazyRead { + + private final VolatileReadData readData; + + TrackingVolatileReadData(final VolatileReadData readData) { + this.readData = readData; + } + + @Override + public long size() throws N5Exception.N5IOException { + + return readData.requireLength(); + } + + @Override + public ReadData materialize(final long offset, final long length) { + + numMaterializeCalls++; + return readData.slice(offset, length).materialize(); + } + + @Override + public void close() { + + readData.close(); + } + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 97d45cc6c..d881df45f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -28,15 +28,10 @@ */ package org.janelia.saalfeldlab.n5.shard; -import com.google.gson.GsonBuilder; import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; import java.net.URI; import java.net.URISyntaxException; -import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.channels.FileChannel; import java.nio.file.*; import java.util.ArrayList; import java.util.Arrays; @@ -49,27 +44,11 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; -import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; -import org.janelia.saalfeldlab.n5.IntArrayDataBlock; -import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedFileChannel; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; -import org.janelia.saalfeldlab.n5.N5FSTest; -import org.janelia.saalfeldlab.n5.N5KeyValueWriter; -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.RawCompression; + +import org.janelia.saalfeldlab.n5.*; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.readdata.LazyRead; -import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; import org.junit.After; import org.junit.Assert; @@ -92,7 +71,7 @@ public class ShardTest { @Override public N5Writer createTempN5Writer() { if (LOCAL_DEBUG) { - final N5Writer writer = new TrackingN5Writer("src/test/resources/test.n5"); + final N5Writer writer = new TrackingN5Writer("src/test/resources/test.n5", new FileSystemKeyValueAccess()); writer.remove(""); // Clear old when starting new test return writer; } @@ -100,7 +79,7 @@ public class ShardTest { final String basePath = new File(tempN5PathName()).toURI().normalize().getPath(); try { String uri = new URI("file", null, basePath, null).toString(); - return new TrackingN5Writer(uri); + return new TrackingN5Writer(uri, new FileSystemKeyValueAccess()); } catch (URISyntaxException e) { e.printStackTrace(); } @@ -677,171 +656,4 @@ private int[] range(int N) { return IntStream.range(0, N).toArray(); } - /** - * An N5Writer that tracks the number of materialize calls performed by - * its underlying key value access. - */ - public static class TrackingN5Writer extends N5KeyValueWriter { - - final TrackingFileSystemKeyValueAccess tkva; - public TrackingN5Writer(String basePath) { - - super( new TrackingFileSystemKeyValueAccess(), basePath, new GsonBuilder(), false); - tkva = (TrackingFileSystemKeyValueAccess)getKeyValueAccess(); - } - - public void resetNumMaterializeCalls() { - tkva.numMaterializeCalls = 0; - } - - public int getNumMaterializeCalls() { - return tkva.numMaterializeCalls; - } - - public void resetNumIsFileCalls() { - tkva.numIsFileCalls = 0; - } - - public int getNumIsFileCalls() { - return tkva.numIsFileCalls; - } - - public void resetTotalBytesRead() { - tkva.totalBytesRead = 0; - } - - public long getTotalBytesRead() { - return tkva.totalBytesRead; - } - - public void resetAllTracking() { - tkva.numMaterializeCalls = 0; - tkva.numIsFileCalls = 0; - tkva.totalBytesRead = 0; - } - } - - private static class TrackingFileSystemKeyValueAccess extends FileSystemKeyValueAccess { - - private int numMaterializeCalls = 0; - private int numIsFileCalls = 0; - private long totalBytesRead = 0; - - protected TrackingFileSystemKeyValueAccess() { - super(); - } - - @Override - public boolean isFile(String normalPath) { - numIsFileCalls++; - return super.isFile(normalPath); - } - - @Override - public VolatileReadData createReadData(final String normalPath) { - try { - return VolatileReadData.from(new TrackingFileLazyRead(Paths.get(normalPath))); - } catch (N5NoSuchKeyException e) { -// return VolatileReadData.from(new NoSuchKeyLazyRead()); - return null; - } - } - - // This can be used in createReadData() above, to also simulate the case that we will have for - // cloud storage KVAs, where the returned VolatileReadData is non-null but will fail on the first - // operation that queries the cloud backend. - private class NoSuchKeyLazyRead implements LazyRead { - - @Override - public ReadData materialize(final long offset, final long length) throws N5Exception.N5IOException { - throw new N5NoSuchKeyException("NoSuchKeyLazyRead"); - } - - @Override - public long size() throws N5Exception.N5IOException { - throw new N5NoSuchKeyException("NoSuchKeyLazyRead"); - } - - @Override - public void close() { - } - } - - private class TrackingFileLazyRead implements LazyRead { - - private final Path path; - private LockedFileChannel lock; - - TrackingFileLazyRead(final Path path) { - this.path = path; - lock = FileSystemKeyValueAccess.lockForReading(path); - } - - @Override - public long size() throws N5Exception.N5IOException { - - if (lock == null) { - throw new N5Exception.N5IOException("FileLazyRead is already closed."); - } - return FileSystemKeyValueAccess.size(path); - } - - @Override - public ReadData materialize(final long offset, final long length) { - - if (lock == null) { - throw new N5Exception.N5IOException("FileLazyRead is already closed."); - } - - numMaterializeCalls++; - try (final FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { - - channel.position(offset); - - final long channelSize = channel.size(); - if (!validBounds(channelSize, offset, length)) { - throw new IndexOutOfBoundsException(); - } - - final long size = length < 0 ? (channelSize - offset) : length; - if (size > Integer.MAX_VALUE) { - throw new IndexOutOfBoundsException("Attempt to materialize too large data"); - } - - final byte[] data = new byte[(int) size]; - totalBytesRead += size; - final ByteBuffer buf = ByteBuffer.wrap(data); - channel.read(buf); - return ReadData.from(data); - - } catch (final NoSuchFileException e) { - throw new N5NoSuchKeyException("No such file", e); - } catch (IOException | UncheckedIOException e) { - throw new N5Exception.N5IOException(e); - } - } - - @Override - public void close() throws IOException { - - if (lock != null) { - lock.close(); - lock = null; - } - } - } - - private static boolean validBounds(long channelSize, long offset, long length) { - - if (offset < 0) - return false; - else if (channelSize > 0 && offset >= channelSize) // offset == 0 and arrayLength == 0 is okay - return false; - else if (length >= 0 && offset + length > channelSize) - return false; - - return true; - } - } - } \ No newline at end of file From 79c60106e69e2a520bcbea86e9b72739ed089349 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 25 Feb 2026 11:15:41 -0500 Subject: [PATCH 4/4] refactor: use `strict`, `permissive`, `unsafe` as more generic terms for the IoPolicy configuration. Signed-off-by: Caleb Hulbert --- .../org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index ec58d6870..83bb1c5ec 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -62,11 +62,11 @@ private static IoPolicy getIoPolicy() { return FsIoPolicy.atomicWithFallback; switch (property) { - case "atomic": + case "strict": return new FsIoPolicy.Atomic(); case "unsafe": return new FsIoPolicy.Unsafe(); - case "atomicFallbackUnsafe": + case "permissive": default: return FsIoPolicy.atomicWithFallback; }