From 8134a15abcafaa70601af5726e9075e7c2514bdc Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 2 Feb 2026 20:44:22 +0100 Subject: [PATCH 01/21] Rename LazyReadData to LazyGeneratedReadData ...because we will need the LazyReadData name for the ReadData that wraps LazyRead. --- ...ReadData.java => LazyGeneratedReadData.java} | 14 +++++++------- .../saalfeldlab/n5/readdata/ReadData.java | 17 ++++++++++++----- .../saalfeldlab/n5/shard/RawShardCodec.java | 2 +- .../saalfeldlab/n5/readdata/ReadDataTests.java | 2 +- 4 files changed, 21 insertions(+), 14 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/readdata/{LazyReadData.java => LazyGeneratedReadData.java} (92%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyGeneratedReadData.java similarity index 92% rename from src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java rename to src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyGeneratedReadData.java index 6f940eea1..bdfdac7d8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyGeneratedReadData.java @@ -36,10 +36,10 @@ import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -class LazyReadData implements ReadData { +class LazyGeneratedReadData implements ReadData { - LazyReadData(final OutputStreamWriter writer) { - this.writer = writer; + LazyGeneratedReadData(final ReadData.Generator generator) { + this.generator = generator; } /** @@ -51,7 +51,7 @@ class LazyReadData implements ReadData { * @param encoder * OutputStreamOperator to use for encoding */ - LazyReadData(final ReadData data, final OutputStreamOperator encoder) { + LazyGeneratedReadData(final ReadData data, final OutputStreamOperator encoder) { this(outputStream -> { try (final OutputStream deflater = encoder.apply(interceptClose.apply(outputStream))) { data.writeTo(deflater); @@ -59,7 +59,7 @@ class LazyReadData implements ReadData { }); } - private final OutputStreamWriter writer; + private final ReadData.Generator generator; private ByteArrayReadData bytes; @@ -70,7 +70,7 @@ public ReadData materialize() throws N5IOException { if (bytes == null) { try { final ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); - writer.writeTo(baos); + generator.writeTo(baos); bytes = new ByteArrayReadData(baos.toByteArray()); } catch (IOException e) { throw new N5IOException(e); @@ -119,7 +119,7 @@ public void writeTo(final OutputStream outputStream) throws N5IOException, Illeg outputStream.write(bytes.allBytes()); } else { final CountingOutputStream cos = new CountingOutputStream(outputStream); - writer.writeTo(cos); + generator.writeTo(cos); length = cos.getByteCount(); } } catch (IOException e) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 27d4bd04b..68b64ff37 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -250,7 +250,7 @@ default void prefetch(final Collection ranges) throws N5IOExcep * @return encoded ReadData */ default ReadData encode(OutputStreamOperator encoder) { - return new LazyReadData(this, encoder); + return new LazyGeneratedReadData(this, encoder); } /** @@ -347,21 +347,28 @@ static ReadData from(final ByteBuffer data) { } } + /** + * Generator supplies the content of a ReadData by writing it to an {@link + * OutputStream} on demand. + *

+ * An example of a {@code Generator} is applying a compressor (an {@link + * OutputStreamOperator}) to an existing ReadData. + */ @FunctionalInterface - interface OutputStreamWriter { + interface Generator { void writeTo(OutputStream outputStream) throws IOException, IllegalStateException; } /** - * Create a new {@code ReadData} that is lazily generated by the given {@link OutputStreamWriter}. + * Create a new {@code ReadData} that is lazily generated by the given {@link Generator}. * * @param generator * generates the data * * @return a new ReadData */ - static ReadData from(OutputStreamWriter generator) { - return new LazyReadData(generator); + static ReadData from(Generator generator) { + return new LazyGeneratedReadData(generator); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java index c277fcea3..e23640333 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java @@ -76,7 +76,7 @@ public ReadData encode(final DataBlock shard) throws N5Exception.N5IOE } final SegmentedReadData data = SegmentedReadData.concatenate(readDatas); - final ReadData.OutputStreamWriter writer; + final ReadData.Generator writer; if (indexLocation == START) { data.materialize(); final NDArray locations = ShardIndex.locations(index, data); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index 1c2358cda..cf7322bd5 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -60,7 +60,7 @@ public void testLazyReadData() throws IOException { final ReadData readData = ReadData.from(out -> { out.write(data); }); - assertTrue(readData instanceof LazyReadData); + assertTrue(readData instanceof LazyGeneratedReadData); readDataTestHelper(readData, -1, N); sliceTestHelper(readData, N); From 058bfafd20b1968c9fadb24374d159813128b5fb Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 2 Feb 2026 20:54:35 +0100 Subject: [PATCH 02/21] fix typos --- .../n5/readdata/segment/ConcatenatedReadData.java | 2 +- .../n5/readdata/segment/DefaultSegmentedReadData.java | 2 +- .../java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java | 2 +- .../janelia/saalfeldlab/n5/shard/DatasetAccessTest.java | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadData.java index 76219df65..2d9357c22 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadData.java @@ -50,7 +50,7 @@ * In particular, it is also possible to concatenate {@code SegmentedReadData}s * with (yet) unknown length. (This is useful for postponing compression of * DataBlocks until they are actually written.) In that case, segment locations - * are only available, after all lengths become known. This happens when this + * are only available after all lengths become known. This happens when this * {@code ConcatenatedReadData} (or all its constituents) is * {@link #materialize() materialized} or {@link #writeTo(OutputStream) written}. */ diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java index 13284c878..22c940904 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java @@ -64,7 +64,7 @@ class DefaultSegmentedReadData implements SegmentedReadData { private final ReadData segmentSource; /** - * The offset of {@code delegate} withr espect to {@code segmentSource}. + * The offset of {@code delegate} with respect to {@code segmentSource}. */ private final long offset; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 05b008147..97863694e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -200,7 +200,7 @@ static int[] blockSizeFromIndexSize(final int[] indexSize) { } /** - * Strips first element (should be {@code LONGS_PER_BLOCK} from the {@code blockSize} array. + * Strips first element (should be {@code LONGS_PER_BLOCK}) from the {@code blockSize} array. */ static int[] indexSizeFromBlockSize(final int[] blockSize) { assert blockSize[ 0 ] == LONGS_PER_BLOCK; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java index 382728854..f7a6fa339 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java @@ -94,7 +94,7 @@ public void setup() { @Test - public void tesWriteReadIndividual() { + public void testWriteReadIndividual() { final PositionValueAccess store = new TestPositionValueAccess(); @@ -116,7 +116,7 @@ public void tesWriteReadIndividual() { } @Test - public void tesWriteReadBulk() { + public void testWriteReadBulk() { final PositionValueAccess store = new TestPositionValueAccess(); @@ -144,7 +144,7 @@ public void tesWriteReadBulk() { } @Test - public void tesDeleteBlock() { + public void testDeleteBlock() { final PositionValueAccess store = new TestPositionValueAccess(); @@ -173,7 +173,7 @@ public void tesDeleteBlock() { } @Test - public void tesDeleteBlocks() { + public void testDeleteBlocks() { final PositionValueAccess store = new TestPositionValueAccess(); From 95d70d9caab677ea19ce77538768732ad9e19f55 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 2 Feb 2026 21:15:06 +0100 Subject: [PATCH 03/21] Add VolatileReadData and let KVA.createReadData return it VolatileReadData is Closable. LazyRead is now also Closable. The VolatileReadData implementation LazyReadData closes its delegate LazyRead when it is closed. The file-system FileLazyRead locks the path it refers to on construction and holds the lock until it is closed (via a wrapping LazyReadData). All usages of KVA.createReadData have been revised to use try-with-resources to make sure VolatileReadData are properly closed. DefaultDatasetAccess has been revised to not eagerly materialize ReadData. This is in preparation of using partial loading and prefetching. (This is a separate topic, not necessary for using VolatileReadData correctly, but was too difficult to disentangle in the git history after the fact...) --- .../n5/FileSystemKeyValueAccess.java | 114 ++++---- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 12 +- .../saalfeldlab/n5/HttpKeyValueAccess.java | 10 +- .../saalfeldlab/n5/KeyValueAccess.java | 76 ++---- .../saalfeldlab/n5/readdata/LazyRead.java | 62 +++++ .../LazyReadData.java} | 43 +-- .../n5/readdata/VolatileReadData.java | 35 +++ .../n5/shard/DefaultDatasetAccess.java | 251 +++++++++++------- .../n5/shard/PositionValueAccess.java | 17 +- .../n5/benchmarks/ReadDataBenchmarks.java | 8 +- .../ReadDataBenchmarksKvaReadData.java | 71 ----- .../n5/readdata/ReadDataTests.java | 11 +- .../n5/shard/DatasetAccessTest.java | 23 +- .../saalfeldlab/n5/shard/ShardTest.java | 179 +++++++++---- .../n5/shard/TestPositionValueAccess.java | 84 +++++- 15 files changed, 638 insertions(+), 358 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyRead.java rename src/main/java/org/janelia/saalfeldlab/n5/{KeyValueAccessReadData.java => readdata/LazyReadData.java} (81%) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 378b1466f..6e5f8ce6a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -28,6 +28,7 @@ */ package org.janelia.saalfeldlab.n5; +import java.nio.file.StandardOpenOption; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; @@ -53,6 +54,8 @@ import java.util.stream.Stream; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import static org.janelia.saalfeldlab.n5.FileKeyLockManager.FILE_LOCK_MANAGER; @@ -78,35 +81,27 @@ public FileSystemKeyValueAccess(final FileSystem fileSystem) { } @Override - public ReadData createReadData(final String normalPath) { - return new KeyValueAccessReadData(new FileLazyRead(normalPath)); + public VolatileReadData createReadData(final String normalPath) { + try { + return VolatileReadData.from(new FileLazyRead(fileSystem.getPath(normalPath))); + } catch (N5NoSuchKeyException e) { + return null; + } } @Override public LockedFileChannel lockForReading(final String normalPath) throws N5IOException { - try { - return FILE_LOCK_MANAGER.lockForReading(fileSystem.getPath(normalPath)); - } catch (final NoSuchFileException e) { - throw new N5NoSuchKeyException("No such file", e); - } catch (IOException | UncheckedIOException e) { - throw new N5IOException("Failed to lock file for reading: " + normalPath, e); - } + return lockForReading(fileSystem.getPath(normalPath)); } @Override public LockedChannel lockForWriting(final String normalPath) throws N5IOException { - try { - return FILE_LOCK_MANAGER.lockForWriting(fileSystem.getPath(normalPath)); - } catch (final NoSuchFileException e) { - throw new N5NoSuchKeyException("No such file", e); - } catch (IOException | UncheckedIOException e) { - throw new N5IOException("Failed to lock file for writing: " + normalPath, e); - } + return lockForWriting(fileSystem.getPath(normalPath)); } - public LockedFileChannel lockForReading(final Path path) throws N5IOException { + protected static LockedFileChannel lockForReading(final Path path) throws N5IOException { try { return FILE_LOCK_MANAGER.lockForReading(path); @@ -117,7 +112,7 @@ public LockedFileChannel lockForReading(final Path path) throws N5IOException { } } - public LockedFileChannel lockForWriting(final Path path) throws N5IOException { + static LockedFileChannel lockForWriting(final Path path) throws N5IOException { try { return FILE_LOCK_MANAGER.lockForWriting(path); @@ -152,8 +147,13 @@ public boolean exists(final String normalPath) { @Override public long size(final String normalPath) { + return size(fileSystem.getPath(normalPath)); + } + + protected static long size(final Path path) { + try { - return Files.size(fileSystem.getPath(normalPath)); + return Files.size(path); } catch (NoSuchFileException e) { throw new N5Exception.N5NoSuchKeyException("No such file", e); } catch (IOException | UncheckedIOException e) { @@ -167,9 +167,9 @@ public String[] listDirectories(final String normalPath) throws N5IOException { final Path path = fileSystem.getPath(normalPath); try (final Stream pathStream = Files.list(path)) { return pathStream - .filter(a -> Files.isDirectory(a)) + .filter(Files::isDirectory) .map(a -> path.relativize(a).toString()) - .toArray(n -> new String[n]); + .toArray(String[]::new); } catch (NoSuchFileException e) { throw new N5NoSuchKeyException("No such file", e); } catch (IOException | UncheckedIOException e) { @@ -184,7 +184,7 @@ public String[] list(final String normalPath) throws N5IOException { try (final Stream pathStream = Files.list(path)) { return pathStream .map(a -> path.relativize(a).toString()) - .toArray(n -> new String[n]); + .toArray(String[]::new); } catch (NoSuchFileException e) { throw new N5NoSuchKeyException("No such file", e); } catch (IOException | UncheckedIOException e) { @@ -489,52 +489,74 @@ protected static void createAndCheckIsDirectory( } } - private class FileLazyRead implements LazyRead { + private static class FileLazyRead implements LazyRead { - private final String normalKey; - FileLazyRead(String normalKey) { - this.normalKey = normalKey; - } + private final Path path; + private LockedFileChannel lock; - @Override - public long size() { - return FileSystemKeyValueAccess.this.size(normalKey); + FileLazyRead(final Path path) { + this.path = path; + lock = lockForReading(path); } - @Override + @Override + public long size() throws N5IOException { + + if (lock == null) { + throw new N5IOException("FileLazyRead is already closed."); + } + return FileSystemKeyValueAccess.size(path); + } + + @Override public ReadData materialize(final long offset, final long length) { - try (final LockedFileChannel lfs = lockForReading(fileSystem.getPath(normalKey))) { - final FileChannel channel = lfs.getFileChannel(); - channel.position(offset); - if (length > Integer.MAX_VALUE) - throw new IOException("Attempt to materialize too large data"); + if (lock == null) { + throw new N5IOException("FileLazyRead is already closed."); + } + + try (final FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { + + channel.position(offset); final long channelSize = channel.size(); - if (!validBounds(channelSize, offset, length)) - throw new IndexOutOfBoundsException(); + if (!validBounds(channelSize, offset, length)) { + throw new IndexOutOfBoundsException(); + } + + final long size = length < 0 ? (channelSize - offset) : length; + if (size > Integer.MAX_VALUE) { + throw new IndexOutOfBoundsException("Attempt to materialize too large data"); + } - final int sz = (int) (length < 0 ? channelSize : length); - final byte[] data = new byte[sz]; - final ByteBuffer buf = ByteBuffer.wrap(data); - channel.read(buf); - return ReadData.from(data); + final byte[] data = new byte[(int) size]; + final ByteBuffer buf = ByteBuffer.wrap(data); + channel.read(buf); + return ReadData.from(data); - } catch (final NoSuchFileException e) { + } catch (final NoSuchFileException e) { throw new N5NoSuchKeyException("No such file", e); } catch (IOException | UncheckedIOException e) { - throw new N5Exception.N5IOException(e); + throw new N5IOException(e); } } + @Override + public void close() throws IOException { + + if (lock != null) { + lock.close(); + lock = null; + } + } } private static boolean validBounds(long channelSize, long offset, long length) { if (offset < 0) return false; - else if (channelSize > 0 && offset >= channelSize) // offset == 0 and arrayLength == 0 is okay + else if (channelSize > 0 && offset >= channelSize) // offset == 0 and channelSize == 0 is okay return false; else if (length >= 0 && offset + length > channelSize) return false; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 482cd3667..5b516fef6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -28,13 +28,12 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.io.UncheckedIOException; import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; import com.google.gson.Gson; @@ -82,11 +81,14 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { final String groupPath = N5URI.normalizeGroupPath(pathName); final String attributesPath = absoluteAttributesPath(groupPath); - try ( final InputStream in = getKeyValueAccess().createReadData(attributesPath).inputStream() ) { - return GsonUtils.readAttributes(new InputStreamReader(in), getGson()); + try (final VolatileReadData readData = getKeyValueAccess().createReadData(attributesPath);) { + if (readData == null) { + return null; + } + return GsonUtils.readAttributes(new InputStreamReader(readData.inputStream()), getGson()); } catch (final N5Exception.N5NoSuchKeyException e) { return null; - } catch (final IOException | UncheckedIOException | N5IOException e) { + } catch (final UncheckedIOException | N5IOException e) { throw new N5IOException("Failed to read attributes from dataset " + pathName, e); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index d8c4cffec..7b0b2614f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -50,6 +50,8 @@ import java.nio.channels.NonWritableChannelException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; /** * A read-only {@link KeyValueAccess} implementation using HTTP. As a result, calling lockForWriting, createDirectories, or delete will throw an {@link N5Exception}. @@ -232,8 +234,8 @@ private HttpURLConnection httpRequest(String normalPath, String method) throws I } @Override - public ReadData createReadData(final String normalPath) { - return new KeyValueAccessReadData(new HttpLazyRead(normalPath)); + public VolatileReadData createReadData(final String normalPath) { + return VolatileReadData.from(new HttpLazyRead(normalPath)); } public LockedChannel lockForReading(final String normalPath) throws N5IOException { @@ -459,6 +461,10 @@ public ReadData materialize(long offset, long length) { throw new N5Exception(e); } } + + @Override + public void close() { + } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 832e5368a..014b60d63 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -37,6 +37,7 @@ import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; /** * Key value read primitives used by {@link N5KeyValueReader} @@ -240,18 +241,28 @@ default URI uri(final String uriString) throws URISyntaxException { boolean isFile( String normalPath ); // TODO: Looks un-used. Remove? /** - * Create a {@link ReadData} through which data at the normal key can be read. + * Create a {@link VolatileReadData} through which data at the normal key + * can be read. *

- * Implementations should read lazily if possible. Consumers may call {@link ReadData#materialize()} to force - * a read operation if needed. + * Implementations should read lazily if possible. Consumers may call {@link + * ReadData#materialize()} to force a read operation if needed. *

- * If supported by this KeyValueAccess implementation, partial reads are possible by calling slice on the output {@link ReadData}. + * If supported by this KeyValueAccess implementation, partial reads are + * possible by {@link ReadData#slice slicing} the returned {@code ReadData}. + *

+ * If the requested key does not exist, either {@code null} is returned or a + * lazy {@code VolatileReadData} that will throw {@code N5NoSuchKeyException} + * when trying to materialize. + * + * @param normalPath + * is expected to be in normalized form, no further efforts are made to normalize it * - * @param normalPath is expected to be in normalized form, no further efforts are made to normalize it - * @return a materialized Read data - * @throws N5IOException if an error occurs + * @return a ReadData or null + * + * @throws N5IOException + * if an error occurs */ - ReadData createReadData( final String normalPath ) throws N5IOException; + VolatileReadData createReadData( final String normalPath ) throws N5IOException; /** * Create a lock on a path for reading. This isn't meant to be kept @@ -272,10 +283,9 @@ default URI uri(final String uriString) throws URISyntaxException { LockedChannel lockForReading( final String normalPath ) throws N5IOException; /** - * Create an exclusive lock on a path for writing. If the file doesn't - * exist yet, it will be created, including all directories leading up to - * it. This lock isn't meant to be kept around. Create, use, [auto]close, - * e.g. + * Create an exclusive lock on a path for writing. If the file doesn't exist + * yet, it will be created, including all directories leading up to it. This + * lock isn't meant to be kept around. Create, use, [auto]close, e.g. * * try (final lock = store.lockForWriting()) { * ... @@ -339,46 +349,4 @@ default URI uri(final String uriString) throws URISyntaxException { */ void delete( final String normalPath ) throws N5IOException; - /** - * A lazy reading strategy for lazy, partial reading of data from some source. - *

- * Implementations of this interface handle the specifics of accessing data from - * their respective sources. - * - * @see ReadData - * @see KeyValueAccessReadData - */ - interface LazyRead { - - /** - * Materializes a portion of the data into a concrete {@link ReadData} - * instance. - *

- * This method performs the actual read operation from the underlying - * source, loading only the requested portion of data. The implementation - * should handle bounds checking and throw appropriate exceptions for - * invalid ranges. - * - * @param offset - * the starting position in the data source - * @param length - * the number of bytes to read, or -1 to read from offset to end - * @return a materialized {@link ReadData} instance containing the requested - * data - * @throws N5IOException - * if any I/O error occurs - */ - ReadData materialize(long offset, long length) throws N5IOException; - - /** - * Returns the total size of the data source in bytes. - * - * @return the size of the data source in bytes - * @throws N5IOException - * if an I/O error occurs while trying to get the length - */ - long size() throws N5IOException; - - } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyRead.java new file mode 100644 index 000000000..897831b86 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyRead.java @@ -0,0 +1,62 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.Closeable; +import java.util.Collection; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; + +/** + * A lazy reading strategy for lazy, partial reading of data from some source. + *

+ * Implementations of this interface handle the specifics of accessing data from + * their respective sources. + * + * @see LazyReadData + */ +public interface LazyRead extends Closeable { + + /** + * Materializes a portion of the data into a concrete {@link ReadData} + * instance. + *

+ * This method performs the actual read operation from the underlying + * source, loading only the requested portion of data. The implementation + * should handle bounds checking and throw appropriate exceptions for + * invalid ranges. + * + * @param offset + * the starting position in the data source + * @param length + * the number of bytes to read, or -1 to read from offset to end + * + * @return a materialized {@link ReadData} instance containing the requested + * data + * + * @throws N5IOException + * if any I/O error occurs + */ + ReadData materialize(long offset, long length) throws N5IOException; + + /** + * Returns the total size of the data source in bytes. + * + * @return the size of the data source in bytes + * + * @throws N5IOException + * if an I/O error occurs while trying to get the length + */ + long size() throws N5IOException; + + /** + * Indicates that the given slices will be subsequently read. + * {@code LazyRead} implementations (optionally) may take steps to prepare + * for these subsequent slices. + * + * @param ranges + * slice ranges to prefetch + * + * @throws N5IOException + * if any I/O error occurs + */ + default void prefetch(final Collection ranges) throws N5IOException { + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java similarity index 81% rename from src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessReadData.java rename to src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java index abfd444b1..5a93e4a8d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -26,30 +26,30 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -package org.janelia.saalfeldlab.n5; +package org.janelia.saalfeldlab.n5.readdata; +import java.io.IOException; import java.io.InputStream; - -import org.janelia.saalfeldlab.n5.KeyValueAccess.LazyRead; +import java.util.Collection; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.readdata.ReadData; /** - * A {@link ReadData} implementation that reads from a {@link KeyValueAccess} - * backend through a {@link LazyRead} object. + * A {@link VolatileReadData} that is backed by a {@link LazyRead}. + *

+ * The {@code LazyRead} is closed when this {@code LazyReadData} is closed. */ -public class KeyValueAccessReadData implements ReadData { +class LazyReadData implements VolatileReadData { private final LazyRead lazyRead; private ReadData materialized; private final long offset; private long length; - public KeyValueAccessReadData(final LazyRead lazyRead) { - this(lazyRead, 0, -1); - } + LazyReadData(final LazyRead lazyRead) { + this(lazyRead, 0, -1); + } - KeyValueAccessReadData(final LazyRead lazyRead, final long offset, final long length) { + private LazyReadData(final LazyRead lazyRead, final long offset, final long length) { this.lazyRead = lazyRead; this.offset = offset; this.length = length; @@ -77,7 +77,7 @@ public ReadData materialize() throws N5IOException { * if an I/O error occurs while trying to get the length */ @Override - public ReadData slice(final long offset, final long length) throws N5IOException { + public ReadData slice(final long offset, final long length) { if (offset < 0) throw new IndexOutOfBoundsException("Negative offset: " + offset); @@ -92,7 +92,7 @@ public ReadData slice(final long offset, final long length) throws N5IOException else lengthArg = length; - return new KeyValueAccessReadData(lazyRead, this.offset + offset, lengthArg); + return new LazyReadData(lazyRead, this.offset + offset, lengthArg); } @Override @@ -120,4 +120,17 @@ public long requireLength() throws N5IOException { return length; } + @Override + public void prefetch(final Collection ranges) throws N5IOException { + lazyRead.prefetch(ranges); + } + + @Override + public void close() throws N5IOException { + try { + lazyRead.close(); + } catch (IOException e) { + throw new N5IOException(e); + } + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java new file mode 100644 index 000000000..122a399ef --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java @@ -0,0 +1,35 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.InputStream; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; + +/** + * During its life-time, the content of a {@code VolatileReadData} should not be + * mutated. + *

+ * Implementations can enforce this by + *

+ */ +public interface VolatileReadData extends ReadData, AutoCloseable { + + @Override + void close() throws N5IOException; + + /** + * Create a new {@code VolatileReadData} that loads (lazily) from {@code lazyRead}. + *

+ * The returned {@code VolatileReadData} is responsible for {@link LazyRead#close() closing} + * + * @param lazyRead + * provides data + * + * @return a new VolatileReadData + */ + static VolatileReadData from(final LazyRead lazyRead) { + return new LazyReadData(lazyRead); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 1a2646582..fc1db3609 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -52,6 +52,7 @@ import org.janelia.saalfeldlab.n5.StringDataBlock; import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; import org.janelia.saalfeldlab.n5.util.SubArrayCopy; @@ -76,8 +77,8 @@ public NestedGrid getGrid() { @Override public DataBlock readBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { final NestedPosition position = grid.nestedPosition(gridPosition); - try { - return readBlockRecursive(pva.get(position.key()), position, grid.numLevels() - 1); + try (final VolatileReadData readData = pva.get(position.key())) { + return readBlockRecursive(readData, position, grid.numLevels() - 1); } catch (N5NoSuchKeyException ignored) { return null; } @@ -120,8 +121,13 @@ public List> readBlocks(final PositionValueAccess pva, final List> split = requests.split(); for (final DataBlockRequests subRequests : split) { final long[] key = subRequests.relativeGridPosition(); - final ReadData readData = getExistingReadData(pva, key); - readBlocksRecursive(readData, subRequests); + try (final VolatileReadData readData = pva.get(key)) { + readBlocksRecursive(readData, subRequests); + } catch (N5NoSuchKeyException ignored) { + // the key didn't exist (as we found out when lazy-reading the index). + // we don't have to do anything: all subRequest blocks remain null. + // on to the next shard. + } } return requests.blocks(duplicates); @@ -151,6 +157,10 @@ private void readBlocksRecursive( if (level == 1 ) { //Base case; read the blocks + + // TODO: collect all the elementPos that we will need and prefetch + // Probably best to add a prefetch method to RawShard? + for (final DataBlockRequest request : requests) { final long[] elementPos = request.position.relative(0); final ReadData elementData = shard.getElementData(elementPos); @@ -172,12 +182,22 @@ private void readBlocksRecursive( @Override public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException { - final NestedPosition position = grid.nestedPosition(dataBlock.getGridPosition()); - final long[] key = position.key(); - - final ReadData existingData = getExistingReadData(pva, key); - final ReadData modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1); - pva.set(key, modifiedData); + if (grid.numLevels() == 1) { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[0]; + pva.set(dataBlock.getGridPosition(), codec.encode(dataBlock)); + } else { + final NestedPosition position = grid.nestedPosition(dataBlock.getGridPosition()); + final long[] key = position.key(); + final ReadData modifiedData; + try (final VolatileReadData existingData = pva.get(key)) { + modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1); + // Here, we are about to write the shard data, but with the new block modified. + // Need to make sure that the read operations happen now before pva.set acquires a write lock + modifiedData.materialize(); + } + pva.set(key, modifiedData); + } } private ReadData writeBlockRecursive( @@ -193,9 +213,7 @@ private ReadData writeBlockRecursive( @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[level]; final long[] gridPos = position.absolute(level); - final RawShard shard = existingReadData == null ? - new RawShard(grid.relativeBlockSize(level)) : - codec.decode(existingReadData, gridPos).getData(); + final RawShard shard = getRawShard(existingReadData, codec, gridPos, level); final long[] elementPos = position.relative(level - 1); final ReadData existingElementData = (level == 1) ? null // if level == 1, we don't need to extract the nested (DataBlock) ReadData because it will be overridden anyway @@ -213,21 +231,27 @@ private ReadData writeBlockRecursive( public void writeBlocks(final PositionValueAccess pva, final List> dataBlocks) throws N5IOException { if (grid.numLevels() == 1) { - dataBlocks.forEach(it -> writeBlock(pva, it)); - return; - } - - // Create a list of DataBlockRequests, sorted such that requests from - // the same (nested) shard are grouped contiguously. - final DataBlockRequests requests = createWriteRequests(dataBlocks); - requests.removeDuplicates(); - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { - final boolean writeFully = subRequests.coversShard(); - final long[] shardKey = subRequests.relativeGridPosition(); - final ReadData existingReadData = writeFully ? null : getExistingReadData(pva, shardKey); - final ReadData writeShardReadData = writeBlocksRecursive(existingReadData, subRequests); - pva.set(shardKey, writeShardReadData); + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[0]; + dataBlocks.forEach(dataBlock -> pva.set(dataBlock.getGridPosition(), codec.encode(dataBlock))); + } else { + // Create a list of DataBlockRequests, sorted such that requests from + // the same (nested) shard are grouped contiguously. + final DataBlockRequests requests = createWriteRequests(dataBlocks); + requests.removeDuplicates(); + final List> split = requests.split(); + for (final DataBlockRequests subRequests : split) { + final boolean writeFully = subRequests.coversShard(); + final long[] shardKey = subRequests.relativeGridPosition(); + final ReadData modifiedData; + try (final VolatileReadData existingData = writeFully ? null : pva.get(shardKey)) { + modifiedData = writeBlocksRecursive(existingData, subRequests); + // Here, we are about to write the shard data, but with the new blocks modified. + // Need to make sure that the read operations happen now before pva.set acquires a write lock + modifiedData.materialize(); + } + pva.set(shardKey, modifiedData); + } } } @@ -250,9 +274,7 @@ private ReadData writeBlocksRecursive( @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[level]; final long[] gridPos = requests.gridPosition(); - final RawShard shard = writeFully ? - new RawShard(grid.relativeBlockSize(level)) : - codec.decode(existingReadData, gridPos).getData(); + final RawShard shard = getRawShard(existingReadData, codec, gridPos, level); if ( level == 1 ) { // Base case, write the blocks @@ -292,8 +314,15 @@ public void writeRegion( for (long[] key : Region.gridPositions(region.minPos().key(), region.maxPos().key())) { final NestedPosition pos = grid.nestedPosition(key, grid.numLevels() - 1); final boolean nestedWriteFully = writeFully || region.fullyContains(pos); - final ReadData existingData = nestedWriteFully ? null : getExistingReadData(pva, key); - final ReadData modifiedData = writeRegionRecursive(existingData, region, blocks, pos); + final ReadData modifiedData; + try (final VolatileReadData existingData = nestedWriteFully ? null : pva.get(key)) { + modifiedData = writeRegionRecursive(existingData, region, blocks, pos); + // Here, we are about to write the shard data, but with the new block modified. + // Need to make sure that the read operations happen now before pva.set acquires a write lock + if (existingData != null && modifiedData != null) { + modifiedData.materialize(); + } + } pva.set(key, modifiedData); } } @@ -313,8 +342,15 @@ public void writeRegion( exec.submit(() -> { final NestedPosition pos = grid.nestedPosition(key, grid.numLevels() - 1); final boolean nestedWriteFully = writeFully || region.fullyContains(pos); - final ReadData existingData = nestedWriteFully ? null : getExistingReadData(pva, key); - final ReadData modifiedData = writeRegionRecursive(existingData, region, blocks, pos); + final ReadData modifiedData; + try (final VolatileReadData existingData = nestedWriteFully ? null : pva.get(key)) { + modifiedData = writeRegionRecursive(existingData, region, blocks, pos); + // Here, we are about to write the shard data, but with the new block modified. + // Need to make sure that the read operations happen now before pva.set acquires a write lock + if (existingData != null && modifiedData != null) { + modifiedData.materialize(); + } + } pva.set(key, modifiedData); }); } @@ -335,11 +371,17 @@ private ReadData writeRegionRecursive( final long[] gridPosition = position.absolute(0); // If the DataBlock is not fully contained in the region, we will - // get existingReadData != null. In that case, we decode the + // get existingReadData != null. In that case, we try to decode the // existing DataBlock and pass it to the BlockSupplier for modification. - final DataBlock existingDataBlock = (existingReadData == null) - ? null - : codec.decode(existingReadData, gridPosition); + // (This might fail with N5NoSuchKeyException if existingReadData + // lazily points to non-existent data.) + DataBlock existingDataBlock = null; + if (existingReadData != null) { + try { + existingDataBlock = codec.decode(existingReadData, gridPosition); + } catch (N5NoSuchKeyException ignored) { + } + } final DataBlock dataBlock = blocks.get(gridPosition, existingDataBlock); // null blocks may be provided when they contain only the fill value @@ -353,9 +395,7 @@ private ReadData writeRegionRecursive( @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[level]; final long[] gridPos = position.absolute(level); - final RawShard shard = existingReadData == null ? - new RawShard(grid.relativeBlockSize(level)) : - codec.decode(existingReadData, gridPos).getData(); + final RawShard shard = getRawShard(existingReadData, codec, gridPos, level); for (NestedPosition pos : region.containedNestedPositions(position)) { final boolean nestedWriteFully = writeFully || region.fullyContains(pos); final long[] elementPos = pos.relative(); @@ -377,25 +417,33 @@ private ReadData writeRegionRecursive( @Override public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { - final NestedPosition position = grid.nestedPosition(gridPosition); - final long[] key = position.key(); if (grid.numLevels() == 1) { // for non-sharded dataset, don't bother getting the value, just remove the key. - try { - return pva.remove(key); - } catch (final Exception e) { - throw new N5Exception("The shard at " + Arrays.toString(key) + " could not be deleted.", e); - } + return pva.remove(gridPosition); } else { - final ReadData existingData = pva.get(key); // TODO: use getExistingReadData() instead !? - final ReadData modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); - if (existingData != null && modifiedData == null) { + final NestedPosition position = grid.nestedPosition(gridPosition); + final long[] key = position.key(); + final ReadData modifiedData; + try (final VolatileReadData existingData = pva.get(key)) { + modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); + if (modifiedData == existingData) { + // nothing changed, the blocks we wanted to delete didn't exist anyway + return false; + } else if (modifiedData != null) { + // Here, we are about to write the shard data, but without the block to be deleted. + // Need to make sure that the read operations happen now before pva.set acquires a write lock + modifiedData.materialize(); + } + } catch (final N5NoSuchKeyException e) { + // the key didn't exist (as we found out when lazy-reading the index) + // so nothing changed, the blocks we wanted to delete didn't exist anyway + return false; + } + if (modifiedData == null) { return pva.remove(key); - } else if (modifiedData != existingData) { + } else { pva.set(key, modifiedData); return true; - } else { - return false; } } } @@ -403,7 +451,7 @@ public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosit private ReadData deleteBlockRecursive( final ReadData existingReadData, final NestedPosition position, - final int level) { + final int level) throws N5NoSuchKeyException { if (level == 0 || existingReadData == null) { return null; } else { @@ -444,37 +492,51 @@ private ReadData deleteBlockRecursive( @Override public boolean deleteBlocks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { - boolean deleted = false; - // for non-sharded datasets, just delete the blocks individually if (grid.numLevels() == 1) { + boolean deleted = false; for (long[] pos : gridPositions) { - deleted |= deleteBlock(pva, pos); + deleted |= pva.remove(pos); } return deleted; - } + } else { - // Create a list of DataBlockRequests and sort it such that requests - // from the same (nested) shard are grouped contiguously. - // Despite the name, createReadRequests() works for delete requests as well ... - final DataBlockRequests requests = createReadRequests(gridPositions); - requests.removeDuplicates(); + // Create a list of DataBlockRequests and sort it such that requests + // from the same (nested) shard are grouped contiguously. + // Despite the name, createReadRequests() works for delete requests as well ... + final DataBlockRequests requests = createReadRequests(gridPositions); + requests.removeDuplicates(); - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { - final boolean writeFully = subRequests.coversShard(); - final long[] key = subRequests.relativeGridPosition(); - final ReadData existingData = writeFully ? null : getExistingReadData(pva, key); - final ReadData modifiedData = deleteBlocksRecursive(existingData, subRequests); - if (existingData != null && modifiedData == null) { - deleted |= pva.remove(key); - } else if (modifiedData != existingData) { - pva.set(key, modifiedData); - deleted = true; + boolean deleted = false; + final List> split = requests.split(); + for (final DataBlockRequests subRequests : split) { + final boolean writeFully = subRequests.coversShard(); + final long[] key = subRequests.relativeGridPosition(); + final ReadData modifiedData; + try (final VolatileReadData existingData = writeFully ? null : pva.get(key)) { + modifiedData = deleteBlocksRecursive(existingData, subRequests);; + if (modifiedData == existingData) { + // nothing changed, the blocks we wanted to delete didn't exist anyway + continue; + } else if (existingData != null && modifiedData != null) { + // Here, we are about to write the shard data, but without the block to be deleted. + // Need to make sure that the read operations happen now before pva.set acquires a write lock + modifiedData.materialize(); + } + } catch (final N5NoSuchKeyException e) { + // the key didn't exist (as we found out when lazy-reading the index) + // so nothing changed, the blocks we wanted to delete didn't exist anyway + continue; + } + if (modifiedData == null) { + deleted |= pva.remove(key); + } else { + pva.set(key, modifiedData); + deleted = true; + } } + return deleted; } - - return deleted; } /** @@ -736,18 +798,29 @@ public void writeShard( // // -- helpers ------------------------------------------------------------- - private static ReadData getExistingReadData(final PositionValueAccess pva, final long[] key) { - // need to read the shard anyway, and currently (Sept 24 2025) - // have no way to tell if the key exists from what is in this method except to attempt - // to materialize and catch the N5NoSuchKeyException - try { - ReadData existingData = pva.get(key); - if (existingData != null) - existingData.materialize(); - return existingData; - } catch (N5NoSuchKeyException e) { - return null; + /** + * If {@code existingReadData != null} try to decode it into a RawShard. + * Otherwise, or if this fails because we find that {@code existingReadData} + * lazily points to non-existent data, return a new empty RawShard. + * + * @param existingReadData data to decode or null + * @param codec shard codec + * @param gridPos position of the shard on the shard grid of the given level + * @param level level of the shard + * @return the decode shard (or a new empty shard) + */ + private RawShard getRawShard( + final ReadData existingReadData, + final BlockCodec codec, + final long[] gridPos, + final int level) { + if (existingReadData != null) { + try { + return codec.decode(existingReadData, gridPos).getData(); + } catch (N5NoSuchKeyException ignored) { + } } + return new RawShard(grid.relativeBlockSize(level)); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java index 5c69ad1a8..d81d2156e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java @@ -38,6 +38,7 @@ import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; /** * Wrap a KeyValueAccess and a dataset URI to be able to get/set values (ReadData) by {@code long[]} key @@ -46,9 +47,13 @@ public interface PositionValueAccess { /** - * Gets the {@link ReadData} for the DataBlock (or shard) at the given - * position in the block (or shard) grid. - * + * Gets the {@link VolatileReadData} for the DataBlock (or shard) at the + * given position in the block (or shard) grid. + *

+ * If the requested key does not exist, either {@code null} is returned or a + * lazy {@code VolatileReadData} that will throw {@code N5NoSuchKeyException} + * when trying to materialize. + * * @param key * The position of the data block or shard * @return ReadData for the given key or {@code null} if the key doesn't @@ -56,7 +61,7 @@ public interface PositionValueAccess { * @throws N5Exception.N5IOException * if an error occurs while reading */ - ReadData get(long[] key) throws N5Exception.N5IOException; + VolatileReadData get(long[] key) throws N5Exception.N5IOException; void set(long[] key, ReadData data) throws N5Exception.N5IOException; @@ -64,7 +69,7 @@ public interface PositionValueAccess { boolean remove(long[] key) throws N5Exception.N5IOException; - public static PositionValueAccess fromKva( + static PositionValueAccess fromKva( final KeyValueAccess kva, final URI uri, final String normalPath, @@ -104,7 +109,7 @@ protected String absolutePath( final long... gridPosition) { } @Override - public ReadData get(long[] key) throws N5IOException { + public VolatileReadData get(long[] key) throws N5IOException { return kva.createReadData(absolutePath(key)); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java index fe47a9550..9dcc81193 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java @@ -41,7 +41,7 @@ import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; -import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -90,10 +90,12 @@ public static void main(String... args) throws RunnerException { @Benchmark public void run(Blackhole hole) throws IOException { - hole.consume(read().materialize()); + try (final VolatileReadData read = read()) { + hole.consume(read.materialize()); + } } - public ReadData read() throws IOException { + public VolatileReadData read() throws IOException { return kva.createReadData(getPath().toString()); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java deleted file mode 100644 index 5a06aa8a6..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java +++ /dev/null @@ -1,71 +0,0 @@ -/*- - * #%L - * Not HDF5 - * %% - * Copyright (C) 2017 - 2025 Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.benchmarks; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; - -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.runner.Runner; -import org.openjdk.jmh.runner.RunnerException; -import org.openjdk.jmh.runner.options.Options; -import org.openjdk.jmh.runner.options.OptionsBuilder; - -@State(Scope.Benchmark) -@Warmup(iterations = 10, time = 100, timeUnit = TimeUnit.MICROSECONDS) -@Measurement(iterations = 100, time = 100, timeUnit = TimeUnit.MICROSECONDS) -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.MICROSECONDS) -@Fork(1) -public class ReadDataBenchmarksKvaReadData extends ReadDataBenchmarks { - - public static void main(String... args) throws RunnerException { - - final Options options = new OptionsBuilder().include(ReadDataBenchmarksKvaReadData.class.getSimpleName() + "\\.") - .build(); - - new Runner(options).run(); - } - - public ReadData read() throws IOException { - - return ((FileSystemKeyValueAccess)kva).createReadData(getPath().toString()); - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index cf7322bd5..220a5a497 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -113,12 +113,13 @@ public void testFileKvaReadData() throws IOException { os.write(data); } - final ReadData readData = new FileSystemKeyValueAccess(FileSystems.getDefault()) - .createReadData(tmpF.getAbsolutePath()); + try( final VolatileReadData readData = new FileSystemKeyValueAccess(FileSystems.getDefault()) + .createReadData(tmpF.getAbsolutePath())) { - assertEquals("file read data length", -1, readData.length()); - assertEquals("file read data length", 128, readData.requireLength()); - sliceTestHelper(readData, N); + assertEquals("file read data length", -1, readData.length()); + assertEquals("file read data length", 128, readData.requireLength()); + sliceTestHelper(readData, N); + } } private void readDataTestHelper( ReadData readData, int N, int materializedN ) throws IOException { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java index f7a6fa339..1d3becba7 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java @@ -35,18 +35,22 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public class DatasetAccessTest { @@ -164,14 +168,25 @@ public void testDeleteBlock() { checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); // if a shard becomes empty the corresponding key should be deleted - assertNotNull(store.get(new long[] {1, 0, 0})); + assertTrue(keyExists(store, new long[] {1, 0, 0})); datasetAccess.deleteBlock(store, new long[] {8, 4, 1}); - assertNull(store.get(new long[] {1, 0, 0})); + assertFalse(keyExists(store, new long[] {1, 0, 0})); // deleting a non-existent block should not fail datasetAccess.deleteBlock(store, new long[] {0, 0, 8}); } + private boolean keyExists(final PositionValueAccess store, final long[] key) { + try (final VolatileReadData data = store.get(key)) { + if (data != null) { + data.requireLength(); + return true; + } + } catch (N5Exception.N5IOException ignored) { + } + return false; + } + @Test public void testDeleteBlocks() { @@ -193,9 +208,9 @@ public void testDeleteBlocks() { checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); // if a shard becomes empty the corresponding key should be deleted - assertNotNull(store.get(new long[] {1, 0, 0})); + assertTrue(keyExists(store, new long[] {1, 0, 0})); datasetAccess.deleteBlocks(store, Arrays.asList(new long[][] {{8, 4, 1}})); - assertNull(store.get(new long[] {1, 0, 0})); + assertFalse(keyExists(store, new long[] {1, 0, 0})); // deleting a non-existent block should not fail datasetAccess.deleteBlocks(store, Arrays.asList(new long[] {0, 0, 8})); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 2b1cc4e95..b7595750a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -29,19 +29,6 @@ package org.janelia.saalfeldlab.n5.shard; import com.google.gson.GsonBuilder; -import org.janelia.saalfeldlab.n5.*; -import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; -import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; @@ -54,8 +41,42 @@ import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.NoSuchFileException; -import java.util.*; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.function.Function; +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; +import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedFileChannel; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; +import org.janelia.saalfeldlab.n5.N5FSTest; +import org.janelia.saalfeldlab.n5.N5KeyValueWriter; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -282,7 +303,7 @@ private void ensureKeysDoNotExist(KeyValueAccess kva, URI uri, String dataset, Assert.assertFalse("Shard at" + shard + " exists but should not.", kva.exists(shard)); } } - + @Test public void writeShardDataSizeTest() { @@ -459,11 +480,11 @@ public void numReadsTest() { writer.readBlocks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0})); System.out.println(writer.getNumMaterializeCalls()); - ArrayList ptList = new ArrayList<>(); - ptList.add(new long[] {0,0}); - ptList.add(new long[] {0,1}); - ptList.add(new long[] {1,0}); - ptList.add(new long[] {1,1}); + ArrayList ptList = new ArrayList<>(); + ptList.add(new long[] {0, 0}); + ptList.add(new long[] {0, 1}); + ptList.add(new long[] {1, 0}); + ptList.add(new long[] {1, 1}); writer.resetNumMaterializeCalls(); writer.readBlocks(dataset, datasetAttributes, ptList); @@ -588,54 +609,100 @@ public boolean isFile(String normalPath) { } @Override - public ReadData createReadData(final String normalPath) { - return new KeyValueAccessReadData(new TrackingFileLazyRead(normalPath)); + public VolatileReadData createReadData(final String normalPath) { + try { + return VolatileReadData.from(new TrackingFileLazyRead(fileSystem.getPath(normalPath))); + } catch (N5NoSuchKeyException e) { +// return VolatileReadData.from(new NoSuchKeyLazyRead()); + return null; + } } - private class TrackingFileLazyRead implements LazyRead { + // This can be used in createReadData() above, to also simulate the case that we will have for + // cloud storage KVAs, where the returned VolatileReadData is non-null but will fail on the first + // operation that queries the cloud backend. + private class NoSuchKeyLazyRead implements LazyRead { + + @Override + public ReadData materialize(final long offset, final long length) throws N5Exception.N5IOException { + throw new N5NoSuchKeyException("NoSuchKeyLazyRead"); + } + + @Override + public long size() throws N5Exception.N5IOException { + throw new N5NoSuchKeyException("NoSuchKeyLazyRead"); + } - private final String normalKey; + @Override + public void close() { + } + } - TrackingFileLazyRead(String normalKey) { - this.normalKey = normalKey; - } + private class TrackingFileLazyRead implements LazyRead { - @Override - public long size() { - return TrackingFileSystemKeyValueAccess.this.size(normalKey); - } + private final Path path; + private LockedFileChannel lock; - @Override - public ReadData materialize(final long offset, final long length) { + TrackingFileLazyRead(final Path path) { + this.path = path; + lock = FileSystemKeyValueAccess.lockForReading(path); + } - numMaterializeCalls++; + @Override + public long size() throws N5Exception.N5IOException { - try (final LockedFileChannel lfs = lockForReading(normalKey)) { - final FileChannel channel = lfs.getFileChannel(); + if (lock == null) { + throw new N5Exception.N5IOException("FileLazyRead is already closed."); + } + return FileSystemKeyValueAccess.size(path); + } - channel.position(offset); - if (length > Integer.MAX_VALUE) - throw new IOException("Attempt to materialize too large data"); + @Override + public ReadData materialize(final long offset, final long length) { - final long channelSize = channel.size(); - if (!validBounds(channelSize, offset, length)) - throw new IndexOutOfBoundsException(); + if (lock == null) { + throw new N5Exception.N5IOException("FileLazyRead is already closed."); + } - final int sz = (int)(length < 0 ? channelSize : length); - totalBytesRead += sz; - final byte[] data = new byte[sz]; - final ByteBuffer buf = ByteBuffer.wrap(data); - channel.read(buf); - return ReadData.from(data); + numMaterializeCalls++; + try (final FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { - } catch (final NoSuchFileException e) { - throw new N5NoSuchKeyException("No such file", e); - } catch (IOException | UncheckedIOException e) { - throw new N5Exception.N5IOException(e); - } - } - } - private static boolean validBounds(long channelSize, long offset, long length) { + channel.position(offset); + + final long channelSize = channel.size(); + if (!validBounds(channelSize, offset, length)) { + throw new IndexOutOfBoundsException(); + } + + final long size = length < 0 ? (channelSize - offset) : length; + if (size > Integer.MAX_VALUE) { + throw new IndexOutOfBoundsException("Attempt to materialize too large data"); + } + + final byte[] data = new byte[(int) size]; + totalBytesRead += size; + final ByteBuffer buf = ByteBuffer.wrap(data); + channel.read(buf); + return ReadData.from(data); + + } catch (final NoSuchFileException e) { + throw new N5NoSuchKeyException("No such file", e); + } catch (IOException | UncheckedIOException e) { + throw new N5Exception.N5IOException(e); + } + } + + @Override + public void close() throws IOException { + + if (lock != null) { + lock.close(); + lock = null; + } + } + } + + private static boolean validBounds(long channelSize, long offset, long length) { if (offset < 0) return false; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java index 99c1a3fad..957b8e5dd 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java @@ -28,21 +28,27 @@ */ package org.janelia.saalfeldlab.n5.shard; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; public class TestPositionValueAccess implements PositionValueAccess { private final Map map = new HashMap<>(); @Override - public ReadData get(final long[] key) { + public VolatileReadData get(final long[] key) { final byte[] bytes = map.get(new Key(key)); - return bytes == null ? null : ReadData.from(bytes); + return bytes == null ? null : new ClosableWrapper(ReadData.from(bytes)); } public void set(final long[] key, final ReadData data) { @@ -91,4 +97,78 @@ public int hashCode() { return Arrays.hashCode(data); } } + + private static class ClosableWrapper implements VolatileReadData { + + private final ReadData delegate; + + ClosableWrapper(final ReadData delegate) { + this.delegate = delegate; + } + + @Override + public long length() { + return delegate.length(); + } + + @Override + public long requireLength() throws N5IOException { + return delegate.requireLength(); + } + + @Override + public ReadData limit(final long length) throws N5IOException { + return delegate.limit(length); + } + + @Override + public ReadData slice(final long offset, final long length) throws N5IOException { + return delegate.slice(offset, length); + } + + @Override + public ReadData slice(final Range range) throws N5IOException { + return delegate.slice(range); + } + + @Override + public InputStream inputStream() throws N5IOException, IllegalStateException { + return delegate.inputStream(); + } + + @Override + public byte[] allBytes() throws N5IOException, IllegalStateException { + return delegate.allBytes(); + } + + @Override + public ByteBuffer toByteBuffer() throws N5IOException, IllegalStateException { + return delegate.toByteBuffer(); + } + + @Override + public ReadData materialize() throws N5IOException { + delegate.materialize(); + return this; + } + + @Override + public void writeTo(final OutputStream outputStream) throws N5IOException, IllegalStateException { + delegate.writeTo(outputStream); + } + + @Override + public void prefetch(final Collection ranges) throws N5IOException { + delegate.prefetch(ranges); + } + + @Override + public ReadData encode(final OutputStreamOperator encoder) { + return delegate.encode(encoder); + } + + @Override + public void close() throws N5IOException { + } + } } From af33e0308c7a89b0535ca4f03606adf92aeaadce Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 27 Jan 2026 19:58:54 -0500 Subject: [PATCH 04/21] feat: add readShard writeShard methods * add a few tests --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 18 ++++++ .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 17 +++++ .../org/janelia/saalfeldlab/n5/N5Reader.java | 25 ++++++++ .../org/janelia/saalfeldlab/n5/N5Writer.java | 19 ++++++ .../saalfeldlab/n5/AbstractN5Test.java | 27 ++++++++ .../n5/http/HttpReaderFsWriter.java | 4 ++ .../saalfeldlab/n5/shard/ShardTest.java | 64 +++++++++++++++++-- 7 files changed, 168 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 5b516fef6..0e40a1c88 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -121,6 +121,24 @@ default List> readBlocks( return convertedDatasetAttributes. getDatasetAccess().readBlocks(posKva, blockPositions); } + @Override + default DataBlock readShard( + final String pathName, + final DatasetAttributes datasetAttributes, + final long... gridPosition) throws N5Exception { + + final DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); + final int shardLevel = convertedDatasetAttributes.getNestedBlockGrid().numLevels() - 1; + try { + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), + convertedDatasetAttributes); + return convertedDatasetAttributes. getDatasetAccess().readShard(posKva, gridPosition, shardLevel); + + } catch (N5Exception.N5NoSuchKeyException e) { + return null; + } + } + @Override default String[] list(final String pathName) throws N5Exception { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index a938e65a3..55262351a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -280,7 +280,24 @@ default void writeBlock( "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, e); } + } + + @Override + default void writeShard( + final String path, + final DatasetAttributes datasetAttributes, + final DataBlock shard) throws N5Exception { + final DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); + final int shardLevel = convertedDatasetAttributes.getNestedBlockGrid().numLevels() - 1; + try { + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes); + convertedDatasetAttributes. getDatasetAccess().writeShard(posKva, shard, shardLevel); + } catch (final UncheckedIOException e) { + throw new N5IOException( + "Failed to write block " + Arrays.toString(shard.getGridPosition()) + " into dataset " + path, + e); + } } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index c5fd251e5..c8b4b39dd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -348,6 +348,31 @@ default List> readBlocks( return blocks; } + /** + * Reads a shard as a {@link DataBlock}. + *

+ * A "shard" is the largest level of the datasets {@link NestedGrid} + * This method's behavior is identical to readBlock for un-sharded datasets. + * + * @param + * the DataBlock data type + * @param pathName + * dataset path + * @param datasetAttributes + * the dataset attributes + * @param gridPosition + * the position in the shard grid + * @return the data block + * @throws N5Exception + * the exception + * + * @see DatasetAttributes#getNestedBlockGrid() + */ + DataBlock readShard( + final String pathName, + final DatasetAttributes datasetAttributes, + final long... gridPosition) throws N5Exception; + /** * Checks if a shard (or block for non-sharded datasets) exists at the * given grid position without reading the data. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 79c5da110..a66099a3e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -278,6 +278,25 @@ default void writeBlocks( } } + /** + * Writes a shard stored as a {@link DataBlock}. + *

+ * A "shard" is the largest level of the datasets {@link NestedGrid}. + * This method's behavior is identical to writeBlock for un-sharded datasets. + * + * @param datasetPath dataset path + * @param datasetAttributes the dataset attributes + * @param dataBlock the data block + * @param the data block data type + * @throws N5Exception the exception + * + * @see DatasetAttributes#getNestedBlockGrid() + */ + void writeShard( + final String pathName, + final DatasetAttributes datasetAttributes, + final DataBlock dataBlock) throws N5Exception; + @FunctionalInterface interface DataBlockSupplier { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 60025e453..dc12d3943 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -474,6 +474,33 @@ public void testWriteReadDoubleBlock() { } } + @Test + public void testWriteReadShard() { + + // test that writeShard behaves the same as writeBlock + // for unsharded datasets + + for (final Compression compression : getCompressions()) { + try (final N5Writer n5 = createTempN5Writer()) { + + n5.createDataset(datasetName, dimensions, blockSize, DataType.INT16, compression); + final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock); + + n5.writeShard(datasetName, attributes, dataBlock); + + // read with readShard + final DataBlock loadedShard = n5.readShard(datasetName, attributes, 0, 0, 0); + assertArrayEquals(shortBlock, (short[])loadedShard.getData()); + + // read with readBlock + final DataBlock loadedDataBlock = n5.readShard(datasetName, attributes, 0, 0, 0); + assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData()); + } + } + } + + @Test public void testMode1WriteReadByteBlock() { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index b5555313a..fe13f0d68 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -276,6 +276,10 @@ public HttpRead writer.writeBlock(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); } + @Override public void writeShard(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { + writer.writeShard(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); + } + @Override public boolean deleteBlock(String datasetPath, long... gridPosition) throws N5Exception { return writer.deleteBlock(datasetPath, gridPosition); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index b7595750a..879f872b5 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -51,6 +51,7 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.stream.IntStream; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; @@ -78,6 +79,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -149,6 +151,11 @@ public void removeTempWriters() { private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] blockSize) { + return getTestAttributes(DataType.UINT8, dimensions, shardSize, blockSize); + } + + private DatasetAttributes getTestAttributes(DataType dataType, long[] dimensions, int[] shardSize, int[] blockSize) { + DefaultShardCodecInfo blockCodec = new DefaultShardCodecInfo( blockSize, new N5BlockCodecInfo(), @@ -160,7 +167,7 @@ private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, return new DatasetAttributes( dimensions, shardSize, - DataType.UINT8, + dataType, blockCodec); } @@ -433,6 +440,51 @@ public void writeReadBlockTest() { } } + @Test + public void writeReadShardTest() { + + try ( final N5Writer n5 = tempN5Factory.createTempN5Writer() ) { + + final int[] shardSize = new int[] {4,4}; + final int shardN = 16; + + final int[] blockSize = new int[] {2,2}; + final int blockN = 4; + + final String dataset = "writeReadShard"; + DatasetAttributes attrs = getTestAttributes(DataType.INT32, new long[]{8, 8}, shardSize, blockSize); + + final int[] shardData = range(shardN); + IntArrayDataBlock shard = new IntArrayDataBlock(shardSize, new long[]{0, 0}, shardData); + + n5.writeShard(dataset, attrs, shard); + DataBlock readShard = n5.readShard(dataset, attrs, 0, 0); + assertArrayEquals(shardData, readShard.getData()); + + + /** + * The 4x4 shard at (0,0) + * and the 2x2 blocks it contains + * + * + * 0 1 | 2 3 + * 4 5 | 6 7 + * ---------------- + * 8 9 | 10 11 + * 12 13 | 14 15 + */ + + assertArrayEquals(new int[]{0, 1, 4, 5}, (int[])n5.readBlock(dataset, attrs, 0, 0).getData()); + assertArrayEquals(new int[]{2, 3, 6, 7}, (int[])n5.readBlock(dataset, attrs, 1, 0).getData()); + assertArrayEquals(new int[]{8, 9, 12, 13}, (int[])n5.readBlock(dataset, attrs, 0, 1).getData()); + assertArrayEquals(new int[]{10, 11, 14, 15}, (int[])n5.readBlock(dataset, attrs, 1, 1).getData()); + } + } + + private int[] range(int N) { + return IntStream.range(0, N).toArray(); + } + /** * Checks how many read calls to the backend are performed for a particular readBlocks * call. At this time (Nov 4 2025), one read for the index, and one read per block are performed. @@ -505,7 +557,7 @@ public void shardExistsTest() { final String dataset = "shardExists"; writer.remove(dataset); - writer.createDataset(dataset, datasetAttributes); + DatasetAttributes attrs = writer.createDataset(dataset, datasetAttributes); final int[] blockSize = datasetAttributes.getBlockSize(); final int numElements = blockSize[0] * blockSize[1]; @@ -518,7 +570,7 @@ public void shardExistsTest() { /* write blocks to shards (0,0), (1,0), and (2,2) */ writer.writeBlocks( dataset, - datasetAttributes, + attrs, new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), /* shard (0, 0) */ new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), /* shard (1, 0) */ new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) /* shard (2, 2) */ @@ -528,7 +580,7 @@ public void shardExistsTest() { Function assertShardExistsTracking = (gridPosition) -> { trackingWriter.resetAllTracking(); - final Boolean exists = writer.shardExists(dataset, datasetAttributes, gridPosition); + final Boolean exists = writer.shardExists(dataset, attrs, gridPosition); assertEquals("isFileCheck incremented", 1, trackingWriter.getNumIsFileCalls()); assertEquals("No Bytes Read", 0, trackingWriter.getTotalBytesRead()); return exists; From 71af9b21b8f0e16af90ac0e9be68a91a08ba329e Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 30 Jan 2026 13:31:11 -0500 Subject: [PATCH 05/21] test: toward more thorough read/write shard tests --- .../org/janelia/saalfeldlab/n5/shard/ShardTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 879f872b5..7c3ccea34 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -51,7 +51,9 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; @@ -478,6 +480,14 @@ public void writeReadShardTest() { assertArrayEquals(new int[]{2, 3, 6, 7}, (int[])n5.readBlock(dataset, attrs, 1, 0).getData()); assertArrayEquals(new int[]{8, 9, 12, 13}, (int[])n5.readBlock(dataset, attrs, 0, 1).getData()); assertArrayEquals(new int[]{10, 11, 14, 15}, (int[])n5.readBlock(dataset, attrs, 1, 1).getData()); + + System.out.println("delete"); + n5.deleteBlock(dataset, attrs, new long[]{1, 1}); + + System.out.println("read"); + final DataBlock partlyEmptyShard = n5.readShard(dataset, attrs, 0, 0); + System.out.println(Arrays.toString(partlyEmptyShard.getData())); + System.out.println("done"); } } From 90e8c4174a4b3a205542d16994f2059c417c40fb Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 30 Jan 2026 14:29:33 -0500 Subject: [PATCH 06/21] test: more read/write Shard tests --- .../saalfeldlab/n5/shard/ShardTest.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 7c3ccea34..b451c7815 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -83,6 +83,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @RunWith(Parameterized.class) @@ -481,13 +482,26 @@ public void writeReadShardTest() { assertArrayEquals(new int[]{8, 9, 12, 13}, (int[])n5.readBlock(dataset, attrs, 0, 1).getData()); assertArrayEquals(new int[]{10, 11, 14, 15}, (int[])n5.readBlock(dataset, attrs, 1, 1).getData()); - System.out.println("delete"); n5.deleteBlock(dataset, attrs, new long[]{1, 1}); - System.out.println("read"); + /** + * After deleting block (1,1) + * + * 0 1 | 2 3 + * 4 5 | 6 7 + * ---------------- + * 8 9 | 0 0 + * 12 13 | 0 0 + */ final DataBlock partlyEmptyShard = n5.readShard(dataset, attrs, 0, 0); - System.out.println(Arrays.toString(partlyEmptyShard.getData())); - System.out.println("done"); + assertArrayEquals(new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 12, 13, 0, 0}, partlyEmptyShard.getData()); + + + // Delete the rest of the blocks + n5.deleteBlocks(dataset, attrs, + Stream.of( new long[] {0,0}, new long[] {1,0}, new long[] {0,1}).collect(Collectors.toList())); + + assertNull(n5.readShard(dataset, attrs, 0, 0)); } } From b4ec497d0d5dfb7a98adc496e0fba9aa462f566f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 30 Jan 2026 14:49:14 -0500 Subject: [PATCH 07/21] doc: fix javadoc references to NestedGrid --- src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java | 2 +- src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index c8b4b39dd..aebbfa67d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -351,7 +351,7 @@ default List> readBlocks( /** * Reads a shard as a {@link DataBlock}. *

- * A "shard" is the largest level of the datasets {@link NestedGrid} + * A "shard" is the largest level of the datasets {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid} * This method's behavior is identical to readBlock for un-sharded datasets. * * @param diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index a66099a3e..64dc94166 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -281,10 +281,10 @@ default void writeBlocks( /** * Writes a shard stored as a {@link DataBlock}. *

- * A "shard" is the largest level of the datasets {@link NestedGrid}. + * A "shard" is the largest level of the datasets {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid}. * This method's behavior is identical to writeBlock for un-sharded datasets. * - * @param datasetPath dataset path + * @param pathName dataset path * @param datasetAttributes the dataset attributes * @param dataBlock the data block * @param the data block data type From 7acd01e067fd8ddd747fbdd7c90d64de12baa326 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 30 Jan 2026 17:01:51 -0500 Subject: [PATCH 08/21] test: Shard test updates * test a list of nulls is returned for non-existing blocks * test number of backend read calls for readBlocks --- .../saalfeldlab/n5/shard/ShardTest.java | 100 ++++++++++++++---- 1 file changed, 79 insertions(+), 21 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index b451c7815..af86a3d9f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -60,6 +60,7 @@ import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; +import org.janelia.saalfeldlab.n5.IntArrayDataBlock; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedFileChannel; import org.janelia.saalfeldlab.n5.N5Exception; @@ -343,6 +344,19 @@ public void writeShardDataSizeTest() { data[i] = (byte)((100) + (10) + i); } + /* + * No blocks or shards exist. + * Calling readBlocks should return a list that is the same length as the requested grid positions, + * and every entry should be null. + */ + final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; + final List> readBlocks = writer.readBlocks(dataset, datasetAttributes, Arrays.asList(newBlockIndices)); + assertEquals(newBlockIndices.length, readBlocks.size()); + assertTrue("readBlocks for empty shard: all blocks null", readBlocks.stream().allMatch(blk -> blk == null)); + + /* + * Now write blocks + */ writer.writeBlocks( dataset, datasetAttributes, @@ -388,20 +402,6 @@ public void writeShardDataSizeTest() { } - @Test - public void readBlocksTest() { - - final N5Writer n5 = tempN5Factory.createTempN5Writer(); - final DatasetAttributes datasetAttributes = getTestAttributes( - new long[]{24, 24}, - new int[]{8, 8}, - new int[]{2, 2}); - - final String dataset = "writeReadBlocks"; - final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; - final List> readBlocks = n5.readBlocks(dataset, datasetAttributes, Arrays.asList(newBlockIndices)); - } - @Test public void writeReadBlockTest() { @@ -495,20 +495,16 @@ public void writeReadShardTest() { */ final DataBlock partlyEmptyShard = n5.readShard(dataset, attrs, 0, 0); assertArrayEquals(new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 12, 13, 0, 0}, partlyEmptyShard.getData()); - - + + // Delete the rest of the blocks - n5.deleteBlocks(dataset, attrs, + n5.deleteBlocks(dataset, attrs, Stream.of( new long[] {0,0}, new long[] {1,0}, new long[] {0,1}).collect(Collectors.toList())); assertNull(n5.readShard(dataset, attrs, 0, 0)); } } - private int[] range(int N) { - return IntStream.range(0, N).toArray(); - } - /** * Checks how many read calls to the backend are performed for a particular readBlocks * call. At this time (Nov 4 2025), one read for the index, and one read per block are performed. @@ -624,6 +620,68 @@ public void shardExistsTest() { Assert.assertFalse("Shard (0,2) should not exist", assertShardExistsTracking.apply(new long[]{0, 2})); } + /** + * Checks how many read calls to the backend are performed for a particular readBlocks + * call. At this time (Jan 4 2026), one read for the index, and one read per block are performed. + */ + @Test + public void testPartialReadAggregationBehavior() { + + final DatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + new int[]{2, 2} + ); + + try (TrackingN5Writer writer = (TrackingN5Writer)tempN5Factory.createTempN5Writer()) { + + final String dataset = "shardExists"; + writer.remove(dataset); + DatasetAttributes attrs = writer.createDataset(dataset, datasetAttributes); + + final int[] blockSize = attrs.getBlockSize(); + final int numElements = blockSize[0] * blockSize[1]; + + final byte[] data = new byte[numElements]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)(i); + } + + // four blocks in shard (0,0) + ArrayList ptList = new ArrayList<>(); + ptList.add(new long[] {0,0}); + ptList.add(new long[] {0,1}); + ptList.add(new long[] {1,0}); + ptList.add(new long[] {1,1}); + + /* write blocks to shard (0,0) */ + writer.writeBlocks( + dataset, + datasetAttributes, + new ByteArrayDataBlock(blockSize, ptList.get(0), data), + new ByteArrayDataBlock(blockSize, ptList.get(1), data), + new ByteArrayDataBlock(blockSize, ptList.get(2), data), + new ByteArrayDataBlock(blockSize, ptList.get(3), data) + ); + + writer.resetNumMaterializeCalls(); + writer.readBlocks(dataset, datasetAttributes, ptList); + + // TODO change this if and when we implement aggregation of read calls + // one for the index, one for each of the four blocks + assertEquals(5, writer.getNumMaterializeCalls()); + + writer.resetNumMaterializeCalls(); + writer.readShard(dataset, datasetAttributes, new long[] {0,0}); + // one for the index, one for each of the four blocks + assertEquals(5, writer.getNumMaterializeCalls()); + } + } + + private int[] range(int N) { + return IntStream.range(0, N).toArray(); + } + /** * An N5Writer that tracks the number of materialize calls performed by * its underlying key value access. From c89ad0d9861a9bdef59caa09a33ae0a049b09bf4 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 2 Feb 2026 22:14:39 +0100 Subject: [PATCH 09/21] clean up --- .../saalfeldlab/n5/shard/ShardTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index af86a3d9f..a4118d753 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -50,7 +50,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -598,9 +598,9 @@ public void shardExistsTest() { TrackingN5Writer trackingWriter = ((TrackingN5Writer) writer); - Function assertShardExistsTracking = (gridPosition) -> { + Predicate assertShardExistsTracking = (gridPosition) -> { trackingWriter.resetAllTracking(); - final Boolean exists = writer.shardExists(dataset, attrs, gridPosition); + final boolean exists = writer.shardExists(dataset, attrs, gridPosition); assertEquals("isFileCheck incremented", 1, trackingWriter.getNumIsFileCalls()); assertEquals("No Bytes Read", 0, trackingWriter.getTotalBytesRead()); return exists; @@ -609,15 +609,15 @@ public void shardExistsTest() { trackingWriter.resetAllTracking(); /* shards that should exist should only check file */ - Assert.assertTrue("Shard (0,0) should exist", assertShardExistsTracking.apply(new long[]{0, 0})); - Assert.assertTrue("Shard (1,0) should exist", assertShardExistsTracking.apply(new long[]{1, 0})); - Assert.assertTrue("Shard (2,2) should exist", assertShardExistsTracking.apply(new long[]{2, 2})); + Assert.assertTrue("Shard (0,0) should exist", assertShardExistsTracking.test(new long[]{0, 0})); + Assert.assertTrue("Shard (1,0) should exist", assertShardExistsTracking.test(new long[]{1, 0})); + Assert.assertTrue("Shard (2,2) should exist", assertShardExistsTracking.test(new long[]{2, 2})); /* shards that should NOT exist */ - Assert.assertFalse("Shard (0,1) should not exist", assertShardExistsTracking.apply(new long[]{0, 1})); - Assert.assertFalse("Shard (1,1) should not exist", assertShardExistsTracking.apply(new long[]{1, 1})); - Assert.assertFalse("Shard (2,0) should not exist", assertShardExistsTracking.apply(new long[]{2, 0})); - Assert.assertFalse("Shard (0,2) should not exist", assertShardExistsTracking.apply(new long[]{0, 2})); + Assert.assertFalse("Shard (0,1) should not exist", assertShardExistsTracking.test(new long[]{0, 1})); + Assert.assertFalse("Shard (1,1) should not exist", assertShardExistsTracking.test(new long[]{1, 1})); + Assert.assertFalse("Shard (2,0) should not exist", assertShardExistsTracking.test(new long[]{2, 0})); + Assert.assertFalse("Shard (0,2) should not exist", assertShardExistsTracking.test(new long[]{0, 2})); } /** From 62313cd66568b876e901761f6fa467c9f8170328 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 9 Feb 2026 17:35:07 +0100 Subject: [PATCH 10/21] Add KeyValueAccess.write(key, data) --- .../n5/FileSystemKeyValueAccess.java | 11 +++++- .../saalfeldlab/n5/HttpKeyValueAccess.java | 6 ++++ .../saalfeldlab/n5/KeyValueAccess.java | 21 ++++++++--- .../n5/shard/PositionValueAccess.java | 36 ++++++++++--------- 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 6e5f8ce6a..a320b093b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -89,6 +89,16 @@ public VolatileReadData createReadData(final String normalPath) { } } + @Override + public void write(final String normalPath, final ReadData data) throws N5IOException { + final Path path = fileSystem.getPath(normalPath); + try (final LockedFileChannel channel = lockForWriting(path)) { + data.writeTo(channel.newOutputStream()); + } catch (IOException e) { + throw new N5IOException(e); + } + } + @Override public LockedFileChannel lockForReading(final String normalPath) throws N5IOException { @@ -491,7 +501,6 @@ protected static void createAndCheckIsDirectory( private static class FileLazyRead implements LazyRead { - private final Path path; private LockedFileChannel lock; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index 7b0b2614f..86340e079 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -255,6 +255,12 @@ public LockedChannel lockForWriting(final String normalPath) throws N5IOExceptio throw new N5Exception("HttpKeyValueAccess is read-only"); } + @Override + public void write(final String normalPath, final ReadData data) throws N5IOException { + + throw new N5Exception("HttpKeyValueAccess is read-only"); + } + /** * List all 'directory'-like children of a path. *

diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 014b60d63..ac94567f5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -43,8 +43,7 @@ * Key value read primitives used by {@link N5KeyValueReader} * implementations. This interface implements a subset of access primitives * provided by {@link FileSystem} to reduce the implementation burden for - * backends - * lacking a {@link FileSystem} implementation (such as AWS-S3). + * backends lacking a {@link FileSystem} implementation (such as AWS-S3). * * @author Stephan Saalfeld */ @@ -262,7 +261,22 @@ default URI uri(final String uriString) throws URISyntaxException { * @throws N5IOException * if an error occurs */ - VolatileReadData createReadData( final String normalPath ) throws N5IOException; + VolatileReadData createReadData(final String normalPath) throws N5IOException; + + /** + * Write {@code data} to the given {@code normalPath}. + *

+ * Existing data at {@code normalPath} will be overridden. + * + * @param normalPath + * is expected to be in normalized form, no further efforts are made to normalize it + * @param data + * the data to write + * + * @throws N5IOException + * if an error occurs + */ + void write(String normalPath, ReadData data) throws N5IOException; /** * Create a lock on a path for reading. This isn't meant to be kept @@ -348,5 +362,4 @@ default URI uri(final String uriString) throws URISyntaxException { * if an error occurs during deletion */ void delete( final String normalPath ) throws N5IOException; - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java index d81d2156e..36a1645fb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java @@ -28,13 +28,10 @@ */ package org.janelia.saalfeldlab.n5.shard; -import java.io.IOException; -import java.io.OutputStream; import java.net.URI; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -63,6 +60,18 @@ public interface PositionValueAccess { */ VolatileReadData get(long[] key) throws N5Exception.N5IOException; + /** + * Write the {@code data} for a DataBlock (or shard) to the given position + * in the block grid. + * + * @param key + * The grid position of the DataBlock (or shard) to write + * @param data + * The data to write + * + * @throws N5Exception.N5IOException + * if an error occurs while writing + */ void set(long[] key, ReadData data) throws N5Exception.N5IOException; boolean exists(long[] key) throws N5Exception.N5IOException; @@ -104,38 +113,31 @@ class KvaPositionValueAccess implements PositionValueAccess { * to the target data block * @return the absolute path to the data block ad gridPosition */ - protected String absolutePath( final long... gridPosition) { + protected String absolutePath(final long... gridPosition) { return kva.compose(uri, normalPath, attributes.relativeBlockPath(gridPosition)); } @Override - public VolatileReadData get(long[] key) throws N5IOException { + public VolatileReadData get(final long[] key) throws N5IOException { return kva.createReadData(absolutePath(key)); } @Override - public boolean exists(long[] key) throws N5IOException { + public boolean exists(final long[] key) throws N5IOException { return kva.isFile(absolutePath(key)); } @Override - public void set(long[] key, ReadData data) throws N5IOException { - + public void set(final long[] key, final ReadData data) throws N5IOException { if (data == null) { remove(key); - return; - } - - try ( final LockedChannel ch = kva.lockForWriting(absolutePath(key)); - final OutputStream outputStream = ch.newOutputStream();) { - data.writeTo(outputStream); - } catch (IOException e) { - throw new N5IOException(e); + } else { + kva.write(absolutePath(key), data); } } @Override - public boolean remove(long[] gridPosition) throws N5IOException { + public boolean remove(final long[] gridPosition) throws N5IOException { final String key = absolutePath(gridPosition); if (!kva.isFile(key)) From 72d4424417b23547a4c6c7e58c8da9d4e1fc9a8e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 5 Feb 2026 11:47:03 -0500 Subject: [PATCH 11/21] fix: attempt to read without locking when locking fails; Some file system mount circumstances can lead to files being read/writeable, but not lockable. In these cases we should attempt to read, without locking, and immediately materialize as a best-effort attempt at ensuring valid data Signed-off-by: Caleb Hulbert --- .../n5/FileSystemKeyValueAccess.java | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index a320b093b..3b7b6ae91 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -28,6 +28,7 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.Closeable; import java.nio.file.StandardOpenOption; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; @@ -81,12 +82,20 @@ public FileSystemKeyValueAccess(final FileSystem fileSystem) { } @Override - public VolatileReadData createReadData(final String normalPath) { + public VolatileReadData createReadData(final String normalPath) throws N5IOException { + Path path = fileSystem.getPath(normalPath); + FileLazyRead fileLazyRead = null; try { - return VolatileReadData.from(new FileLazyRead(fileSystem.getPath(normalPath))); - } catch (N5NoSuchKeyException e) { - return null; + fileLazyRead = new FileLazyRead(path); + } catch (N5NoSuchKeyException e) { + throw e; + } catch (N5IOException e) { + /* Try to create without locking, and immediately materialize */ + fileLazyRead = new FileLazyRead(path, false); + fileLazyRead.materialize(0, 0); } + + return VolatileReadData.from(fileLazyRead); } @Override @@ -501,12 +510,25 @@ protected static void createAndCheckIsDirectory( private static class FileLazyRead implements LazyRead { + private static final Closeable NO_OP = () -> { }; + private final Path path; - private LockedFileChannel lock; + private Closeable lock; FileLazyRead(final Path path) { + this(path, true); + } + + FileLazyRead(final Path path, final Boolean requireLock ) { this.path = path; - lock = lockForReading(path); + try { + lock = lockForReading(path); + } catch (Exception e) { + if (requireLock) { + throw e; + } + lock = NO_OP; + } } @Override From 84aa00c45639a8153116066eaa5f37212fe77b67 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 10 Feb 2026 16:06:42 -0500 Subject: [PATCH 12/21] feat: support atomic, unsafe, and fallback policies for file system read/write operations Signed-off-by: Caleb Hulbert --- .../n5/FileSystemKeyValueAccess.java | 73 +++++++++++-------- .../janelia/saalfeldlab/n5/FsIoPolicy.java | 61 ++++++++++++++++ .../org/janelia/saalfeldlab/n5/IoPolicy.java | 47 ++++++++++++ 3 files changed, 150 insertions(+), 31 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 3b7b6ae91..19b5eacea 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -69,6 +69,24 @@ */ public class FileSystemKeyValueAccess implements KeyValueAccess { + private static final IoPolicy ioPolicy = getIoPolicy(); + + private static IoPolicy getIoPolicy() { + String property = System.getProperty("n5.ioPolicy"); + if (property == null) + return FsIoPolicy.atomicWithFallback; + + switch (property) { + case "atomic": + return new FsIoPolicy.Atomic(); + case "unsafe": + return new FsIoPolicy.Unsafe(); + case "atomicFallbackUnsafe": + default: + return FsIoPolicy.atomicWithFallback; + } + } + protected final FileSystem fileSystem; /** @@ -83,19 +101,14 @@ public FileSystemKeyValueAccess(final FileSystem fileSystem) { @Override public VolatileReadData createReadData(final String normalPath) throws N5IOException { - Path path = fileSystem.getPath(normalPath); - FileLazyRead fileLazyRead = null; + try { - fileLazyRead = new FileLazyRead(path); - } catch (N5NoSuchKeyException e) { - throw e; - } catch (N5IOException e) { - /* Try to create without locking, and immediately materialize */ - fileLazyRead = new FileLazyRead(path, false); - fileLazyRead.materialize(0, 0); + return ioPolicy.read(normalPath); + } catch (final NoSuchFileException e) { + throw new N5NoSuchKeyException("No such file", e); + } catch (IOException | UncheckedIOException e) { + throw new N5IOException("Failed to lock file for reading: " + normalPath, e); } - - return VolatileReadData.from(fileLazyRead); } @Override @@ -105,6 +118,12 @@ public void write(final String normalPath, final ReadData data) throws N5IOExcep data.writeTo(channel.newOutputStream()); } catch (IOException e) { throw new N5IOException(e); + try { + ioPolicy.write(normalPath, data); + } catch (final NoSuchFileException e) { + throw new N5NoSuchKeyException("No such file", e); + } catch (IOException | UncheckedIOException e) { + throw new N5IOException("Failed to lock file for writing: " + normalPath, e); } } @@ -174,9 +193,9 @@ protected static long size(final Path path) { try { return Files.size(path); } catch (NoSuchFileException e) { - throw new N5Exception.N5NoSuchKeyException("No such file", e); + throw new N5NoSuchKeyException("No such file", e); } catch (IOException | UncheckedIOException e) { - throw new N5Exception.N5IOException(e); + throw new N5IOException(e); } } @@ -331,17 +350,13 @@ public void delete(final String normalPath) throws N5IOException { final Path path = fileSystem.getPath(normalPath); if (Files.isRegularFile(path)) - try (final LockedChannel channel = lockForWriting(path)) { - Files.delete(path); - } + ioPolicy.delete(normalPath); else { try (final Stream pathStream = Files.walk(path)) { for (final Iterator i = pathStream.sorted(Comparator.reverseOrder()).iterator(); i.hasNext();) { final Path childPath = i.next(); if (Files.isRegularFile(childPath)) - try (final LockedChannel channel = lockForWriting(childPath)) { - Files.delete(childPath); - } + ioPolicy.delete(childPath.toString()); else tryDelete(childPath); } @@ -355,7 +370,7 @@ public void delete(final String normalPath) throws N5IOException { protected static void tryDelete(final Path path) throws IOException { try { - Files.delete(path); + ioPolicy.delete(path.toString()); } catch (final DirectoryNotEmptyException e) { /* * Even though path is expected to be an empty directory, sometimes @@ -365,7 +380,7 @@ protected static void tryDelete(final Path path) throws IOException { try { /* wait and reattempt */ Thread.sleep(100); - Files.delete(path); + ioPolicy.delete(path.toString()); } catch (final InterruptedException ex) { e.printStackTrace(); Thread.currentThread().interrupt(); @@ -508,27 +523,23 @@ protected static void createAndCheckIsDirectory( } } - private static class FileLazyRead implements LazyRead { + static class FileLazyRead implements LazyRead { private static final Closeable NO_OP = () -> { }; private final Path path; private Closeable lock; - FileLazyRead(final Path path) { + FileLazyRead(final Path path) throws IOException { this(path, true); } - FileLazyRead(final Path path, final Boolean requireLock ) { + FileLazyRead(final Path path, final boolean requireLock ) throws IOException { this.path = path; - try { - lock = lockForReading(path); - } catch (Exception e) { - if (requireLock) { - throw e; - } + if (requireLock) + lock = FILE_LOCK_MANAGER.lockForReading(path); + else lock = NO_OP; - } } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java new file mode 100644 index 000000000..457896acd --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java @@ -0,0 +1,61 @@ +package org.janelia.saalfeldlab.n5; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.*; + +import static org.janelia.saalfeldlab.n5.FileKeyLockManager.FILE_LOCK_MANAGER; + +public class FsIoPolicy { + + static final IoPolicy atomicWithFallback = IoPolicy.withFallback(new Atomic(), new Unsafe()); + + public static class Unsafe implements IoPolicy { + @Override + public void write(String key, ReadData readData) throws IOException { + final Path path = Paths.get(key); + Files.copy(readData.inputStream(), path, StandardCopyOption.REPLACE_EXISTING); + } + + @Override + public VolatileReadData read(final String key) throws IOException { + final Path path = Paths.get(key); + FileSystemKeyValueAccess.FileLazyRead fileLazyRead = new FileSystemKeyValueAccess.FileLazyRead(path, false); + return VolatileReadData.from(fileLazyRead); + } + + @Override + public void delete(final String key) throws IOException { + final Path path = Paths.get(key); + Files.deleteIfExists(path); + } + } + + public static class Atomic implements IoPolicy { + @Override + public void write(String key, ReadData readData) throws IOException { + final Path path = Paths.get(key); + try (LockedFileChannel channel = FILE_LOCK_MANAGER.lockForWriting(path)) { + readData.writeTo(channel.newOutputStream()); + } + } + + @Override + public VolatileReadData read(String key) throws IOException { + final Path path = Paths.get(key); + FileSystemKeyValueAccess.FileLazyRead fileLazyRead = new FileSystemKeyValueAccess.FileLazyRead(path, true); + return VolatileReadData.from(fileLazyRead); + } + + @Override + public void delete(final String key) throws IOException { + final Path path = Paths.get(key); + try (LockedFileChannel ignore = FILE_LOCK_MANAGER.lockForWriting(path)) { + Files.delete(path); + } + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java b/src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java new file mode 100644 index 000000000..875b2a326 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java @@ -0,0 +1,47 @@ +package org.janelia.saalfeldlab.n5; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; + +import java.io.IOException; + +interface IoPolicy { + + void write(String key, ReadData readData) throws IOException; + + VolatileReadData read(String key) throws IOException; + + void delete(String key) throws IOException; + + static IoPolicy withFallback(IoPolicy primary, IoPolicy fallback) { + return new IoPolicy() { + + @Override + public void write(String key, ReadData readData) throws IOException { + try { + primary.write(key, readData); + } catch (IOException e) { + fallback.write(key, readData); + } + } + + @Override + public VolatileReadData read(String key) throws IOException { + try { + return primary.read(key); + } catch (IOException e) { + return fallback.read(key); + } + } + + @Override + public void delete(String key) throws IOException { + try { + primary.delete(key); + } catch (IOException e) { + fallback.delete(key); + } + } + }; + } +} From 63cb0c1c88b8fc6e1d6cc100954e2d5cdb0ef281 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 10 Feb 2026 16:12:37 -0500 Subject: [PATCH 13/21] feat: kva.createReadData throws N5NoSuchKeyException for missing keys, so we handle it as `null` in PositionValueAccess Signed-off-by: Caleb Hulbert --- .../n5/FileSystemKeyValueAccess.java | 21 +++---------------- .../saalfeldlab/n5/KeyValueAccess.java | 9 ++++---- .../n5/shard/PositionValueAccess.java | 6 +++++- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 19b5eacea..c853698c5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -28,26 +28,16 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.Closeable; -import java.nio.file.StandardOpenOption; +import java.io.*; +import java.nio.file.*; + import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.net.URI; import java.net.URISyntaxException; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.FileSystem; -import java.nio.file.FileSystemException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.attribute.FileAttribute; import java.util.Arrays; import java.util.Comparator; @@ -113,11 +103,6 @@ public VolatileReadData createReadData(final String normalPath) throws N5IOExcep @Override public void write(final String normalPath, final ReadData data) throws N5IOException { - final Path path = fileSystem.getPath(normalPath); - try (final LockedFileChannel channel = lockForWriting(path)) { - data.writeTo(channel.newOutputStream()); - } catch (IOException e) { - throw new N5IOException(e); try { ioPolicy.write(normalPath, data); } catch (final NoSuchFileException e) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index ac94567f5..a3802796c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -249,14 +249,15 @@ default URI uri(final String uriString) throws URISyntaxException { * If supported by this KeyValueAccess implementation, partial reads are * possible by {@link ReadData#slice slicing} the returned {@code ReadData}. *

- * If the requested key does not exist, either {@code null} is returned or a - * lazy {@code VolatileReadData} that will throw {@code N5NoSuchKeyException} - * when trying to materialize. + * The resulting {@code VolatileReadData} is potentially lazy. If the requested + * key does not exist, it will throw {@code N5NoSuchKeyException}. Whether + * the exception is thrown when {@link KeyValueAccess#createReadData(String)}] is called, + * or when trying to materialize the {@code VolatileReadData} is implementation dependent. * * @param normalPath * is expected to be in normalized form, no further efforts are made to normalize it * - * @return a ReadData or null + * @return a ReadData * * @throws N5IOException * if an error occurs diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java index 36a1645fb..80035b958 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java @@ -119,7 +119,11 @@ protected String absolutePath(final long... gridPosition) { @Override public VolatileReadData get(final long[] key) throws N5IOException { - return kva.createReadData(absolutePath(key)); + try { + return kva.createReadData(absolutePath(key)); + } catch (N5Exception.N5NoSuchKeyException e) { + return null; + } } @Override From 705a3935d0086658d00053d3a414873dfafbab6b Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 10 Feb 2026 16:13:03 -0500 Subject: [PATCH 14/21] chore: deprecate `lockForReading/Writing` methods in favor of `createReadData` and `write` Signed-off-by: Caleb Hulbert --- .../org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java | 4 ++++ src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index c853698c5..92bb126dc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -113,17 +113,20 @@ public void write(final String normalPath, final ReadData data) throws N5IOExcep } @Override + @Deprecated public LockedFileChannel lockForReading(final String normalPath) throws N5IOException { return lockForReading(fileSystem.getPath(normalPath)); } @Override + @Deprecated public LockedChannel lockForWriting(final String normalPath) throws N5IOException { return lockForWriting(fileSystem.getPath(normalPath)); } + @Deprecated protected static LockedFileChannel lockForReading(final Path path) throws N5IOException { try { @@ -135,6 +138,7 @@ protected static LockedFileChannel lockForReading(final Path path) throws N5IOEx } } + @Deprecated static LockedFileChannel lockForWriting(final Path path) throws N5IOException { try { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index a3802796c..e6465c5b8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -294,7 +294,9 @@ default URI uri(final String uriString) throws URISyntaxException { * @return the locked channel * @throws N5IOException * if a locked channel could not be created + * @deprecated migrate to {@link KeyValueAccess#createReadData(String)} */ + @Deprecated LockedChannel lockForReading( final String normalPath ) throws N5IOException; /** @@ -313,7 +315,9 @@ default URI uri(final String uriString) throws URISyntaxException { * @return the locked channel * @throws N5IOException * if a locked channel could not be created + * @deprecated migrate to {@link KeyValueAccess#write(String, ReadData)} */ + @Deprecated LockedChannel lockForWriting( final String normalPath ) throws N5IOException; /** From 72d7279943bf31404ced1caa73adefca37c3cd59 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 10 Feb 2026 16:13:47 -0500 Subject: [PATCH 15/21] fix(test): rename CodecSerialization so it's name matches he surefire regex Signed-off-by: Caleb Hulbert --- .../{CodecSerialization.java => CodecSerializationTest.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/test/java/org/janelia/saalfeldlab/n5/serialization/{CodecSerialization.java => CodecSerializationTest.java} (99%) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerializationTest.java similarity index 99% rename from src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java rename to src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerializationTest.java index 31861b250..b06cae606 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerializationTest.java @@ -47,7 +47,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; -public class CodecSerialization { +public class CodecSerializationTest { private Gson gson; From 5fdfd1c9f687b17d23254d0f1af9a41bcb1b8d33 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 10 Feb 2026 16:14:07 -0500 Subject: [PATCH 16/21] refactor(test): move lock specific tests to it's own file Signed-off-by: Caleb Hulbert --- .../janelia/saalfeldlab/n5/FsLockTest.java | 207 ++++++++++++++++++ .../org/janelia/saalfeldlab/n5/N5FSTest.java | 174 --------------- 2 files changed, 207 insertions(+), 174 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/FsLockTest.java diff --git a/src/test/java/org/janelia/saalfeldlab/n5/FsLockTest.java b/src/test/java/org/janelia/saalfeldlab/n5/FsLockTest.java new file mode 100644 index 000000000..89836200a --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/FsLockTest.java @@ -0,0 +1,207 @@ +package org.janelia.saalfeldlab.n5; + +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class FsLockTest { + + private static final FileKeyLockManager LOCK_MANAGER = FileKeyLockManager.FILE_LOCK_MANAGER; + + private static String tempPathName() { + + try { + final File tmpFile = Files.createTempDirectory("fs-key-lock-test-").toFile(); + tmpFile.delete(); + tmpFile.mkdir(); + tmpFile.deleteOnExit(); + return tmpFile.getCanonicalPath(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void testReadLock() throws IOException { + + final Path path = Paths.get(tempPathName(), "lock"); + path.toFile().createNewFile(); + assertTrue("File Created", path.toFile().exists()); + LockedChannel lock = LOCK_MANAGER.lockForReading(path); + lock.close(); + lock = LOCK_MANAGER.lockForReading(path); + + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future future = exec.submit(() -> { + LOCK_MANAGER.lockForWriting(path).close(); + return null; + }); + + try { + System.out.println("Trying to acquire locked readable channel..."); + System.out.println(future.get(3, TimeUnit.SECONDS)); + fail("Lock broken!"); + } catch (final TimeoutException e) { + System.out.println("Lock held!"); + future.cancel(true); + } catch (final InterruptedException | ExecutionException e) { + future.cancel(true); + System.out.println("Test was interrupted!"); + } finally { + lock.close(); + Files.delete(path); + } + + exec.shutdownNow(); + } + + @Test + public void testWriteLock() throws IOException { + + final Path path = Paths.get(tempPathName(), "lock"); + final LockedChannel lock = LOCK_MANAGER.lockForWriting(path); + System.out.println("locked"); + + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future future = exec.submit(() -> { + LOCK_MANAGER.lockForReading(path).close(); + return null; + }); + + try { + System.out.println("Trying to acquire locked writable channel..."); + System.out.println(future.get(3, TimeUnit.SECONDS)); + fail("Lock broken!"); + } catch (final TimeoutException e) { + System.out.println("Lock held!"); + future.cancel(true); + } catch (final InterruptedException | ExecutionException e) { + future.cancel(true); + System.out.println("Test was interrupted!"); + } finally { + lock.close(); + Files.delete(path); + } + + exec.shutdownNow(); + } + + @Test + public void testFSLockRelease() throws IOException, ExecutionException, InterruptedException, TimeoutException { + + + final Path path = Paths.get(tempPathName(), "lock"); + final ExecutorService exec = Executors.newFixedThreadPool(2); + + // first thread acquires the lock, waits for 200ms then should release it + exec.submit(() -> { + try { + try(final LockedChannel lock = LOCK_MANAGER.lockForWriting(path)) { + lock.newReader(); + Thread.sleep(200); + } + } catch (IOException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }); + + // second thread waits for the lock. + // it should get it within a few seconds. + final Future future = exec.submit(() -> { + LOCK_MANAGER.lockForWriting(path).close(); + return null; + }); + + future.get(3, TimeUnit.SECONDS); + Files.delete(path); + exec.shutdownNow(); + } + + @Test + public void testReadLockBehavior() throws IOException, InterruptedException, ExecutionException, TimeoutException { + + final Path path = Paths.get(tempPathName(), "read-lock"); + path.toFile().createNewFile(); + + final ExecutorService exec = Executors.newFixedThreadPool(3); + + final AtomicBoolean v = new AtomicBoolean(false); + + // first thread acquires a read lock, waits for 200ms + Future f = exec.submit(() -> { + try { + try(final LockedChannel lock = LOCK_MANAGER.lockForReading(path)) { + lock.newReader(); + Thread.sleep(200); + + // ensure that the other thread updated the value + assertTrue(v.get()); + + } + } catch (IOException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }); + + // second thread gets a read lock + // and should not be blocked + // this thread updates the boolean + exec.submit(() -> { + try( final LockedChannel lock = LOCK_MANAGER.lockForReading(path)) { + lock.newReader(); + v.set(true); + } + return null; + }); + + f.get(3, TimeUnit.SECONDS); + exec.shutdownNow(); + Files.delete(path); + } + + @Test + public void testWriteLockBehavior() throws IOException, ExecutionException, InterruptedException, TimeoutException { + + + final Path path = Paths.get(tempPathName(), "lock"); + final ExecutorService exec = Executors.newFixedThreadPool(2); + + // first thread acquires the lock, waits for 200ms then should release it + exec.submit(() -> { + try { + try(final LockedChannel lock = LOCK_MANAGER.lockForWriting(path)) { + lock.newReader(); + Thread.sleep(200); + } + } catch (IOException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }); + + // second thread waits for the lock. + // it should get it within a few seconds. + final Future future = exec.submit(() -> { + LOCK_MANAGER.lockForWriting(path).close(); + return null; + }); + + future.get(3, TimeUnit.SECONDS); + Files.delete(path); + exec.shutdownNow(); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 64bd88a3c..baca25ea9 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -41,7 +41,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.attribute.FileAttribute; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -49,9 +48,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import org.junit.Ignore; import org.junit.Test; import com.google.gson.GsonBuilder; @@ -64,8 +61,6 @@ */ public class N5FSTest extends AbstractN5Test { - private static final FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); - private static String tempN5PathName() { try { @@ -107,173 +102,4 @@ protected N5Reader createN5Reader( return new N5FSReader(location, gson); } - - @Test - public void testReadLock() throws IOException { - - final Path path = Paths.get(tempN5PathName(), "lock"); - LockedChannel lock = access.lockForWriting(path); - lock.close(); - lock = access.lockForReading(path); - System.out.println("locked"); - - final ExecutorService exec = Executors.newSingleThreadExecutor(); - final Future future = exec.submit(() -> { - access.lockForWriting(path).close(); - return null; - }); - - try { - System.out.println("Trying to acquire locked readable channel..."); - System.out.println(future.get(3, TimeUnit.SECONDS)); - fail("Lock broken!"); - } catch (final TimeoutException e) { - System.out.println("Lock held!"); - future.cancel(true); - } catch (final InterruptedException | ExecutionException e) { - future.cancel(true); - System.out.println("Test was interrupted!"); - } finally { - lock.close(); - Files.delete(path); - } - - exec.shutdownNow(); - } - - @Test - public void testWriteLock() throws IOException { - - final Path path = Paths.get(tempN5PathName(), "lock"); - final LockedChannel lock = access.lockForWriting(path); - System.out.println("locked"); - - final ExecutorService exec = Executors.newSingleThreadExecutor(); - final Future future = exec.submit(() -> { - access.lockForReading(path).close(); - return null; - }); - - try { - System.out.println("Trying to acquire locked writable channel..."); - System.out.println(future.get(3, TimeUnit.SECONDS)); - fail("Lock broken!"); - } catch (final TimeoutException e) { - System.out.println("Lock held!"); - future.cancel(true); - } catch (final InterruptedException | ExecutionException e) { - future.cancel(true); - System.out.println("Test was interrupted!"); - } finally { - lock.close(); - Files.delete(path); - } - - exec.shutdownNow(); - } - - @Test - public void testFSLockRelease() throws IOException, ExecutionException, InterruptedException, TimeoutException { - - - final Path path = Paths.get(tempN5PathName(), "lock"); - final ExecutorService exec = Executors.newFixedThreadPool(2); - - // first thread acquires the lock, waits for 200ms then should release it - exec.submit(() -> { - try( final LockedChannel lock = access.lockForWriting(path)) { - lock.newReader(); - Thread.sleep(200); - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - }); - - // second thread waits for the lock. - // it should get it within a few seconds. - final Future future = exec.submit(() -> { - access.lockForWriting(path).close(); - return null; - }); - - future.get(3, TimeUnit.SECONDS); - Files.delete(path); - exec.shutdownNow(); - } - - @Test - public void testReadLockBehavior() throws IOException, InterruptedException, ExecutionException, TimeoutException { - - final Path path = Paths.get(tempN5PathName(), "read-lock"); - path.toFile().createNewFile(); - - final ExecutorService exec = Executors.newFixedThreadPool(3); - - final AtomicBoolean v = new AtomicBoolean(false); - - // first thread acquires a read lock, waits for 200ms - Future f = exec.submit(() -> { - try( final LockedChannel lock = access.lockForReading(path)) { - lock.newReader(); - Thread.sleep(200); - - // ensure that the other thread updated the value - assertTrue(v.get()); - - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - }); - - // second thread gets a read lock - // and should not be blocked - // this thread updates the boolean - exec.submit(() -> { - try( final LockedChannel lock = access.lockForReading(path)) { - lock.newReader(); - v.set(true); - } - return null; - }); - - f.get(3, TimeUnit.SECONDS); - exec.shutdownNow(); - Files.delete(path); - } - - @Test - public void testWriteLockBehavior() throws IOException, ExecutionException, InterruptedException, TimeoutException { - - - final Path path = Paths.get(tempN5PathName(), "lock"); - final ExecutorService exec = Executors.newFixedThreadPool(2); - - // first thread acquires the lock, waits for 200ms then should release it - exec.submit(() -> { - try( final LockedChannel lock = access.lockForWriting(path)) { - lock.newReader(); - Thread.sleep(200); - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - }); - - // second thread waits for the lock. - // it should get it within a few seconds. - final Future future = exec.submit(() -> { - access.lockForWriting(path).close(); - return null; - }); - - future.get(3, TimeUnit.SECONDS); - Files.delete(path); - exec.shutdownNow(); - } - } From 13000b58d8ea48d7c87133755dd88a72a7ec51ac Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 10 Feb 2026 16:23:54 -0500 Subject: [PATCH 17/21] feat: don't throw on NoSuchFileException for delete Signed-off-by: Caleb Hulbert --- .../org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 92bb126dc..7e5343c3a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -351,6 +351,8 @@ public void delete(final String normalPath) throws N5IOException { } } } + } catch (NoSuchFileException ignore) { + /* It doesn't exist; that's sufficient for us to not complain on a `delete` call */ } catch (IOException | UncheckedIOException e) { throw new N5IOException("Failed to delete file at " + normalPath, e); } From 614ad705cecdb03e23d8ea1193e987552dae76e6 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 10 Feb 2026 16:25:05 -0500 Subject: [PATCH 18/21] BREAKING: remove `fileSystem` parameter/field from FileSystemKeyValueAccess. In practice, it was not consistently used. Most `Paths` and `Files` static methods internally get a file system from the FileSystemProvider Signed-off-by: Caleb Hulbert --- .../n5/FileSystemKeyValueAccess.java | 50 +++++++------------ .../janelia/saalfeldlab/n5/N5FSReader.java | 2 +- .../janelia/saalfeldlab/n5/N5FSWriter.java | 2 +- 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 7e5343c3a..cb9066fba 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -77,18 +77,6 @@ private static IoPolicy getIoPolicy() { } } - protected final FileSystem fileSystem; - - /** - * Opens a {@link FileSystemKeyValueAccess} with a {@link FileSystem}. - * - * @param fileSystem the file system - */ - public FileSystemKeyValueAccess(final FileSystem fileSystem) { - - this.fileSystem = fileSystem; - } - @Override public VolatileReadData createReadData(final String normalPath) throws N5IOException { @@ -116,14 +104,14 @@ public void write(final String normalPath, final ReadData data) throws N5IOExcep @Deprecated public LockedFileChannel lockForReading(final String normalPath) throws N5IOException { - return lockForReading(fileSystem.getPath(normalPath)); + return lockForReading(Paths.get(normalPath)); } @Override @Deprecated public LockedChannel lockForWriting(final String normalPath) throws N5IOException { - return lockForWriting(fileSystem.getPath(normalPath)); + return lockForWriting(Paths.get(normalPath)); } @Deprecated @@ -153,28 +141,28 @@ static LockedFileChannel lockForWriting(final Path path) throws N5IOException { @Override public boolean isDirectory(final String normalPath) { - final Path path = fileSystem.getPath(normalPath); + final Path path = Paths.get(normalPath); return Files.isDirectory(path); } @Override public boolean isFile(final String normalPath) { - final Path path = fileSystem.getPath(normalPath); + final Path path = Paths.get(normalPath); return Files.isRegularFile(path); } @Override public boolean exists(final String normalPath) { - final Path path = fileSystem.getPath(normalPath); + final Path path = Paths.get(normalPath); return Files.exists(path); } @Override public long size(final String normalPath) { - return size(fileSystem.getPath(normalPath)); + return size(Paths.get(normalPath)); } protected static long size(final Path path) { @@ -191,7 +179,7 @@ protected static long size(final Path path) { @Override public String[] listDirectories(final String normalPath) throws N5IOException { - final Path path = fileSystem.getPath(normalPath); + final Path path = Paths.get(normalPath); try (final Stream pathStream = Files.list(path)) { return pathStream .filter(Files::isDirectory) @@ -207,7 +195,7 @@ public String[] listDirectories(final String normalPath) throws N5IOException { @Override public String[] list(final String normalPath) throws N5IOException { - final Path path = fileSystem.getPath(normalPath); + final Path path = Paths.get(normalPath); try (final Stream pathStream = Files.list(path)) { return pathStream .map(a -> path.relativize(a).toString()) @@ -222,8 +210,9 @@ public String[] list(final String normalPath) throws N5IOException { @Override public String[] components(final String path) { - final Path fsPath = fileSystem.getPath(path); + final Path fsPath = Paths.get(path); final Path root = fsPath.getRoot(); + String separator = root.getFileSystem().getSeparator(); final String[] components; int o; if (root == null) { @@ -239,7 +228,6 @@ public String[] components(final String path) { String name = fsPath.getName(i - o).toString(); /* Preserve trailing slash on final component if present*/ if (i == components.length - 1) { - final String separator = fileSystem.getSeparator(); final String trailingSeparator = path.endsWith(separator) ? separator : path.endsWith("/") ? "/" : ""; name += trailingSeparator; } @@ -251,7 +239,7 @@ public String[] components(final String path) { @Override public String parent(final String path) { - final Path parent = fileSystem.getPath(path).getParent(); + final Path parent = Paths.get(path).getParent(); if (parent == null) return null; else @@ -261,8 +249,8 @@ public String parent(final String path) { @Override public String relativize(final String path, final String base) { - final Path basePath = fileSystem.getPath(base); - return basePath.relativize(fileSystem.getPath(path)).toString(); + final Path basePath = Paths.get(base); + return basePath.relativize(Paths.get(path)).toString(); } /** @@ -277,7 +265,7 @@ public String relativize(final String path, final String base) { @Override public String normalize(final String path) { - return fileSystem.getPath(path).normalize().toString(); + return Paths.get(path).normalize().toString(); } @Override @@ -300,8 +288,8 @@ public String compose(final String... components) { if (components == null || components.length == 0) return null; if (components.length == 1) - return fileSystem.getPath(components[0]).toString(); - return fileSystem.getPath(components[0], Arrays.copyOfRange(components, 1, components.length)).normalize().toString(); + return Paths.get(components[0]).toString(); + return Paths.get(components[0], Arrays.copyOfRange(components, 1, components.length)).normalize().toString(); } @Override public String compose(URI uri, String... components) { @@ -324,7 +312,7 @@ public String compose(final String... components) { public void createDirectories(final String normalPath) throws N5IOException { try { - createDirectories(fileSystem.getPath(normalPath)); + createDirectories(Paths.get(normalPath)); } catch (NoSuchFileException e) { throw new N5NoSuchKeyException("No such file", e); } catch (IOException | UncheckedIOException e) { @@ -336,13 +324,13 @@ public void createDirectories(final String normalPath) throws N5IOException { public void delete(final String normalPath) throws N5IOException { try { - final Path path = fileSystem.getPath(normalPath); + final Path path = Paths.get(normalPath); if (Files.isRegularFile(path)) ioPolicy.delete(normalPath); else { try (final Stream pathStream = Files.walk(path)) { - for (final Iterator i = pathStream.sorted(Comparator.reverseOrder()).iterator(); i.hasNext();) { + for (final Iterator i = pathStream.sorted(Comparator.reverseOrder()).iterator(); i.hasNext(); ) { final Path childPath = i.next(); if (Files.isRegularFile(childPath)) ioPolicy.delete(childPath.toString()); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 4397b4544..70904544a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -69,7 +69,7 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final bo throws N5Exception { super( - new FileSystemKeyValueAccess(FileSystems.getDefault()), + new FileSystemKeyValueAccess(), basePath, gsonBuilder, cacheMeta); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index d856ebb22..3a57881dd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -70,7 +70,7 @@ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final bo throws N5Exception { super( - new FileSystemKeyValueAccess(FileSystems.getDefault()), + new FileSystemKeyValueAccess(), basePath, gsonBuilder, cacheAttributes); From c02cd7902d7cf317c3622d0f1c69cc7d2b357495 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 10 Feb 2026 16:34:53 -0500 Subject: [PATCH 19/21] cleanup(test): cleanup uses of old FileSystemKVA constructor Signed-off-by: Caleb Hulbert --- .../janelia/saalfeldlab/n5/N5CachedFSTest.java | 2 +- .../n5/benchmarks/N5BlockWriteBenchmarks.java | 2 +- .../n5/benchmarks/ReadDataBenchmarks.java | 2 +- .../n5/kva/FileSystemKeyValueAccessTest.java | 2 +- .../saalfeldlab/n5/readdata/ReadDataTests.java | 2 +- .../janelia/saalfeldlab/n5/shard/ShardTest.java | 15 +++++---------- 6 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 4a7a94b8b..cb9467b74 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -171,7 +171,7 @@ public void cacheBehaviorTest() throws IOException, URISyntaxException { final String loc = tempN5Location(); // make an uncached n5 writer - try (final N5TrackingStorage n5 = new N5TrackingStorage(new FileSystemKeyValueAccess(FileSystems.getDefault()), loc, + try (final N5TrackingStorage n5 = new N5TrackingStorage(new FileSystemKeyValueAccess(), loc, new GsonBuilder(), true)) { cacheBehaviorHelper(n5); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java index 01802d879..ffa35156f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java @@ -113,7 +113,7 @@ public void setup() { File tmpDir; try { tmpDir = Files.createTempDirectory("n5-blockWriteBenchmark-").toFile(); - FileSystemKeyValueAccess kva = new FileSystemKeyValueAccess(FileSystems.getDefault()); + FileSystemKeyValueAccess kva = new FileSystemKeyValueAccess(); n5 = new N5KeyValueWriter(kva, tmpDir.getAbsolutePath(), new GsonBuilder(), true); int[] blockSize = new int[numDimensions]; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java index 9dcc81193..19254700d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java @@ -109,7 +109,7 @@ protected Path getPath() { public void setup() throws IOException { random = new Random(); - kva = new FileSystemKeyValueAccess(FileSystems.getDefault()); + kva = new FileSystemKeyValueAccess(); basePath = Files.createTempDirectory("ReadDataBenchmark-"); tmpPaths = new ArrayList<>(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java index ff1bf517f..9f017a133 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java @@ -57,7 +57,7 @@ public class FileSystemKeyValueAccessTest extends AbstractKeyValueAccessTest { private static String separator = FileSystems.getDefault().getSeparator(); - private static final FileSystemKeyValueAccess fileSystemKva = new FileSystemKeyValueAccess(FileSystems.getDefault()); + private static final FileSystemKeyValueAccess fileSystemKva = new FileSystemKeyValueAccess(); @Override protected KeyValueAccess newKeyValueAccess(URI root) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index 220a5a497..c4f3fa51b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -113,7 +113,7 @@ public void testFileKvaReadData() throws IOException { os.write(data); } - try( final VolatileReadData readData = new FileSystemKeyValueAccess(FileSystems.getDefault()) + try( final VolatileReadData readData = new FileSystemKeyValueAccess() .createReadData(tmpF.getAbsolutePath())) { assertEquals("file read data length", -1, readData.length()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index a4118d753..97d45cc6c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -37,12 +37,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; +import java.nio.file.*; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -691,7 +686,7 @@ public static class TrackingN5Writer extends N5KeyValueWriter { final TrackingFileSystemKeyValueAccess tkva; public TrackingN5Writer(String basePath) { - super( new TrackingFileSystemKeyValueAccess(FileSystems.getDefault()), basePath, new GsonBuilder(), false); + super( new TrackingFileSystemKeyValueAccess(), basePath, new GsonBuilder(), false); tkva = (TrackingFileSystemKeyValueAccess)getKeyValueAccess(); } @@ -732,8 +727,8 @@ private static class TrackingFileSystemKeyValueAccess extends FileSystemKeyValue private int numIsFileCalls = 0; private long totalBytesRead = 0; - protected TrackingFileSystemKeyValueAccess(FileSystem fileSystem) { - super(fileSystem); + protected TrackingFileSystemKeyValueAccess() { + super(); } @Override @@ -745,7 +740,7 @@ public boolean isFile(String normalPath) { @Override public VolatileReadData createReadData(final String normalPath) { try { - return VolatileReadData.from(new TrackingFileLazyRead(fileSystem.getPath(normalPath))); + return VolatileReadData.from(new TrackingFileLazyRead(Paths.get(normalPath))); } catch (N5NoSuchKeyException e) { // return VolatileReadData.from(new NoSuchKeyLazyRead()); return null; From b6832cab4854b4a76d4412e188ce79ad11e129df Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 10 Feb 2026 16:42:10 -0500 Subject: [PATCH 20/21] fix: root patch may be null, get separator from fsPath Signed-off-by: Caleb Hulbert --- .../org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index cb9066fba..0bf0fbe06 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -212,7 +212,7 @@ public String[] components(final String path) { final Path fsPath = Paths.get(path); final Path root = fsPath.getRoot(); - String separator = root.getFileSystem().getSeparator(); + final String separator = fsPath.getFileSystem().getSeparator(); final String[] components; int o; if (root == null) { From 8ab32fce20a9780bf126c60c8581d2e8fa95f29e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 11 Feb 2026 09:28:47 -0500 Subject: [PATCH 21/21] feat: make IoPolicy public for use in other KVA implementations Signed-off-by: Caleb Hulbert --- src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java b/src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java index 875b2a326..b1dcb3044 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java @@ -5,7 +5,7 @@ import java.io.IOException; -interface IoPolicy { +public interface IoPolicy { void write(String key, ReadData readData) throws IOException;