From 0f062cd7532b92f50d37442badc8915ef2898736 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 6 Mar 2026 09:56:46 -0500 Subject: [PATCH 01/41] fix: don't lock when deleting directories --- src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java index 5d01dd919..2a3e26a6e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java @@ -69,6 +69,8 @@ public VolatileReadData read(String key) throws IOException { @Override public void delete(final String key) throws IOException { final Path path = Paths.get(key); + if (!Files.isRegularFile(path)) + Files.delete(path); try (LockedFileChannel ignore = FILE_LOCK_MANAGER.lockForWriting(path)) { Files.delete(path); } From 1de4a7b1d5b312f8eaebb7e2a9571e14418d1e41 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 9 Mar 2026 10:36:51 -0400 Subject: [PATCH 02/41] fix: ensure the VolatileReadData closes on InputStream close Signed-off-by: Caleb Hulbert --- .../saalfeldlab/n5/BufferedKvaLockedChannel.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java index 70ed4ba60..495eb8fbd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java @@ -1,6 +1,8 @@ package org.janelia.saalfeldlab.n5; +import org.apache.commons.io.input.ProxyInputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import java.io.*; @@ -23,7 +25,15 @@ public Reader newReader() throws N5Exception.N5IOException { @Override public InputStream newInputStream() throws N5Exception.N5IOException { - return kva.createReadData(key).inputStream(); + + VolatileReadData volatileReadData = kva.createReadData(key); + return new ProxyInputStream(volatileReadData.inputStream()) { + @Override + public void close() throws IOException { + volatileReadData.close(); + super.close(); + } + }; } @Override From e463dc17aab4a165826ab0f0953df67fe6614a9e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 13 Mar 2026 10:27:08 -0400 Subject: [PATCH 03/41] fix: ignore UncheckedIOException too for `closeQuietly` --- src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java b/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java index 520a2b367..c8c6fc691 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java @@ -2,6 +2,7 @@ import java.io.Closeable; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; @@ -126,8 +127,8 @@ private static void closeQuietly(final FileChannel fileChannel) { if (fileChannel != null) { try { fileChannel.close(); - } catch (final IOException ignored) { + } catch (final IOException | UncheckedIOException ignored) { } - } + } } } From 957a1edca26fc9ac7d97a2ec2fb61a514f8ccb27 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 13 Mar 2026 10:27:54 -0400 Subject: [PATCH 04/41] fix: don't throw exception after a successful read of data from a channel when an exception is thrown during `close()` --- .../java/org/janelia/saalfeldlab/n5/FsIoPolicy.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java index 2a3e26a6e..196cbc2ab 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java @@ -112,6 +112,7 @@ public ReadData materialize(final long offset, final long length) { throw new N5Exception.N5IOException("FileLazyRead is already closed."); } + ReadData readData = null; try (final FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { channel.position(offset); @@ -129,13 +130,19 @@ public ReadData materialize(final long offset, final long length) { final byte[] data = new byte[(int) size]; final ByteBuffer buf = ByteBuffer.wrap(data); channel.read(buf); - return ReadData.from(data); + readData = ReadData.from(data); } catch (final NoSuchFileException e) { throw new N5Exception.N5NoSuchKeyException("No such file", e); } catch (IOException | UncheckedIOException e) { - throw new N5Exception.N5IOException(e); + /* Occasionally (frequently for some source remote mounted file systems) this can throw exceptions during + * `channel.close()` which is called automatically in the try-with-resources block. In this case, we have + * successfully read the data, and we can return it, and ignore the exception. + * */ + if (readData == null) + throw new N5Exception.N5IOException(e); } + return readData; } @Override From eab40eee13f699be8494530e4006d944cc80e546 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 13 Mar 2026 10:31:26 -0400 Subject: [PATCH 05/41] fix: close order inside-out --- .../org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java index 495eb8fbd..59b7a6534 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java @@ -30,8 +30,8 @@ public InputStream newInputStream() throws N5Exception.N5IOException { return new ProxyInputStream(volatileReadData.inputStream()) { @Override public void close() throws IOException { - volatileReadData.close(); super.close(); + volatileReadData.close(); } }; } From 6f78537c71715602c7cf7a79cf13e1dccb20b139 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 16 Mar 2026 11:53:04 -0400 Subject: [PATCH 06/41] refactor!: block -> chunk, shard -> block --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 14 +-- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 28 +++--- .../org/janelia/saalfeldlab/n5/N5Reader.java | 35 ++++---- .../org/janelia/saalfeldlab/n5/N5Writer.java | 60 ++++++------- .../saalfeldlab/n5/shard/DatasetAccess.java | 53 +++++++---- .../n5/shard/DefaultDatasetAccess.java | 71 ++++++--------- .../saalfeldlab/n5/AbstractN5Test.java | 90 +++++++++---------- .../janelia/saalfeldlab/n5/N5Benchmark.java | 6 +- .../saalfeldlab/n5/N5ReadBenchmark.java | 4 +- .../n5/backward/CompatibilityTest.java | 8 +- .../n5/backward/CreateSampleData.java | 8 +- .../n5/benchmarks/N5BlockWriteBenchmarks.java | 6 +- .../saalfeldlab/n5/codec/BlockCodecTests.java | 4 +- .../n5/http/HttpReaderFsWriter.java | 24 ++--- .../n5/shard/DatasetAccessTest.java | 52 +++++------ .../saalfeldlab/n5/shard/ShardTest.java | 61 +++++++------ .../saalfeldlab/n5/shard/WriteShardTest.java | 14 +-- .../saalfeldlab/n5/shard/WriteShardTest2.java | 12 +-- .../n5/shard/WriteShardTestTruncate.java | 12 +-- 19 files changed, 281 insertions(+), 281 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 0e40a1c88..25eb01994 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -94,7 +94,7 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { } @Override - default DataBlock readBlock( + default DataBlock readChunk( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { @@ -103,7 +103,7 @@ default DataBlock readBlock( try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), convertedDatasetAttributes); - return convertedDatasetAttributes. getDatasetAccess().readBlock(posKva, gridPosition); + return convertedDatasetAttributes. getDatasetAccess().readChunk(posKva, gridPosition); } catch (N5Exception.N5NoSuchKeyException e) { return null; @@ -111,18 +111,18 @@ default DataBlock readBlock( } @Override - default List> readBlocks( + default List> readChunks( final String pathName, final DatasetAttributes datasetAttributes, final List blockPositions) throws N5Exception { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), convertedDatasetAttributes); - return convertedDatasetAttributes. getDatasetAccess().readBlocks(posKva, blockPositions); + return convertedDatasetAttributes. getDatasetAccess().readChunks(posKva, blockPositions); } @Override - default DataBlock readShard( + default DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { @@ -132,7 +132,7 @@ default DataBlock readShard( try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), convertedDatasetAttributes); - return convertedDatasetAttributes. getDatasetAccess().readShard(posKva, gridPosition, shardLevel); + return convertedDatasetAttributes. getDatasetAccess().readBlock(posKva, gridPosition, shardLevel); } catch (N5Exception.N5NoSuchKeyException e) { return null; @@ -172,7 +172,7 @@ default String absoluteAttributesPath(final String normalPath) { } @Override - default boolean shardExists( + default boolean blockExists( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index f5bacc94f..0a2e87089 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -257,7 +257,7 @@ default void writeRegion( } @Override - default void writeBlocks( + default void writeChunks( final String datasetPath, final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { @@ -265,15 +265,15 @@ default void writeBlocks( DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes); - convertedDatasetAttributes.getDatasetAccess().writeBlocks(posKva, Arrays.asList(dataBlocks)); + convertedDatasetAttributes.getDatasetAccess().writeChunks(posKva, Arrays.asList(dataBlocks)); } catch (final UncheckedIOException e) { throw new N5IOException( - "Failed to write blocks into dataset " + datasetPath, e); + "Failed to write chunks into dataset " + datasetPath, e); } } @Override - default void writeBlock( + default void writeChunk( final String path, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception { @@ -281,28 +281,28 @@ default void writeBlock( DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes); - convertedDatasetAttributes. getDatasetAccess().writeBlock(posKva, dataBlock); + convertedDatasetAttributes. getDatasetAccess().writeChunk(posKva, dataBlock); } catch (final UncheckedIOException e) { throw new N5IOException( - "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, + "Failed to write chunk " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, e); } } @Override - default void writeShard( + default void writeBlock( final String path, final DatasetAttributes datasetAttributes, - final DataBlock shard) throws N5Exception { + final DataBlock dataBlock) throws N5Exception { final DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); final int shardLevel = convertedDatasetAttributes.getNestedBlockGrid().numLevels() - 1; try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes); - convertedDatasetAttributes. getDatasetAccess().writeShard(posKva, shard, shardLevel); + convertedDatasetAttributes. getDatasetAccess().writeBlock(posKva, dataBlock, shardLevel); } catch (final UncheckedIOException e) { throw new N5IOException( - "Failed to write block " + Arrays.toString(shard.getGridPosition()) + " into dataset " + path, + "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, e); } } @@ -320,22 +320,22 @@ default boolean remove(final String path) throws N5Exception { } @Override - default boolean deleteBlock( + default boolean deleteChunk( final String path, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); - return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition); + return datasetAttributes.getDatasetAccess().deleteChunk(posKva, gridPosition); } @Override - default boolean deleteBlocks( + default boolean deleteChunks( final String path, final DatasetAttributes datasetAttributes, final List gridPositions) throws N5Exception { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); - return datasetAttributes.getDatasetAccess().deleteBlocks(posKva, gridPositions); + return datasetAttributes.getDatasetAccess().deleteChunks(posKva, gridPositions); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index aebbfa67d..c5103e140 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -298,7 +298,7 @@ default DatasetAttributes getConvertedDatasetAttributes(final DatasetAttributes /** - * Reads a {@link DataBlock}. + * Reads a chunk as a {@link DataBlock}. * * @param * the DataBlock data type @@ -312,13 +312,13 @@ default DatasetAttributes getConvertedDatasetAttributes(final DatasetAttributes * @throws N5Exception * the exception */ - DataBlock readBlock( + DataBlock readChunk( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception; /** - * Reads multiple {@link DataBlock}s. + * Reads multiple chunks as {@link DataBlock}s. *

* Implementations may optimize / batch read operations when possible, e.g. * in the case that the datasets are sharded. @@ -335,7 +335,7 @@ DataBlock readBlock( * @throws N5Exception * the exception */ - default List> readBlocks( + default List> readChunks( final String pathName, final DatasetAttributes datasetAttributes, final List gridPositions) throws N5Exception { @@ -343,16 +343,16 @@ default List> readBlocks( DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); final ArrayList> blocks = new ArrayList<>(); for( final long[] p : gridPositions ) - blocks.add(readBlock(pathName, convertedDatasetAttributes, p)); + blocks.add(readChunk(pathName, convertedDatasetAttributes, p)); return blocks; } /** - * Reads a shard as a {@link DataBlock}. + * Reads a block, returning a {@link DataBlock}. Will be a chunk or shard (if the dataset is sharded). *

- * A "shard" is the largest level of the datasets {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid} - * This method's behavior is identical to readBlock for un-sharded datasets. + * A block is the highest (coarsest) level of the dataset's {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid} + * This method's behavior is identical to {@link #readChunk} for un-sharded datasets. * * @param * the DataBlock data type @@ -361,39 +361,40 @@ default List> readBlocks( * @param datasetAttributes * the dataset attributes * @param gridPosition - * the position in the shard grid + * the position in the block grid * @return the data block * @throws N5Exception * the exception * * @see DatasetAttributes#getNestedBlockGrid() */ - DataBlock readShard( + DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception; /** - * Checks if a shard (or block for non-sharded datasets) exists at the - * given grid position without reading the data. + * Checks if a block exists at the given grid position without reading the data. *

* This method only checks for the presence of the key value for the gridPosition, it does not - * read or validate the contents. + * read or validate the contents. As a result, this method refers to chunks in un-sharded datasets + * or shards in sharded datasets. * * @param pathName * dataset path * @param datasetAttributes * the dataset attributes * @param gridPosition - * the shard grid position (or block position for non-sharded datasets) - * @return true if the shard (or block) file exists + * the block grid position + * @return true if the block file exists * @throws N5Exception * the exception */ - boolean shardExists( + boolean blockExists( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception; + /** * Load a {@link DataBlock} as a {@link Serializable}. The offset is given * in @@ -419,7 +420,7 @@ default T readSerializedBlock( final long... gridPosition) throws N5Exception, ClassNotFoundException { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(attributes); - final DataBlock block = readBlock(dataset, convertedDatasetAttributes, gridPosition); + final DataBlock block = readChunk(dataset, convertedDatasetAttributes, gridPosition); if (block == null) return null; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 22ab44667..4c248bec6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -241,29 +241,29 @@ default DatasetAttributes createDataset( } /** - * Writes a {@link DataBlock}. + * Writes a chunk represented by a {@link DataBlock}. * * @param datasetPath dataset path * @param datasetAttributes the dataset attributes - * @param dataBlock the data block + * @param dataBlock the chunk as a DataBlock * @param the data block data type * @throws N5Exception the exception */ - void writeBlock( + void writeChunk( final String datasetPath, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception; /** - * Write multiple data blocks, useful for request aggregation. + * Write multiple chunks represented by {@link DataBlock}s, useful for aggregation. * * @param datasetPath dataset path * @param datasetAttributes the dataset attributes - * @param dataBlocks the data block + * @param dataBlocks the chunks * @param the data block data type * @throws N5Exception the exception */ - default void writeBlocks( + default void writeChunks( final String datasetPath, final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { @@ -271,15 +271,15 @@ default void writeBlocks( // default method is naive DatasetAttributes convertedAttributes = getConvertedDatasetAttributes(datasetAttributes); for (DataBlock block : dataBlocks) { - writeBlock(datasetPath, convertedAttributes, block); + writeChunk(datasetPath, convertedAttributes, block); } } /** - * Writes a shard stored as a {@link DataBlock}. + * Writes a block stored as a {@link DataBlock}. *

- * A "shard" is the largest level of the datasets {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid}. - * This method's behavior is identical to writeBlock for un-sharded datasets. + * A block is the highest (coarsest) level of the dataset's {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid}. + * This method's behavior is identical to {@link #writeChunk} for un-sharded datasets. * * @param pathName dataset path * @param datasetAttributes the dataset attributes @@ -289,7 +289,7 @@ default void writeBlocks( * * @see DatasetAttributes#getNestedBlockGrid() */ - void writeShard( + void writeBlock( final String pathName, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception; @@ -345,51 +345,51 @@ void writeRegion( ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException; /** - * Deletes the block at {@code gridPosition}. + * Deletes the chunk at {@code gridPosition}. * * @param datasetPath dataset path - * @param gridPosition position of block to be deleted - * @throws N5Exception if the block exists but could not be deleted + * @param gridPosition position of chunk to be deleted + * @throws N5Exception if the chunk exists but could not be deleted * - * @return {@code true} if the block at {@code gridPosition} existed and was deleted. + * @return {@code true} if the chunk at {@code gridPosition} existed and was deleted. */ - default boolean deleteBlock( + default boolean deleteChunk( final String datasetPath, final long... gridPosition) throws N5Exception { final DatasetAttributes datasetAttributes = getDatasetAttributes(datasetPath); - return deleteBlock(datasetPath, datasetAttributes, gridPosition); + return deleteChunk(datasetPath, datasetAttributes, gridPosition); } /** - * Deletes the block at {@code gridPosition}. + * Deletes the chunk at {@code gridPosition}. * * @param datasetPath the dataset path * @param datasetAttributes the dataset attributes - * @param gridPosition position of block to be deleted - * @throws N5Exception if the block exists but could not be deleted + * @param gridPosition position of chunk to be deleted + * @throws N5Exception if the chunk exists but could not be deleted * - * @return {@code true} if the block at {@code gridPosition} existed and was deleted. + * @return {@code true} if the chunk at {@code gridPosition} existed and was deleted. */ - boolean deleteBlock( + boolean deleteChunk( String datasetPath, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception; /** - * Deletes the blocks at the given {@code gridPositions}. + * Deletes the chunks at the given {@code gridPositions}. * * @param datasetPath dataset path * @param gridPositions a list of grid positions - * @return {@code true} if any of the specified blocks existed and was deleted - * @throws N5Exception if any of the block exists but could not be deleted + * @return {@code true} if any of the specified chunks existed and was deleted + * @throws N5Exception if any of the chunks did exist but could not be deleted */ - default boolean deleteBlocks( + default boolean deleteChunks( String datasetPath, DatasetAttributes datasetAttributes, List gridPositions) throws N5Exception { boolean deleted = false; for (long[] pos : gridPositions) { - deleted |= deleteBlock(datasetPath, datasetAttributes, pos); + deleted |= deleteChunk(datasetPath, datasetAttributes, pos); } return deleted; } @@ -419,6 +419,6 @@ default void writeSerializedBlock( } final byte[] bytes = byteOutputStream.toByteArray(); final DataBlock dataBlock = new ByteArrayDataBlock(null, gridPosition, bytes); - writeBlock(datasetPath, datasetAttributes, dataBlock); + writeChunk(datasetPath, datasetAttributes, dataBlock); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index b09df6782..f6d158261 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -38,7 +38,7 @@ import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; /** - * Wrap an instantiated DataBlock/shard codec hierarchy to implement (single and + * Wrap an instantiated DataBlock/block codec hierarchy to implement (single and * batch) DataBlock read/write methods. * * @param @@ -63,13 +63,13 @@ public interface DatasetAccess { * @throws N5IOException * if any error occurs while reading or decoding the block */ - DataBlock readBlock(PositionValueAccess pva, long[] gridPosition) throws N5IOException; + DataBlock readChunk(PositionValueAccess pva, long[] gridPosition) throws N5IOException; /** * Read the {@code DataBlock}s at the given {@code gridPositions}. *

* The returned {@code List>} is in the same order as the - * requested {@code gridPositions}. That is, the {@code DataBlock} at indwex + * requested {@code gridPositions}. That is, the {@code DataBlock} at index * {@code i} has grid coordinates {@code gridPositions.get(i)}. *

* If a requested block doesn't exist, then the corresponding element in the @@ -80,21 +80,36 @@ public interface DatasetAccess { * @param pva * dataset storage * @param gridPositions - * list of grid position of the DataBlocks to read + * list of grid positions of the DataBlocks to read * @return list of DataBlocks * * @throws N5IOException * if any error occurs while reading or decoding blocks */ - List> readBlocks(PositionValueAccess pva, List gridPositions) throws N5IOException; + List> readChunks(PositionValueAccess pva, List gridPositions) throws N5IOException; - void writeBlock(PositionValueAccess pva, DataBlock dataBlock) throws N5IOException; + /** + * Writes a chunk to the given storage position. + * + */ + void writeChunk(PositionValueAccess pva, DataBlock dataBlock) throws N5IOException; - void writeBlocks(PositionValueAccess pva, List> blocks) throws N5IOException; + /** + * Writes multiple chunks to the given storage positions. + */ + void writeChunks(PositionValueAccess pva, List> blocks) throws N5IOException; - boolean deleteBlock(PositionValueAccess pva, long[] gridPosition) throws N5IOException; + /** + * Deletes the chunk at {@code gridPosition}. @return true if it existed and + * was deleted. + */ + boolean deleteChunk(PositionValueAccess pva, long[] gridPosition) throws N5IOException; - boolean deleteBlocks(PositionValueAccess pva, List positions) throws N5IOException; + /** + * Deletes the chunks at the given {@code positions}. @return true if any + * existed and were deleted. + */ + boolean deleteChunks(PositionValueAccess pva, List positions) throws N5IOException; /** * @@ -106,7 +121,7 @@ public interface DatasetAccess { * @param blocks * is asked to create blocks within the given region * @param writeFully - * if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * if false, merge existing data in blocks/chunks that overlap the region boundary. if true, override everything. * * @throws N5IOException */ @@ -128,9 +143,9 @@ void writeRegion( * @param blocks * is asked to create blocks within the given region. must be thread-safe. * @param writeFully - * if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * if false, merge existing data in blocks/chunks that overlap the region boundary. if true, override everything. * @param exec - * used to parallelize over blocks and shards + * used to parallelize over chunks and blocks * * @throws N5Exception * @throws InterruptedException @@ -146,7 +161,7 @@ void writeRegion( ) throws N5Exception, InterruptedException, ExecutionException; /** - * Read a full shard at {@code shardGridPosition} at the given nesting + * Read a block at {@code shardGridPosition} at the given nesting * {@code level}. The shard data is rearranged and assembled into a (large) * {@code DataBlock}. *

@@ -163,27 +178,27 @@ void writeRegion( * @return * @throws N5IOException */ - DataBlock readShard(PositionValueAccess pva, long[] shardGridPosition, int level) throws N5IOException; + DataBlock readBlock(PositionValueAccess pva, long[] shardGridPosition, int level) throws N5IOException; /** - * Write a full shard the given nesting {@code level}. The shard data is + * Write a full block at the given nesting {@code level}. The block data is * given as a (large) {@code DataBlock} that will be sliced, rearranged, and * written as (level-0) DataBlocks. *

* {@code dataBlock.getGridPosition()} is the grid position with respect to * {@code level}. For example, if {@code level==1}, then this refers to the - * position on the shard grid. + * position on the block grid. * * @param pva * dataset storage * @param dataBlock - * shard/block to write + * block to write * @param level - * grid level of the shard/block to write. + * grid level of the block to write. * * @throws N5IOException */ - void writeShard(PositionValueAccess pva, DataBlock dataBlock, int level) throws N5IOException; + void writeBlock(PositionValueAccess pva, DataBlock dataBlock, int level) throws N5IOException; NestedGrid getGrid(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index fc1db3609..3c652a353 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -71,20 +71,17 @@ public NestedGrid getGrid() { return grid; } - // - // -- readBlock ----------------------------------------------------------- - @Override - public DataBlock readBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { + public DataBlock readChunk(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { final NestedPosition position = grid.nestedPosition(gridPosition); try (final VolatileReadData readData = pva.get(position.key())) { - return readBlockRecursive(readData, position, grid.numLevels() - 1); + return readChunkRecursive(readData, position, grid.numLevels() - 1); } catch (N5NoSuchKeyException ignored) { return null; } } - private DataBlock readBlockRecursive( + private DataBlock readChunkRecursive( final ReadData readData, final NestedPosition position, final int level) { @@ -98,19 +95,16 @@ private DataBlock readBlockRecursive( @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[level]; final RawShard shard = codec.decode(readData, position.absolute(level)).getData(); - return readBlockRecursive(shard.getElementData(position.relative(level - 1)), position, level - 1); + return readChunkRecursive(shard.getElementData(position.relative(level - 1)), position, level - 1); } } - // - // -- readBlocks ---------------------------------------------------------- - @Override - public List> readBlocks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { + public List> readChunks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { // for non-sharded datasets, just read the blocks individually if (grid.numLevels() == 1) { - return gridPositions.stream().map(pos -> readBlock(pva, pos)).collect(Collectors.toList()); + return gridPositions.stream().map(pos -> readChunk(pva, pos)).collect(Collectors.toList()); } // Create a list of DataBlockRequests and sort it such that requests @@ -122,7 +116,7 @@ public List> readBlocks(final PositionValueAccess pva, final List subRequests : split) { final long[] key = subRequests.relativeGridPosition(); try (final VolatileReadData readData = pva.get(key)) { - readBlocksRecursive(readData, subRequests); + readChunksRecursive(readData, subRequests); } catch (N5NoSuchKeyException ignored) { // the key didn't exist (as we found out when lazy-reading the index). // we don't have to do anything: all subRequest blocks remain null. @@ -139,7 +133,7 @@ public List> readBlocks(final PositionValueAccess pva, final List requests ) { @@ -164,23 +158,20 @@ private void readBlocksRecursive( for (final DataBlockRequest request : requests) { final long[] elementPos = request.position.relative(0); final ReadData elementData = shard.getElementData(elementPos); - request.block = readBlockRecursive(elementData, request.position, 0); + request.block = readChunkRecursive(elementData, request.position, 0); } } else { // level > 1 final List> split = requests.split(); for (final DataBlockRequests subRequests : split) { final long[] subShardPosition = subRequests.relativeGridPosition(); final ReadData elementData = shard.getElementData(subShardPosition); - readBlocksRecursive(elementData, subRequests); + readChunksRecursive(elementData, subRequests); } } } - // - // -- writeBlock ---------------------------------------------------------- - @Override - public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException { + public void writeChunk(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException { if (grid.numLevels() == 1) { @SuppressWarnings("unchecked") @@ -224,11 +215,8 @@ private ReadData writeBlockRecursive( } } - // - // -- writeBlocks --------------------------------------------------------- - @Override - public void writeBlocks(final PositionValueAccess pva, final List> dataBlocks) throws N5IOException { + public void writeChunks(final PositionValueAccess pva, final List> dataBlocks) throws N5IOException { if (grid.numLevels() == 1) { @SuppressWarnings("unchecked") @@ -245,7 +233,7 @@ public void writeBlocks(final PositionValueAccess pva, final List> final long[] shardKey = subRequests.relativeGridPosition(); final ReadData modifiedData; try (final VolatileReadData existingData = writeFully ? null : pva.get(shardKey)) { - modifiedData = writeBlocksRecursive(existingData, subRequests); + modifiedData = writeChunksRecursive(existingData, subRequests); // Here, we are about to write the shard data, but with the new blocks modified. // Need to make sure that the read operations happen now before pva.set acquires a write lock modifiedData.materialize(); @@ -261,7 +249,7 @@ public void writeBlocks(final PositionValueAccess pva, final List> * @param existingReadData encoded existing shard data (to decode and partially override) * @param requests for blocks within the shard to be written */ - private ReadData writeBlocksRecursive( + private ReadData writeChunksRecursive( final ReadData existingReadData, // may be null final DataBlockRequests requests ) { @@ -289,7 +277,7 @@ private ReadData writeBlocksRecursive( final boolean nestedWriteFully = writeFully || subRequests.coversShard(); final long[] elementPos = subRequests.relativeGridPosition(); final ReadData existingElementData = nestedWriteFully ? null : shard.getElementData(elementPos); - final ReadData modifiedElementData = writeBlocksRecursive(existingElementData, subRequests); + final ReadData modifiedElementData = writeChunksRecursive(existingElementData, subRequests); shard.setElementData(modifiedElementData, elementPos); } } @@ -297,9 +285,6 @@ private ReadData writeBlocksRecursive( return codec.encode(new RawShardDataBlock(gridPos, shard)); } - // - // -- writeRegion --------------------------------------------------------- - @Override public void writeRegion( final PositionValueAccess pva, @@ -416,7 +401,7 @@ private ReadData writeRegionRecursive( // -- deleteBlock --------------------------------------------------------- @Override - public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { + public boolean deleteChunk(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { if (grid.numLevels() == 1) { // for non-sharded dataset, don't bother getting the value, just remove the key. return pva.remove(gridPosition); @@ -425,7 +410,7 @@ public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosit final long[] key = position.key(); final ReadData modifiedData; try (final VolatileReadData existingData = pva.get(key)) { - modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); + modifiedData = deleteChunkRecursive(existingData, position, grid.numLevels() - 1); if (modifiedData == existingData) { // nothing changed, the blocks we wanted to delete didn't exist anyway return false; @@ -448,7 +433,7 @@ public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosit } } - private ReadData deleteBlockRecursive( + private ReadData deleteChunkRecursive( final ReadData existingReadData, final NestedPosition position, final int level) throws N5NoSuchKeyException { @@ -466,7 +451,7 @@ private ReadData deleteBlockRecursive( // This shard remains unchanged. return existingReadData; } else { - final ReadData modifiedElementData = deleteBlockRecursive(existingElementData, position, level - 1); + final ReadData modifiedElementData = deleteChunkRecursive(existingElementData, position, level - 1); if (modifiedElementData == existingElementData) { // The nested shard was not modified. // This shard remains unchanged. @@ -490,7 +475,7 @@ private ReadData deleteBlockRecursive( // -- deleteBlocks -------------------------------------------------------- @Override - public boolean deleteBlocks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { + public boolean deleteChunks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { // for non-sharded datasets, just delete the blocks individually if (grid.numLevels() == 1) { @@ -652,14 +637,14 @@ private ReadData deleteBlocksRecursive( // easily revisit and change this heuristic. @Override - public DataBlock readShard( + public DataBlock readBlock( final PositionValueAccess pva, final long[] shardGridPosition, final int level ) throws N5IOException { if (level == 0) { - return readBlock(pva, shardGridPosition); + return readChunk(pva, shardGridPosition); } final long[] shardPixelPos = grid.pixelPosition(shardGridPosition, level); @@ -671,10 +656,10 @@ public DataBlock readShard( for (int d = 0; d < n; ++d) { shardSizeInPixels[d] = Math.min(defaultShardSize[d], (int) (datasetSize[d] - shardPixelPos[d])); } - return readShardInternal(pva, shardGridPosition, shardSizeInPixels, level); + return readBlockInternal(pva, shardGridPosition, shardSizeInPixels, level); } - private DataBlock readShardInternal( + private DataBlock readBlockInternal( final PositionValueAccess pva, final long[] shardGridPosition, final int[] shardSizeInPixels, // expected size of this shard in pixels @@ -705,7 +690,7 @@ private DataBlock readShardInternal( // read all blocks in (gridMin, gridMax) and filter out missing blocks final List blockPositions = Region.gridPositions(gridMin, gridMax); - final List> blocks = readBlocks(pva, blockPositions) + final List> blocks = readChunks(pva, blockPositions) .stream().filter(Objects::nonNull).collect(Collectors.toList()); if (blocks.isEmpty()) { return null; @@ -735,14 +720,14 @@ private DataBlock readShardInternal( // -- writeShard ---------------------------------------------------------- @Override - public void writeShard( + public void writeBlock( final PositionValueAccess pva, final DataBlock dataBlock, final int level ) throws N5IOException { if (level == 0) { - writeBlock(pva, dataBlock); + writeChunk(pva, dataBlock); return; } @@ -792,7 +777,7 @@ public void writeShard( blocks.add(block); } - writeBlocks(pva, blocks); + writeChunks(pva, blocks); } // diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index dc12d3943..15f197ff0 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -265,10 +265,10 @@ public void testBlocksLargerThanDimensions() { } final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(largeBlockSize, new long[]{0, 0, 0}, data); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); // Read the block back - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertNotNull("Block should be readable", loadedDataBlock); assertArrayEquals("Block size should match", largeBlockSize, loadedDataBlock.getSize()); assertArrayEquals("Block data should match", data, (byte[])loadedDataBlock.getData()); @@ -294,9 +294,9 @@ public void testUnalignedBlocksTruncatedAtEnd() { data0[i] = i + 1000; } final IntArrayDataBlock dataBlock0 = new IntArrayDataBlock(truncatedBlockSize0, new long[]{1, 0, 0}, data0); - n5.writeBlock(datasetName, attributes, dataBlock0); + n5.writeChunk(datasetName, attributes, dataBlock0); - final DataBlock loadedBlock0 = n5.readBlock(datasetName, attributes, 1, 0, 0); + final DataBlock loadedBlock0 = n5.readChunk(datasetName, attributes, 1, 0, 0); assertNotNull("Truncated block should be readable", loadedBlock0); assertArrayEquals("Truncated block data should match", data0, (int[])loadedBlock0.getData()); @@ -308,9 +308,9 @@ public void testUnalignedBlocksTruncatedAtEnd() { data1[i] = i + 2000; } final IntArrayDataBlock dataBlock1 = new IntArrayDataBlock(truncatedBlockSize1, new long[]{0, 2, 0}, data1); - n5.writeBlock(datasetName, attributes, dataBlock1); + n5.writeChunk(datasetName, attributes, dataBlock1); - final DataBlock loadedBlock1 = n5.readBlock(datasetName, attributes, 0, 2, 0); + final DataBlock loadedBlock1 = n5.readChunk(datasetName, attributes, 0, 2, 0); assertNotNull("Truncated block should be readable", loadedBlock1); assertArrayEquals("Truncated block data should match", data1, (int[])loadedBlock1.getData()); @@ -322,9 +322,9 @@ public void testUnalignedBlocksTruncatedAtEnd() { data2[i] = i + 3000; } final IntArrayDataBlock dataBlock2 = new IntArrayDataBlock(truncatedBlockSize2, new long[]{0, 0, 4}, data2); - n5.writeBlock(datasetName, attributes, dataBlock2); + n5.writeChunk(datasetName, attributes, dataBlock2); - final DataBlock loadedBlock2 = n5.readBlock(datasetName, attributes, 0, 0, 4); + final DataBlock loadedBlock2 = n5.readChunk(datasetName, attributes, 0, 0, 4); assertNotNull("Truncated block should be readable", loadedBlock2); assertArrayEquals("Truncated block data should match", data2, (int[])loadedBlock2.getData()); } @@ -343,9 +343,9 @@ public void testWriteReadByteBlock() { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, new long[]{0, 0, 0}, byteBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); } } @@ -365,9 +365,9 @@ public void testWriteReadStringBlock() { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final StringDataBlock dataBlock = new StringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0L, 0L, 0L); + final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0L, 0L, 0L); assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); } @@ -386,9 +386,9 @@ public void testWriteReadShortBlock() { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData()); } @@ -408,9 +408,9 @@ public void testWriteReadIntBlock() { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final IntArrayDataBlock dataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertArrayEquals(intBlock, (int[])loadedDataBlock.getData()); } @@ -430,9 +430,9 @@ public void testWriteReadLongBlock() { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize, new long[]{0, 0, 0}, longBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertArrayEquals(longBlock, (long[])loadedDataBlock.getData()); } @@ -448,9 +448,9 @@ public void testWriteReadFloatBlock() { n5.createDataset(datasetName, dimensions, blockSize, DataType.FLOAT32, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final FloatArrayDataBlock dataBlock = new FloatArrayDataBlock(blockSize, new long[]{0, 0, 0}, floatBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertArrayEquals(floatBlock, (float[])loadedDataBlock.getData(), 0.001f); } @@ -465,9 +465,9 @@ public void testWriteReadDoubleBlock() { n5.createDataset(datasetName, dimensions, blockSize, DataType.FLOAT64, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final DoubleArrayDataBlock dataBlock = new DoubleArrayDataBlock(blockSize, new long[]{0, 0, 0}, doubleBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertArrayEquals(doubleBlock, (double[])loadedDataBlock.getData(), 0.001); } @@ -487,14 +487,14 @@ public void testWriteReadShard() { final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock); - n5.writeShard(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); // read with readShard - final DataBlock loadedShard = n5.readShard(datasetName, attributes, 0, 0, 0); + final DataBlock loadedShard = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(shortBlock, (short[])loadedShard.getData()); // read with readBlock - final DataBlock loadedDataBlock = n5.readShard(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData()); } } @@ -515,9 +515,9 @@ public void testMode1WriteReadByteBlock() { n5.createDataset(datasetName, dimensions, differentBlockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(differentBlockSize, new long[]{0, 0, 0}, byteBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); } @@ -581,22 +581,22 @@ public void testWriteInvalidBlock() { // write a block that is too large final ByteArrayDataBlock bigDataBlock = new ByteArrayDataBlock(biggerBlockSize, new long[]{0, 0, 0}, biggerData); - n5.writeBlock(datasetName, attributes, bigDataBlock); + n5.writeChunk(datasetName, attributes, bigDataBlock); - final DataBlock loadedBigDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + final DataBlock loadedBigDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertArrayEquals(biggerData, (byte[])loadedBigDataBlock.getData()); // write a block that is too small final ByteArrayDataBlock smallDataBlock = new ByteArrayDataBlock(smallerBlockSize, new long[]{0, 0, 0}, smallerData); - n5.writeBlock(datasetName, attributes, smallDataBlock); + n5.writeChunk(datasetName, attributes, smallDataBlock); - final DataBlock loadedSmallDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + final DataBlock loadedSmallDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertArrayEquals(smallerData, (byte[])loadedSmallDataBlock.getData()); // write a block of the wrong type final FloatArrayDataBlock floatDataBlock = new FloatArrayDataBlock(blockSize, new long[]{0, 0, 0}, floatData); assertThrows(ClassCastException.class, () -> { - n5.writeBlock(datasetName, attributes, floatDataBlock); + n5.writeChunk(datasetName, attributes, floatDataBlock); }); } } @@ -610,15 +610,15 @@ public void testOverwriteBlock() { final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final IntArrayDataBlock randomDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); - n5.writeBlock(datasetName, attributes, randomDataBlock); - final DataBlock loadedRandomDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + n5.writeChunk(datasetName, attributes, randomDataBlock); + final DataBlock loadedRandomDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertArrayEquals(intBlock, (int[])loadedRandomDataBlock.getData()); // test the case where the resulting file becomes shorter (because the data compresses better) final int[] emptyBlock = new int[DataBlock.getNumElements(blockSize)]; final IntArrayDataBlock emptyDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, emptyBlock); - n5.writeBlock(datasetName, attributes, emptyDataBlock); - final DataBlock loadedEmptyDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + n5.writeChunk(datasetName, attributes, emptyDataBlock); + final DataBlock loadedEmptyDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertArrayEquals(emptyBlock, (int[])loadedEmptyDataBlock.getData()); } } @@ -1052,7 +1052,7 @@ public void testDeepList() throws ExecutionException, InterruptedException { final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64); final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize, new long[]{0, 0, 0}, new long[blockNumElements]); n5.createDataset(datasetName, datasetAttributes); - n5.writeBlock(datasetName, datasetAttributes, dataBlock); + n5.writeChunk(datasetName, datasetAttributes, dataBlock); final List datasetList = Arrays.asList(n5.deepList("/")); for (final String subGroup : subGroupNames) @@ -1296,24 +1296,24 @@ public void testDelete() { final long[] position2 = {0, 1, 2}; // no blocks should exist to begin with - assertNull(n5.readBlock(datasetName, attributes, position1)); + assertNull(n5.readChunk(datasetName, attributes, position1)); final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, position1, byteBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); // block should exist at position1 but not at position2 - final DataBlock readBlock = n5.readBlock(datasetName, attributes, position1); + final DataBlock readBlock = n5.readChunk(datasetName, attributes, position1); assertNotNull(readBlock); assertTrue(readBlock instanceof ByteArrayDataBlock); assertArrayEquals(byteBlock, ((ByteArrayDataBlock)readBlock).getData()); - assertTrue("deleting existing block should return true", n5.deleteBlock(datasetName, position1)); - assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position1)); - assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position2)); + assertTrue("deleting existing block should return true", n5.deleteChunk(datasetName, position1)); + assertFalse("deleting non-existing block should return false", n5.deleteChunk(datasetName, position1)); + assertFalse("deleting non-existing block should return false", n5.deleteChunk(datasetName, position2)); // no block should exist anymore - assertNull(n5.readBlock(datasetName, attributes, position1)); - assertNull(n5.readBlock(datasetName, attributes, position2)); + assertNull(n5.readChunk(datasetName, attributes, position1)); + assertNull(n5.readChunk(datasetName, attributes, position2)); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java index 546a4072f..198aeae71 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java @@ -141,7 +141,7 @@ public void testDocExample() { n5.createDataset(compressedDatasetName, new long[]{1, 2, 3}, new int[]{1, 2, 3}, DataType.UINT16, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(compressedDatasetName); final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(new int[]{1, 2, 3}, new long[]{0, 0, 0}, dataBlockData); - n5.writeBlock(compressedDatasetName, attributes, dataBlock); + n5.writeChunk(compressedDatasetName, attributes, dataBlock); } catch (final N5Exception e) { fail(e.getMessage()); } @@ -165,7 +165,7 @@ public void benchmarkWritingSpeed() { for (int y = 0; y < nBlocks; ++y) for (int x = 0; x < nBlocks; ++x) { final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(new int[]{64, 64, 64}, new long[]{x, y, z}, data); - n5.writeBlock(compressedDatasetName, attributes, dataBlock); + n5.writeChunk(compressedDatasetName, attributes, dataBlock); } } catch (final N5Exception e) { fail(e.getMessage()); @@ -255,7 +255,7 @@ public void benchmarkParallelWritingSpeed() { exec.submit( () -> { final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(new int[]{64, 64, 64}, new long[]{fx, fy, fz}, data); - n5.writeBlock(compressedDatasetName, attributes, dataBlock); + n5.writeChunk(compressedDatasetName, attributes, dataBlock); return true; })); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java index ecf7643a7..0b959fb5c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java @@ -114,7 +114,7 @@ public void setup() { n5.createDataset(datasetName, dimensions, blockSize, DataType.FLOAT64, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final DoubleArrayDataBlock dataBlock = new DoubleArrayDataBlock(blockSize, new long[] {0, 0, 0}, doubleBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); } n5 = new N5FSReader(basePath, new GsonBuilder()); @@ -129,7 +129,7 @@ public void setup() { @BenchmarkMode( Mode.AverageTime ) @OutputTimeUnit( TimeUnit.MILLISECONDS ) public void bench() { - n5.readBlock(datasetName, attributes, 0, 0, 0); + n5.readChunk(datasetName, attributes, 0, 0, 0); } public static void main( final String... args ) throws RunnerException, IOException diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java index bb579c813..e5a705422 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java @@ -76,7 +76,7 @@ public void backwardReadHelper(final String base, final String dsetPath) throws byte value = 0; long[] p = new long[2]; - DataBlock b00 = n5.readBlock(dsetPath, attrs, p); + DataBlock b00 = n5.readChunk(dsetPath, attrs, p); assertNotNull(b00); assertArrayEquals(new int[]{5,4}, b00.getSize()); assertArrayEquals(expectedData(20, value), b00.getData()); @@ -84,7 +84,7 @@ public void backwardReadHelper(final String base, final String dsetPath) throws p[0] = 1; p[1] = 0; value++; - DataBlock b10 = n5.readBlock(dsetPath, attrs, p); + DataBlock b10 = n5.readChunk(dsetPath, attrs, p); assertNotNull(b10); assertArrayEquals(new int[]{2,4}, b10.getSize()); assertArrayEquals(expectedData(8, value), b10.getData()); @@ -92,7 +92,7 @@ public void backwardReadHelper(final String base, final String dsetPath) throws p[0] = 0; p[1] = 1; value++; - DataBlock b01 = n5.readBlock(dsetPath, attrs, p); + DataBlock b01 = n5.readChunk(dsetPath, attrs, p); assertNotNull(b01); assertArrayEquals(new int[]{5,1}, b01.getSize()); assertArrayEquals(expectedData(5, value), b01.getData()); @@ -100,7 +100,7 @@ public void backwardReadHelper(final String base, final String dsetPath) throws p[0] = 1; p[1] = 1; value++; - DataBlock b11 = n5.readBlock(dsetPath, attrs, p); + DataBlock b11 = n5.readChunk(dsetPath, attrs, p); assertNotNull(b11); assertArrayEquals(new int[]{2,1}, b11.getSize()); assertArrayEquals(expectedData(2, value), b11.getData()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java index 032534b1e..c40b67f26 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java @@ -62,28 +62,28 @@ public static N5FSWriter createSampleData(String baseDir, String dataset, Compre byte val = 0; long[] pos = new long[]{0, 0}; - n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + n5.writeChunk(dsetPath, attrs, createDataBlock(blkSize, pos, val)); pos[0] = 1; pos[1] = 0; blkSize[0] = 2; blkSize[1] = 4; val++; - n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + n5.writeChunk(dsetPath, attrs, createDataBlock(blkSize, pos, val)); pos[0] = 0; pos[1] = 1; blkSize[0] = 5; blkSize[1] = 1; val++; - n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + n5.writeChunk(dsetPath, attrs, createDataBlock(blkSize, pos, val)); pos[0] = 1; pos[1] = 1; blkSize[0] = 2; blkSize[1] = 1; val++; - n5.writeBlock(dsetPath, attrs, createDataBlock( blkSize, pos, val )); + n5.writeChunk(dsetPath, attrs, createDataBlock( blkSize, pos, val )); return n5; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java index ffa35156f..9961906fe 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java @@ -138,7 +138,7 @@ public void setup() { blocks.add(blk); // write data into the read group - n5.writeBlock(readGroup, dsetAttrs, blk); + n5.writeChunk(readGroup, dsetAttrs, blk); } } catch (final IOException e) { @@ -151,7 +151,7 @@ public void setup() { public void writeBenchmark() throws IOException { blocks.forEach(blk -> { - n5.writeBlock(writeGroup, dsetAttrs, blk); + n5.writeChunk(writeGroup, dsetAttrs, blk); }); } @@ -161,7 +161,7 @@ public void readBenchmark(Blackhole hole) throws IOException { final long[] p = new long[numDimensions]; for (int i = 0; i < numBlocks; i++) { p[0] = i; - hole.consume(n5.readBlock(readGroup, dsetAttrs, p)); + hole.consume(n5.readChunk(readGroup, dsetAttrs, p)); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java index 1e37b496b..c0d40b10f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java @@ -159,8 +159,8 @@ public void testEmptyBlock() throws Exception { // Test encode/decode final ByteArrayDataBlock emptyBlock = new ByteArrayDataBlock(blockSize, gridPosition, new byte[0]); - access.writeBlock(store, emptyBlock); - final DataBlock decoded = access.readBlock(store, gridPosition); + access.writeChunk(store, emptyBlock); + final DataBlock decoded = access.readChunk(store, gridPosition); assertEquals("Empty block should have 0 elements", 0, decoded.getNumElements()); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index fe13f0d68..661d556cf 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -116,9 +116,9 @@ public HttpRead return reader.getDatasetAttributes(pathName); } - @Override public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { + @Override public DataBlock readChunk(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { - return reader.readBlock(pathName, getConvertedDatasetAttributes(datasetAttributes), gridPosition); + return reader.readChunk(pathName, getConvertedDatasetAttributes(datasetAttributes), gridPosition); } @Override public T readSerializedBlock(String dataset, DatasetAttributes attributes, long... gridPosition) throws N5Exception, ClassNotFoundException { @@ -272,22 +272,22 @@ public HttpRead return convertedDatasetAttributes; } - @Override public void writeBlock(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { - writer.writeBlock(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); + @Override public void writeChunk(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { + writer.writeChunk(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); } - @Override public void writeShard(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { - writer.writeShard(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); + @Override public void writeBlock(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { + writer.writeBlock(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); } - @Override public boolean deleteBlock(String datasetPath, long... gridPosition) throws N5Exception { + @Override public boolean deleteChunk(String datasetPath, long... gridPosition) throws N5Exception { - return writer.deleteBlock(datasetPath, gridPosition); + return writer.deleteChunk(datasetPath, gridPosition); } - @Override public boolean deleteBlocks(String datasetPath, DatasetAttributes datasetAttributes, List gridPositions) throws N5Exception { + @Override public boolean deleteChunks(String datasetPath, DatasetAttributes datasetAttributes, List gridPositions) throws N5Exception { - return writer.deleteBlocks(datasetPath, datasetAttributes, gridPositions); + return writer.deleteChunks(datasetPath, datasetAttributes, gridPositions); } @Override public void writeSerializedBlock(Serializable object, String datasetPath, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { @@ -315,8 +315,8 @@ public HttpRead writer.writeAttributes(normalGroupPath, attributes); } - @Override public void writeBlocks(String datasetPath, DatasetAttributes datasetAttributes, DataBlock... dataBlocks) throws N5Exception { + @Override public void writeChunks(String datasetPath, DatasetAttributes datasetAttributes, DataBlock... dataBlocks) throws N5Exception { - writer.writeBlocks(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlocks); + writer.writeChunks(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlocks); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java index 1d3becba7..9d54874c0 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java @@ -103,20 +103,20 @@ public void testWriteReadIndividual() { final PositionValueAccess store = new TestPositionValueAccess(); // write some blocks, filled with constant values - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 0, 0}, 1)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 0, 0}, 2)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 1, 0}, 3)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 1, 0}, 4)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {3, 2, 1}, 5)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {8, 4, 1}, 6)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {0, 0, 0}, 1)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {1, 0, 0}, 2)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {0, 1, 0}, 3)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {1, 1, 0}, 4)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {3, 2, 1}, 5)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {8, 4, 1}, 6)); // verify that the written blocks can be read back with the correct values - checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), true, 1); - checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); - checkBlock(datasetAccess.readBlock(store, new long[] {0, 1, 0}), true, 3); - checkBlock(datasetAccess.readBlock(store, new long[] {1, 1, 0}), true, 4); - checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5); - checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6); + checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), true, 1); + checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); + checkBlock(datasetAccess.readChunk(store, new long[] {0, 1, 0}), true, 3); + checkBlock(datasetAccess.readChunk(store, new long[] {1, 1, 0}), true, 4); + checkBlock(datasetAccess.readChunk(store, new long[] {3, 2, 1}), true, 5); + checkBlock(datasetAccess.readChunk(store, new long[] {8, 4, 1}), true, 6); } @Test @@ -132,13 +132,13 @@ public void testWriteReadBulk() { for (int i = 0; i < writeGridPositions.size(); i++) { writeBlocks.add(createDataBlock(dataBlockSize, writeGridPositions.get(i), 1 + i)); } - datasetAccess.writeBlocks(store, writeBlocks); + datasetAccess.writeChunks(store, writeBlocks); // verify that the written blocks can be read back with the correct values final List readGridPositions = Arrays.asList(new long[][] { {1, 0, 0}, {0, 0, 0}, {0, 1, 0}, {2, 4, 2}, {3, 2, 1}, {8, 4, 1} }); - final List> readBlocks = datasetAccess.readBlocks(store, readGridPositions); + final List> readBlocks = datasetAccess.readChunks(store, readGridPositions); checkBlock(readBlocks.get(0), true, 2); checkBlock(readBlocks.get(1), true, 1); checkBlock(readBlocks.get(2), true, 3); @@ -160,20 +160,20 @@ public void testDeleteBlock() { for (int i = 0; i < writeGridPositions.size(); i++) { writeBlocks.add(createDataBlock(dataBlockSize, writeGridPositions.get(i), 1 + i)); } - datasetAccess.writeBlocks(store, writeBlocks); + datasetAccess.writeChunks(store, writeBlocks); // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) - datasetAccess.deleteBlock(store, new long[] {0, 0, 0}); - checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); - checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); + datasetAccess.deleteChunk(store, new long[] {0, 0, 0}); + checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), false, 1); + checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); // if a shard becomes empty the corresponding key should be deleted assertTrue(keyExists(store, new long[] {1, 0, 0})); - datasetAccess.deleteBlock(store, new long[] {8, 4, 1}); + datasetAccess.deleteChunk(store, new long[] {8, 4, 1}); assertFalse(keyExists(store, new long[] {1, 0, 0})); // deleting a non-existent block should not fail - datasetAccess.deleteBlock(store, new long[] {0, 0, 8}); + datasetAccess.deleteChunk(store, new long[] {0, 0, 8}); } private boolean keyExists(final PositionValueAccess store, final long[] key) { @@ -200,20 +200,20 @@ public void testDeleteBlocks() { for (int i = 0; i < writeGridPositions.size(); i++) { writeBlocks.add(createDataBlock(dataBlockSize, writeGridPositions.get(i), 1 + i)); } - datasetAccess.writeBlocks(store, writeBlocks); + datasetAccess.writeChunks(store, writeBlocks); // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) - datasetAccess.deleteBlocks(store, Arrays.asList(new long[][] {{0, 0, 0}, {4, 2, 2}, {3, 2, 1}})); - checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); - checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); + datasetAccess.deleteChunks(store, Arrays.asList(new long[][] {{0, 0, 0}, {4, 2, 2}, {3, 2, 1}})); + checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), false, 1); + checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); // if a shard becomes empty the corresponding key should be deleted assertTrue(keyExists(store, new long[] {1, 0, 0})); - datasetAccess.deleteBlocks(store, Arrays.asList(new long[][] {{8, 4, 1}})); + datasetAccess.deleteChunks(store, Arrays.asList(new long[][] {{8, 4, 1}})); assertFalse(keyExists(store, new long[] {1, 0, 0})); // deleting a non-existent block should not fail - datasetAccess.deleteBlocks(store, Arrays.asList(new long[] {0, 0, 8})); + datasetAccess.deleteChunks(store, Arrays.asList(new long[] {0, 0, 8})); } private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index d881df45f..2e1fb9fd7 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -177,7 +177,7 @@ public void writeReadBlocksTest() { data[i] = (byte)((100) + (10) + i); } - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ @@ -214,7 +214,7 @@ public void writeReadBlocksTest() { final long[][] blockIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; for (long[] blockIndex : blockIndices) { - final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); + final DataBlock block = writer.readChunk(dataset, datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); } @@ -223,7 +223,7 @@ public void writeReadBlocksTest() { data2[i] = (byte)(10 + i); } - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ @@ -255,16 +255,16 @@ public void writeReadBlocksTest() { final long[][] oldBlockIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; for (long[] blockIndex : oldBlockIndices) { - final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); + final DataBlock block = writer.readChunk(dataset, datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); } final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; final List newBlockIndexList = Arrays.asList(newBlockIndices); - final List> readBlocks = writer.readBlocks(dataset, datasetAttributes, newBlockIndexList); + final List> readBlocks = writer.readChunks(dataset, datasetAttributes, newBlockIndexList); for (int i = 0; i < newBlockIndices.length; i++) { final long[] blockIndex = newBlockIndices[i]; - final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); + final DataBlock block = writer.readChunk(dataset, datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])block.getData()); final DataBlock blockFromReadBlocks = readBlocks.get(i); Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])blockFromReadBlocks.getData()); @@ -324,14 +324,14 @@ public void writeShardDataSizeTest() { * and every entry should be null. */ final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; - final List> readBlocks = writer.readBlocks(dataset, datasetAttributes, Arrays.asList(newBlockIndices)); + final List> readBlocks = writer.readChunks(dataset, datasetAttributes, Arrays.asList(newBlockIndices)); assertEquals(newBlockIndices.length, readBlocks.size()); assertTrue("readBlocks for empty shard: all blocks null", readBlocks.stream().allMatch(blk -> blk == null)); /* * Now write blocks */ - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ @@ -400,15 +400,15 @@ public void writeReadBlockTest() { for (int i = 0; i < data.length; i++) { data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); } - writer.writeBlock(dataset, datasetAttributes, dataBlock); + writer.writeChunk(dataset, datasetAttributes, dataBlock); - final DataBlock block = writer.readBlock(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); + final DataBlock block = writer.readChunk(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); for (Map.Entry entry : writtenBlocks.entrySet()) { final long[] otherGridPosition = entry.getKey(); final byte[] otherData = entry.getValue(); - final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); + final DataBlock otherBlock = writer.readChunk(dataset, datasetAttributes, otherGridPosition); Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); } @@ -426,7 +426,6 @@ public void writeReadShardTest() { final int shardN = 16; final int[] blockSize = new int[] {2,2}; - final int blockN = 4; final String dataset = "writeReadShard"; DatasetAttributes attrs = getTestAttributes(DataType.INT32, new long[]{8, 8}, shardSize, blockSize); @@ -434,9 +433,9 @@ public void writeReadShardTest() { final int[] shardData = range(shardN); IntArrayDataBlock shard = new IntArrayDataBlock(shardSize, new long[]{0, 0}, shardData); - n5.writeShard(dataset, attrs, shard); - DataBlock readShard = n5.readShard(dataset, attrs, 0, 0); - assertArrayEquals(shardData, readShard.getData()); + n5.writeBlock(dataset, attrs, shard); + DataBlock readBlock = n5.readBlock(dataset, attrs, 0, 0); + assertArrayEquals(shardData, readBlock.getData()); /** @@ -451,12 +450,12 @@ public void writeReadShardTest() { * 12 13 | 14 15 */ - assertArrayEquals(new int[]{0, 1, 4, 5}, (int[])n5.readBlock(dataset, attrs, 0, 0).getData()); - assertArrayEquals(new int[]{2, 3, 6, 7}, (int[])n5.readBlock(dataset, attrs, 1, 0).getData()); - assertArrayEquals(new int[]{8, 9, 12, 13}, (int[])n5.readBlock(dataset, attrs, 0, 1).getData()); - assertArrayEquals(new int[]{10, 11, 14, 15}, (int[])n5.readBlock(dataset, attrs, 1, 1).getData()); + assertArrayEquals(new int[]{0, 1, 4, 5}, (int[])n5.readChunk(dataset, attrs, 0, 0).getData()); + assertArrayEquals(new int[]{2, 3, 6, 7}, (int[])n5.readChunk(dataset, attrs, 1, 0).getData()); + assertArrayEquals(new int[]{8, 9, 12, 13}, (int[])n5.readChunk(dataset, attrs, 0, 1).getData()); + assertArrayEquals(new int[]{10, 11, 14, 15}, (int[])n5.readChunk(dataset, attrs, 1, 1).getData()); - n5.deleteBlock(dataset, attrs, new long[]{1, 1}); + n5.deleteChunk(dataset, attrs, new long[]{1, 1}); /** * After deleting block (1,1) @@ -467,15 +466,15 @@ public void writeReadShardTest() { * 8 9 | 0 0 * 12 13 | 0 0 */ - final DataBlock partlyEmptyShard = n5.readShard(dataset, attrs, 0, 0); + final DataBlock partlyEmptyShard = n5.readBlock(dataset, attrs, 0, 0); assertArrayEquals(new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 12, 13, 0, 0}, partlyEmptyShard.getData()); // Delete the rest of the blocks - n5.deleteBlocks(dataset, attrs, + n5.deleteChunks(dataset, attrs, Stream.of( new long[] {0,0}, new long[] {1,0}, new long[] {0,1}).collect(Collectors.toList())); - assertNull(n5.readShard(dataset, attrs, 0, 0)); + assertNull(n5.readBlock(dataset, attrs, 0, 0)); } } @@ -505,7 +504,7 @@ public void numReadsTest() { data[i] = (byte)((100) + (10) + i); } - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ @@ -523,7 +522,7 @@ public void numReadsTest() { ); writer.resetNumMaterializeCalls(); - writer.readBlocks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0})); + writer.readChunks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0})); System.out.println(writer.getNumMaterializeCalls()); ArrayList ptList = new ArrayList<>(); @@ -533,7 +532,7 @@ public void numReadsTest() { ptList.add(new long[] {1, 1}); writer.resetNumMaterializeCalls(); - writer.readBlocks(dataset, datasetAttributes, ptList); + writer.readChunks(dataset, datasetAttributes, ptList); System.out.println(writer.getNumMaterializeCalls()); System.out.println(""); } @@ -562,7 +561,7 @@ public void shardExistsTest() { } /* write blocks to shards (0,0), (1,0), and (2,2) */ - writer.writeBlocks( + writer.writeChunks( dataset, attrs, new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), /* shard (0, 0) */ @@ -574,7 +573,7 @@ public void shardExistsTest() { Predicate assertShardExistsTracking = (gridPosition) -> { trackingWriter.resetAllTracking(); - final boolean exists = writer.shardExists(dataset, attrs, gridPosition); + final boolean exists = writer.blockExists(dataset, attrs, gridPosition); assertEquals("isFileCheck incremented", 1, trackingWriter.getNumIsFileCalls()); assertEquals("No Bytes Read", 0, trackingWriter.getTotalBytesRead()); return exists; @@ -629,7 +628,7 @@ public void testPartialReadAggregationBehavior() { ptList.add(new long[] {1,1}); /* write blocks to shard (0,0) */ - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, new ByteArrayDataBlock(blockSize, ptList.get(0), data), @@ -639,14 +638,14 @@ public void testPartialReadAggregationBehavior() { ); writer.resetNumMaterializeCalls(); - writer.readBlocks(dataset, datasetAttributes, ptList); + writer.readChunks(dataset, datasetAttributes, ptList); // TODO change this if and when we implement aggregation of read calls // one for the index, one for each of the four blocks assertEquals(5, writer.getNumMaterializeCalls()); writer.resetNumMaterializeCalls(); - writer.readShard(dataset, datasetAttributes, new long[] {0,0}); + writer.readBlock(dataset, datasetAttributes, new long[] {0,0}); // one for the index, one for each of the four blocks assertEquals(5, writer.getNumMaterializeCalls()); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java index 5c900a4a6..02737334f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java @@ -100,7 +100,7 @@ public static void main(String[] args) { System.out.println("shard.getSize() = " + Arrays.toString(shard.getSize())); System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); System.out.println(); - datasetAccess.writeShard(store, shard, 1); + datasetAccess.writeBlock(store, shard, 1); System.out.println(); System.out.println(); } @@ -110,7 +110,7 @@ public static void main(String[] args) { System.out.println("shard.getSize() = " + Arrays.toString(shard.getSize())); System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); System.out.println(); - datasetAccess.writeShard(store, shard, 1); + datasetAccess.writeBlock(store, shard, 1); System.out.println(); System.out.println(); } @@ -201,7 +201,7 @@ public void testWriteNullBlockRemovesShard() throws Exception { final long[] blockGridPosition = {3}; final DataBlock block = createDataBlock(datablockSize, blockGridPosition, 100); - datasetAccess.writeBlock(store, block); + datasetAccess.writeChunk(store, block); // Verify the shard exists assertTrue("Shard should exist after writing block", store.exists(shardKey)); @@ -241,7 +241,7 @@ public void testWriteNullBlockRemovesShard() throws Exception { assertFalse("Shard should not exist at the start of the test", store.exists(shardKey2)); // write blocks - datasetAccess.writeBlocks(store, Streams.of(block1, block2, block3).collect(Collectors.toList())); + datasetAccess.writeChunks(store, Streams.of(block1, block2, block3).collect(Collectors.toList())); // Verify the shard exists assertTrue("Shard should exist after writing blocks", store.exists(shardKey1)); @@ -260,16 +260,16 @@ public void testWriteNullBlockRemovesShard() throws Exception { assertTrue("Shard should still exist because was not affected", store.exists(shardKey2)); // Verify we can still read block [2] - final DataBlock readBlock = datasetAccess.readBlock(store, new long[]{2}); + final DataBlock readBlock = datasetAccess.readChunk(store, new long[]{2}); assertTrue("Block [2] should still be readable", readBlock != null); assertTrue("Block [2] data should match", Arrays.equals(block1.getData(), readBlock.getData())); // Verify block [3] is gone - final DataBlock readBlock2 = datasetAccess.readBlock(store, new long[]{3}); + final DataBlock readBlock2 = datasetAccess.readChunk(store, new long[]{3}); assertNull("Block [3] should be null after removal", readBlock2); // Verify block [4] exists - final DataBlock readBlock3 = datasetAccess.readBlock(store, new long[]{4}); + final DataBlock readBlock3 = datasetAccess.readChunk(store, new long[]{4}); assertTrue("Block [4] should still be readable", readBlock3 != null); assertTrue("Block [4] data should match", Arrays.equals(block3.getData(), readBlock3.getData())); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java index 86c0308a2..452bfce23 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java @@ -90,7 +90,7 @@ public static void main(String[] args) { System.out.println("shard.getGridPosition() = " + Arrays.toString(shard.getGridPosition())); System.out.println("shard.getSize() = " + Arrays.toString(shard.getSize())); System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); - datasetAccess.writeShard(store, shard, 1); + datasetAccess.writeBlock(store, shard, 1); // we should get these DataBlock values @@ -103,15 +103,15 @@ public static void main(String[] args) { // 31, 32, 33, | 34, 35, 36 System.out.println("{4, 2}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {4, 2}).getData())); + datasetAccess.readChunk(store, new long[] {4, 2}).getData())); System.out.println("{5, 2}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {5, 2}).getData())); + datasetAccess.readChunk(store, new long[] {5, 2}).getData())); System.out.println("{4, 3}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {4, 3}).getData())); + datasetAccess.readChunk(store, new long[] {4, 3}).getData())); System.out.println("{5, 3}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {5, 3}).getData())); + datasetAccess.readChunk(store, new long[] {5, 3}).getData())); - final DataBlock readShard = datasetAccess.readShard(store, new long[] {2, 1}, 1); + final DataBlock readShard = datasetAccess.readBlock(store, new long[] {2, 1}, 1); System.out.println("readShard.getData() = " + Arrays.toString(readShard.getData())); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java index 87737668f..73f239d50 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java @@ -90,7 +90,7 @@ public static void main(String[] args) { System.out.println("shard.getGridPosition() = " + Arrays.toString(shard.getGridPosition())); System.out.println("shard.getSize() = " + Arrays.toString(shard.getSize())); System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); - datasetAccess.writeShard(store, shard, 1); + datasetAccess.writeBlock(store, shard, 1); // we should get these DataBlock values @@ -103,13 +103,13 @@ public static void main(String[] args) { // 31, 32, 33, | 34, 35, 36 System.out.println("{10, 4}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {10, 4}).getData())); + datasetAccess.readChunk(store, new long[] {10, 4}).getData())); System.out.println("{11, 4}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {11, 4}).getData())); + datasetAccess.readChunk(store, new long[] {11, 4}).getData())); System.out.println("{10, 5}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {10, 5}).getData())); + datasetAccess.readChunk(store, new long[] {10, 5}).getData())); System.out.println("{11, 5}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {11, 5}).getData())); + datasetAccess.readChunk(store, new long[] {11, 5}).getData())); // 1, 2, 3, | 4 @@ -119,7 +119,7 @@ public static void main(String[] args) { // 19, 20, 21, | 22 // 25, 26, 27, | 28 - final DataBlock readShard = datasetAccess.readShard(store, new long[] {5, 2}, 1); + final DataBlock readShard = datasetAccess.readBlock(store, new long[] {5, 2}, 1); System.out.println("readShard.getData() = " + Arrays.toString(readShard.getData())); } From ba4778b825510dfa4b078ba66139ac39a0dd7fa0 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 16 Mar 2026 13:51:44 -0400 Subject: [PATCH 07/41] feat: add getChunkSize * getBlockSize should correspond to readBlock / writeBlock * useful to have an analogue for readChunk / writeChunk --- .../saalfeldlab/n5/DatasetAttributes.java | 26 ++++++++------ .../saalfeldlab/n5/DatasetAttributesTest.java | 36 ++++++++++--------- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index af25c729d..71a62298c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -89,13 +89,13 @@ public class DatasetAttributes implements Serializable { private final long[] dimensions; + // number of samples per chunk per dimension + private final int[] chunkSize; + // number of samples per block per dimension + // identical to chunkSize for non-sharded datasets private final int[] blockSize; - // TODO add a getter? - // the shard size - private final int[] outerBlockSize; - private final DataType dataType; private final JsonElement defaultValue; @@ -108,7 +108,7 @@ public class DatasetAttributes implements Serializable { public DatasetAttributes( final long[] dimensions, - final int[] outerBlockSize, + final int[] blockSize, final DataType dataType, final JsonElement defaultValue, final BlockCodecInfo blockCodecInfo, @@ -117,7 +117,7 @@ public DatasetAttributes( this.dimensions = dimensions; this.dataType = dataType; - this.outerBlockSize = outerBlockSize; + this.blockSize = blockSize; this.defaultValue = defaultValue == null ? JsonNull.INSTANCE : defaultValue; this.blockCodecInfo = blockCodecInfo == null ? defaultBlockCodecInfo() : blockCodecInfo; @@ -131,7 +131,7 @@ public DatasetAttributes( .toArray(DataCodecInfo[]::new); access = createDatasetAccess(); - blockSize = access.getGrid().getBlockSize(0); + chunkSize = access.getGrid().getBlockSize(0); } public DatasetAttributes( @@ -200,7 +200,7 @@ protected DatasetAccess createDatasetAccess() { // NestedGrid validates block sizes, so instantiate it before creating the blockCodecs // blockCodecInfo.create below could fail unexpecedly with invalid // blockSizes so validate first - blockSizes[m - 1] = outerBlockSize; + blockSizes[m - 1] = blockSize; BlockCodecInfo tmpInfo = blockCodecInfo; for (int l = m - 1; l > 0; --l) { final ShardCodecInfo info = (ShardCodecInfo)tmpInfo; @@ -274,6 +274,11 @@ public int getNumDimensions() { return dimensions.length; } + public int[] getChunkSize() { + + return chunkSize; + } + public int[] getBlockSize() { return blockSize; @@ -334,7 +339,6 @@ public NestedGrid getNestedBlockGrid() { return getDatasetAccess().getGrid(); } - public BlockCodecInfo getBlockCodecInfo() { return blockCodecInfo; @@ -359,7 +363,7 @@ public HashMap asMap() { final HashMap map = new HashMap<>(); map.put(DIMENSIONS_KEY, dimensions); - map.put(BLOCK_SIZE_KEY, blockSize); + map.put(BLOCK_SIZE_KEY, chunkSize); map.put(DATA_TYPE_KEY, dataType); map.put(COMPRESSION_KEY, getCompression()); return map; @@ -425,7 +429,7 @@ public static class DatasetAttributesAdapter implements JsonSerializer Date: Mon, 16 Mar 2026 16:18:39 -0400 Subject: [PATCH 08/41] fix: ShardTest uses getChunkSize --- src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 2e1fb9fd7..4b9504471 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -386,7 +386,7 @@ public void writeReadBlockTest() { writer.remove(dataset); writer.createDataset(dataset, datasetAttributes); - final int[] blockSize = datasetAttributes.getBlockSize(); + final int[] chunkSize = datasetAttributes.getChunkSize(); final DataType dataType = datasetAttributes.getDataType(); final int numElements = 2 * 2; @@ -395,7 +395,7 @@ public void writeReadBlockTest() { for (int idx1 = 1; idx1 >= 0; idx1--) { for (int idx2 = 1; idx2 >= 0; idx2--) { final long[] gridPosition = {idx1, idx2}; - final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(blockSize, gridPosition, numElements); + final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(chunkSize, gridPosition, numElements); byte[] data = dataBlock.getData(); for (int i = 0; i < data.length; i++) { data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); From 4ceb1182b658597ee1c5b50346900e4e64c16dd4 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 16 Mar 2026 16:19:24 -0400 Subject: [PATCH 09/41] doc: writeRegion and DataBlockSupplier --- src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 4c248bec6..5d3bd6978 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -301,7 +301,7 @@ interface DataBlockSupplier { * * @param gridPos * @param existingDataBlock - * existing data to be merged into the new data block (maybe {@code null}) + * existing data to be merged into the new data block (may be {@code null}) * * @return data block at the given gridPos */ @@ -313,7 +313,7 @@ interface DataBlockSupplier { * @param datasetAttributes the dataset attributes * @param min min pixel coordinate of region to write * @param size size in pixels of region to write - * @param dataBlocks is asked to create blocks within the given region + * @param dataBlocks is asked to create chunks within the given region * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. * @throws N5Exception the exception */ @@ -332,7 +332,7 @@ void writeRegion( * @param size size in pixels of region to write * @param dataBlocks is asked to create blocks within the given region * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. - * @param exec used to parallelize over blocks and shards + * @param exec used to parallelize over blocks (chunks and shards) * @throws N5Exception the exception */ void writeRegion( From fe52660304cb5f7faf031652a3d84c10a88b41e7 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 19 Mar 2026 15:35:42 -0400 Subject: [PATCH 10/41] feat: add N5Writer.deleteBlock * that deletes blocks or chunks --- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 10 ++++++ .../org/janelia/saalfeldlab/n5/N5Writer.java | 31 +++++++++++++++++++ .../saalfeldlab/n5/AbstractN5Test.java | 13 ++++++-- .../saalfeldlab/n5/shard/ShardTest.java | 19 ++++++++++-- 4 files changed, 68 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 0a2e87089..42386d368 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -319,6 +319,16 @@ default boolean remove(final String path) throws N5Exception { return true; } + @Override + default boolean deleteBlock( + final String path, + final DatasetAttributes datasetAttributes, + final long... gridPosition) throws N5Exception { + + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); + return posKva.remove(gridPosition); + } + @Override default boolean deleteChunk( final String path, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 5d3bd6978..60c5700b1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -344,6 +344,37 @@ void writeRegion( boolean writeFully, ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException; + /** + * Deletes the block at {@code gridPosition}. + * + * @param datasetPath dataset path + * @param gridPosition position of block to be deleted + * @throws N5Exception if the block exists but could not be deleted + * + * @return {@code true} if the block at {@code gridPosition} existed and was deleted. + */ + default boolean deleteBlock( + final String datasetPath, + final long... gridPosition) throws N5Exception { + final DatasetAttributes datasetAttributes = getDatasetAttributes(datasetPath); + return deleteBlock(datasetPath, datasetAttributes, gridPosition); + } + + /** + * Deletes the block at {@code gridPosition}. + * + * @param datasetPath the dataset path + * @param datasetAttributes the dataset attributes + * @param gridPosition position of block to be deleted + * @throws N5Exception if the block exists but could not be deleted + * + * @return {@code true} if the block at {@code gridPosition} existed and was deleted. + */ + boolean deleteBlock( + String datasetPath, + DatasetAttributes datasetAttributes, + long... gridPosition) throws N5Exception; + /** * Deletes the chunk at {@code gridPosition}. * diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 15f197ff0..6fb0e3e05 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1307,9 +1307,16 @@ public void testDelete() { assertTrue(readBlock instanceof ByteArrayDataBlock); assertArrayEquals(byteBlock, ((ByteArrayDataBlock)readBlock).getData()); - assertTrue("deleting existing block should return true", n5.deleteChunk(datasetName, position1)); - assertFalse("deleting non-existing block should return false", n5.deleteChunk(datasetName, position1)); - assertFalse("deleting non-existing block should return false", n5.deleteChunk(datasetName, position2)); + assertTrue("deleting existing chunk should return true", n5.deleteChunk(datasetName, position1)); + assertFalse("deleting non-existing chunk should return false", n5.deleteChunk(datasetName, position1)); + assertFalse("deleting non-existing chunk should return false", n5.deleteChunk(datasetName, position2)); + + // for an unsharded dataset, deleteChunk and deleteBlock behave identically + assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position1)); + assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position2)); + + n5.writeChunk(datasetName, attributes, dataBlock); + assertTrue("deleting existing block should return true", n5.deleteBlock(datasetName, position1)); // no block should exist anymore assertNull(n5.readChunk(datasetName, attributes, position1)); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 4b9504471..f4032bd7a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -58,6 +58,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -425,10 +426,10 @@ public void writeReadShardTest() { final int[] shardSize = new int[] {4,4}; final int shardN = 16; - final int[] blockSize = new int[] {2,2}; + final int[] chunkSize = new int[] {2,2}; final String dataset = "writeReadShard"; - DatasetAttributes attrs = getTestAttributes(DataType.INT32, new long[]{8, 8}, shardSize, blockSize); + DatasetAttributes attrs = getTestAttributes(DataType.INT32, new long[]{8, 8}, shardSize, chunkSize); final int[] shardData = range(shardN); IntArrayDataBlock shard = new IntArrayDataBlock(shardSize, new long[]{0, 0}, shardData); @@ -475,6 +476,20 @@ public void writeReadShardTest() { Stream.of( new long[] {0,0}, new long[] {1,0}, new long[] {0,1}).collect(Collectors.toList())); assertNull(n5.readBlock(dataset, attrs, 0, 0)); + + + // write the shard again + n5.writeBlock(dataset, attrs, shard); + + // delete the block + // ensure it returns true because the block exists + assertTrue(n5.deleteBlock(dataset, attrs, shard.getGridPosition())); + + // ensure it returns false when the block does not exist + assertFalse(n5.deleteBlock(dataset, attrs, shard.getGridPosition())); + + // readBlock must return null for the deleted block + assertNull(n5.readBlock(dataset, attrs, shard.getGridPosition())); } } From 944a3ea5a849fa9c97669740abe1a9ccd56cee5f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Sat, 21 Mar 2026 11:20:31 -0400 Subject: [PATCH 11/41] test: AbstractN5Test changes related to chunk-block refactor * add clarifying javadoc * test blockSize == chunkSize when not sharded * fix block / chunk equivalence test and comments --- .../saalfeldlab/n5/AbstractN5Test.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 6fb0e3e05..d6e261ae8 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -74,6 +74,9 @@ /** * Abstract base class for testing N5 functionality. * Subclasses are expected to provide a specific N5 implementation to be tested by defining the {@link #createN5Writer()} method. + *

+ * This class does not create sharded datasets. Its tests generally call read/writeChunk, not read/writeBlock despite + * test methods using the generic term "block". * * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> * @author Igor Pisarev <pisarevi@janelia.hhmi.org> @@ -162,7 +165,7 @@ protected N5Writer createN5Writer(final String location) throws IOException, URI return createN5Writer(location, new GsonBuilder()); } - /* Tests that overide this should enusre that the `N5Writer` created will remove its container on close() */ + /* Tests that override this should ensure that the `N5Writer` created will remove its container on close() */ protected abstract N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException, URISyntaxException; protected N5Reader createN5Reader(final String location) throws IOException, URISyntaxException { @@ -243,6 +246,7 @@ public void testCreateDataset() { } assertArrayEquals(dimensions, info.getDimensions()); assertArrayEquals(blockSize, info.getBlockSize()); + assertArrayEquals("blockSize == chunkSize when not sharded", blockSize, info.getChunkSize()); assertEquals(DataType.UINT64, info.getDataType()); } @@ -254,8 +258,8 @@ public void testBlocksLargerThanDimensions() { final int[] largeBlockSize = new int[]{5, 7, 10}; try (final N5Writer n5 = createTempN5Writer()) { - n5.createDataset(datasetName, smallDimensions, largeBlockSize, DataType.UINT8, new RawCompression()); - final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + final DatasetAttributes attributes = n5.createDataset( + datasetName, smallDimensions, largeBlockSize, DataType.UINT8, new RawCompression()); // Create a block that is larger than the dataset dimensions final int numElements = largeBlockSize[0] * largeBlockSize[1] * largeBlockSize[2]; @@ -475,11 +479,9 @@ public void testWriteReadDoubleBlock() { } @Test - public void testWriteReadShard() { - - // test that writeShard behaves the same as writeBlock - // for unsharded datasets + public void testReadChunkVsBlock() { + // test that readBlock behaves the same as readChunk for unsharded datasets for (final Compression compression : getCompressions()) { try (final N5Writer n5 = createTempN5Writer()) { @@ -489,18 +491,17 @@ public void testWriteReadShard() { n5.writeChunk(datasetName, attributes, dataBlock); - // read with readShard + // read with readBlock final DataBlock loadedShard = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(shortBlock, (short[])loadedShard.getData()); - // read with readBlock - final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + // read with readChunk + final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData()); } } } - @Test public void testMode1WriteReadByteBlock() { From 3587f6b9e6f7a56952f2499c7036eda04fa9f916 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 23 Mar 2026 13:28:09 -0400 Subject: [PATCH 12/41] test: AbstractN5Test back to using read/writeBlock * improve testReadChunkVsBlock --- .../saalfeldlab/n5/AbstractN5Test.java | 100 +++++++++--------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index d6e261ae8..bc3510ddd 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -75,8 +75,8 @@ * Abstract base class for testing N5 functionality. * Subclasses are expected to provide a specific N5 implementation to be tested by defining the {@link #createN5Writer()} method. *

- * This class does not create sharded datasets. Its tests generally call read/writeChunk, not read/writeBlock despite - * test methods using the generic term "block". + * This class does not create sharded datasets. Its tests generally call read/writeBlock which are equivalent to read/writeChunk + * for the cases being tested here. The test {@link #testReadChunkVsBlock} checks that the equivalence between these methods holds. * * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> * @author Igor Pisarev <pisarevi@janelia.hhmi.org> @@ -269,10 +269,10 @@ public void testBlocksLargerThanDimensions() { } final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(largeBlockSize, new long[]{0, 0, 0}, data); - n5.writeChunk(datasetName, attributes, dataBlock); + n5.writeBlock(datasetName, attributes, dataBlock); // Read the block back - final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertNotNull("Block should be readable", loadedDataBlock); assertArrayEquals("Block size should match", largeBlockSize, loadedDataBlock.getSize()); assertArrayEquals("Block data should match", data, (byte[])loadedDataBlock.getData()); @@ -298,9 +298,9 @@ public void testUnalignedBlocksTruncatedAtEnd() { data0[i] = i + 1000; } final IntArrayDataBlock dataBlock0 = new IntArrayDataBlock(truncatedBlockSize0, new long[]{1, 0, 0}, data0); - n5.writeChunk(datasetName, attributes, dataBlock0); + n5.writeBlock(datasetName, attributes, dataBlock0); - final DataBlock loadedBlock0 = n5.readChunk(datasetName, attributes, 1, 0, 0); + final DataBlock loadedBlock0 = n5.readBlock(datasetName, attributes, 1, 0, 0); assertNotNull("Truncated block should be readable", loadedBlock0); assertArrayEquals("Truncated block data should match", data0, (int[])loadedBlock0.getData()); @@ -312,9 +312,9 @@ public void testUnalignedBlocksTruncatedAtEnd() { data1[i] = i + 2000; } final IntArrayDataBlock dataBlock1 = new IntArrayDataBlock(truncatedBlockSize1, new long[]{0, 2, 0}, data1); - n5.writeChunk(datasetName, attributes, dataBlock1); + n5.writeBlock(datasetName, attributes, dataBlock1); - final DataBlock loadedBlock1 = n5.readChunk(datasetName, attributes, 0, 2, 0); + final DataBlock loadedBlock1 = n5.readBlock(datasetName, attributes, 0, 2, 0); assertNotNull("Truncated block should be readable", loadedBlock1); assertArrayEquals("Truncated block data should match", data1, (int[])loadedBlock1.getData()); @@ -326,16 +326,14 @@ public void testUnalignedBlocksTruncatedAtEnd() { data2[i] = i + 3000; } final IntArrayDataBlock dataBlock2 = new IntArrayDataBlock(truncatedBlockSize2, new long[]{0, 0, 4}, data2); - n5.writeChunk(datasetName, attributes, dataBlock2); + n5.writeBlock(datasetName, attributes, dataBlock2); - final DataBlock loadedBlock2 = n5.readChunk(datasetName, attributes, 0, 0, 4); + final DataBlock loadedBlock2 = n5.readBlock(datasetName, attributes, 0, 0, 4); assertNotNull("Truncated block should be readable", loadedBlock2); assertArrayEquals("Truncated block data should match", data2, (int[])loadedBlock2.getData()); } } - - @Test public void testWriteReadByteBlock() { @@ -347,9 +345,9 @@ public void testWriteReadByteBlock() { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, new long[]{0, 0, 0}, byteBlock); - n5.writeChunk(datasetName, attributes, dataBlock); + n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); } } @@ -369,9 +367,9 @@ public void testWriteReadStringBlock() { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final StringDataBlock dataBlock = new StringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); - n5.writeChunk(datasetName, attributes, dataBlock); + n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0L, 0L, 0L); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0L, 0L, 0L); assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); } @@ -390,9 +388,9 @@ public void testWriteReadShortBlock() { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock); - n5.writeChunk(datasetName, attributes, dataBlock); + n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData()); } @@ -412,9 +410,9 @@ public void testWriteReadIntBlock() { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final IntArrayDataBlock dataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); - n5.writeChunk(datasetName, attributes, dataBlock); + n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(intBlock, (int[])loadedDataBlock.getData()); } @@ -434,9 +432,9 @@ public void testWriteReadLongBlock() { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize, new long[]{0, 0, 0}, longBlock); - n5.writeChunk(datasetName, attributes, dataBlock); + n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(longBlock, (long[])loadedDataBlock.getData()); } @@ -452,9 +450,9 @@ public void testWriteReadFloatBlock() { n5.createDataset(datasetName, dimensions, blockSize, DataType.FLOAT32, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final FloatArrayDataBlock dataBlock = new FloatArrayDataBlock(blockSize, new long[]{0, 0, 0}, floatBlock); - n5.writeChunk(datasetName, attributes, dataBlock); + n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(floatBlock, (float[])loadedDataBlock.getData(), 0.001f); } @@ -469,9 +467,9 @@ public void testWriteReadDoubleBlock() { n5.createDataset(datasetName, dimensions, blockSize, DataType.FLOAT64, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final DoubleArrayDataBlock dataBlock = new DoubleArrayDataBlock(blockSize, new long[]{0, 0, 0}, doubleBlock); - n5.writeChunk(datasetName, attributes, dataBlock); + n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(doubleBlock, (double[])loadedDataBlock.getData(), 0.001); } @@ -485,19 +483,25 @@ public void testReadChunkVsBlock() { for (final Compression compression : getCompressions()) { try (final N5Writer n5 = createTempN5Writer()) { + final short[] shortData1 = new short[shortBlock.length]; + for( int i = 0; i < shortBlock.length; i++) + shortData1[i] = (short)(2 * shortBlock[i] + 3); + n5.createDataset(datasetName, dimensions, blockSize, DataType.INT16, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); - final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock); + final ShortArrayDataBlock dataBlock0 = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock); + final ShortArrayDataBlock dataBlock1 = new ShortArrayDataBlock(blockSize, new long[]{1, 0, 0}, shortData1); - n5.writeChunk(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock0); + n5.writeBlock(datasetName, attributes, dataBlock1); // read with readBlock - final DataBlock loadedShard = n5.readBlock(datasetName, attributes, 0, 0, 0); - assertArrayEquals(shortBlock, (short[])loadedShard.getData()); + assertArrayEquals(shortBlock, (short[])n5.readBlock(datasetName, attributes, 0, 0, 0).getData()); + assertArrayEquals(shortData1, (short[])n5.readBlock(datasetName, attributes, 1, 0, 0).getData()); // read with readChunk - final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); - assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData()); + assertArrayEquals(shortBlock, (short[])n5.readChunk(datasetName, attributes, 0, 0, 0).getData()); + assertArrayEquals(shortData1, (short[])n5.readChunk(datasetName, attributes, 1, 0, 0).getData()); } } } @@ -516,9 +520,9 @@ public void testMode1WriteReadByteBlock() { n5.createDataset(datasetName, dimensions, differentBlockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(differentBlockSize, new long[]{0, 0, 0}, byteBlock); - n5.writeChunk(datasetName, attributes, dataBlock); + n5.writeBlock(datasetName, attributes, dataBlock); - final DataBlock loadedDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); } @@ -582,22 +586,22 @@ public void testWriteInvalidBlock() { // write a block that is too large final ByteArrayDataBlock bigDataBlock = new ByteArrayDataBlock(biggerBlockSize, new long[]{0, 0, 0}, biggerData); - n5.writeChunk(datasetName, attributes, bigDataBlock); + n5.writeBlock(datasetName, attributes, bigDataBlock); - final DataBlock loadedBigDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); + final DataBlock loadedBigDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(biggerData, (byte[])loadedBigDataBlock.getData()); // write a block that is too small final ByteArrayDataBlock smallDataBlock = new ByteArrayDataBlock(smallerBlockSize, new long[]{0, 0, 0}, smallerData); - n5.writeChunk(datasetName, attributes, smallDataBlock); + n5.writeBlock(datasetName, attributes, smallDataBlock); - final DataBlock loadedSmallDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); + final DataBlock loadedSmallDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(smallerData, (byte[])loadedSmallDataBlock.getData()); // write a block of the wrong type final FloatArrayDataBlock floatDataBlock = new FloatArrayDataBlock(blockSize, new long[]{0, 0, 0}, floatData); assertThrows(ClassCastException.class, () -> { - n5.writeChunk(datasetName, attributes, floatDataBlock); + n5.writeBlock(datasetName, attributes, floatDataBlock); }); } } @@ -611,15 +615,15 @@ public void testOverwriteBlock() { final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final IntArrayDataBlock randomDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); - n5.writeChunk(datasetName, attributes, randomDataBlock); - final DataBlock loadedRandomDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); + n5.writeBlock(datasetName, attributes, randomDataBlock); + final DataBlock loadedRandomDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(intBlock, (int[])loadedRandomDataBlock.getData()); // test the case where the resulting file becomes shorter (because the data compresses better) final int[] emptyBlock = new int[DataBlock.getNumElements(blockSize)]; final IntArrayDataBlock emptyDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, emptyBlock); - n5.writeChunk(datasetName, attributes, emptyDataBlock); - final DataBlock loadedEmptyDataBlock = n5.readChunk(datasetName, attributes, 0, 0, 0); + n5.writeBlock(datasetName, attributes, emptyDataBlock); + final DataBlock loadedEmptyDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(emptyBlock, (int[])loadedEmptyDataBlock.getData()); } } @@ -1053,7 +1057,7 @@ public void testDeepList() throws ExecutionException, InterruptedException { final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64); final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize, new long[]{0, 0, 0}, new long[blockNumElements]); n5.createDataset(datasetName, datasetAttributes); - n5.writeChunk(datasetName, datasetAttributes, dataBlock); + n5.writeBlock(datasetName, datasetAttributes, dataBlock); final List datasetList = Arrays.asList(n5.deepList("/")); for (final String subGroup : subGroupNames) @@ -1297,10 +1301,10 @@ public void testDelete() { final long[] position2 = {0, 1, 2}; // no blocks should exist to begin with - assertNull(n5.readChunk(datasetName, attributes, position1)); + assertNull(n5.readBlock(datasetName, attributes, position1)); final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, position1, byteBlock); - n5.writeChunk(datasetName, attributes, dataBlock); + n5.writeBlock(datasetName, attributes, dataBlock); // block should exist at position1 but not at position2 final DataBlock readBlock = n5.readChunk(datasetName, attributes, position1); @@ -1320,8 +1324,8 @@ public void testDelete() { assertTrue("deleting existing block should return true", n5.deleteBlock(datasetName, position1)); // no block should exist anymore - assertNull(n5.readChunk(datasetName, attributes, position1)); - assertNull(n5.readChunk(datasetName, attributes, position2)); + assertNull(n5.readBlock(datasetName, attributes, position1)); + assertNull(n5.readBlock(datasetName, attributes, position2)); } } From 156d92ca7bfefbc5d719a599b8a6e7abb1e85010 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 24 Mar 2026 23:12:57 +0100 Subject: [PATCH 13/41] clean up and fix typos --- src/main/java/org/janelia/saalfeldlab/n5/DataType.java | 1 + .../java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java | 2 +- src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java | 4 ++-- src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java | 3 --- src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java | 2 +- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 08bebf1e6..4e0ff8d9a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -37,6 +37,7 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; + /** * Enumerates available data types. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 42386d368..538d33b19 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -84,7 +84,7 @@ default void createGroup(final String path) throws N5Exception { /** * Helper method that writes an attributes tree into the store - * + *

* TODO This method is not part of the public API and should be protected * in Java versions greater than 8 * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 3d3db1aeb..e7155bbda 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -94,7 +94,7 @@ static T readAttribute( * @param gson * used to deserialize {@code attribute} * @param type - * to desrialize {@code attribute} as + * to deserialize {@code attribute} as * @param * return type represented by {@link Type type} * @return the deserialized attribute object, or {@code null} if @@ -548,7 +548,7 @@ static void writeAttribute( } /** - * Writes the attributes JsonElemnt to a given {@link Writer}. + * Writes the attributes JsonElement to a given {@link Writer}. * This will overwrite any existing attributes. * * @param writer diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java index a97eeab51..5881e33a7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java @@ -28,9 +28,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.io.UncheckedIOException; - public class N5Exception extends RuntimeException { public N5Exception() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index c5103e140..e135ae4d6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -601,7 +601,7 @@ default String[] deepListDatasets(final String pathName) throws N5Exception { * Helper method to recursively list all groups and datasets. This method is not part of the * public API and is accessible only because Java 8 does not support private * interface methods yet. - * + *

* TODO make private when committing to Java versions newer than 8 * * @param n5 the n5 reader From 908c42cdf778dc718f0a637b7330386aa7d3b95c Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 24 Mar 2026 23:13:19 +0100 Subject: [PATCH 14/41] Avoid stream creation --- src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index e135ae4d6..e45164008 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -673,7 +673,7 @@ default String[] deepList( results.add(result.substring(normalPathName.length() + groupSeparator.length())); } - return results.stream().toArray(String[]::new); + return results.toArray(new String[0]); } /** @@ -760,7 +760,7 @@ default String[] deepListDatasets( results.add(result.substring(normalPathName.length() + groupSeparator.length())); } - return results.stream().toArray(String[]::new); + return results.toArray(new String[0]); } /** From a6ffbc56b32d6da549634892db9b67265fde56d9 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 24 Mar 2026 23:14:32 +0100 Subject: [PATCH 15/41] fix javadoc and argument names --- .../org/janelia/saalfeldlab/n5/N5Reader.java | 7 +- .../janelia/saalfeldlab/n5/shard/Nesting.java | 65 ++++++++++--------- .../saalfeldlab/n5/DatasetAttributesTest.java | 6 +- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index e45164008..98e73af93 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -45,6 +45,7 @@ import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; /** * A simple structured container for hierarchies of chunked @@ -351,7 +352,7 @@ default List> readChunks( /** * Reads a block, returning a {@link DataBlock}. Will be a chunk or shard (if the dataset is sharded). *

- * A block is the highest (coarsest) level of the dataset's {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid} + * A block is the highest (coarsest) level of the dataset's {@link NestedGrid} * This method's behavior is identical to {@link #readChunk} for un-sharded datasets. * * @param @@ -376,6 +377,10 @@ DataBlock readBlock( /** * Checks if a block exists at the given grid position without reading the data. *

+ * A block is the highest (coarsest) level of the dataset's {@link + * NestedGrid}, that is, a shard if the dataset is sharded, or a chunk + * otherwise. + *

* This method only checks for the presence of the key value for the gridPosition, it does not * read or validate the contents. As a result, this method refers to chunks in un-sharded datasets * or shards in sharded datasets. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java index ff3e6e164..29106b09c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java @@ -37,14 +37,15 @@ * sharded N5 datasets. *

* This class provides classes for representing and navigating nested/sharded - * dataset layouts for the N5 API. In a sharded dataset, data blocks are grouped - * into higher-level containers called shards, which can themselves be nested - * within parent shards, creating a multi-level hierarchy. + * dataset layouts for the N5 API. In a sharded dataset, level-0 data blocks + * (called chunks) are grouped into higher-level containers called + * shards, which can themselves be nested within parent shards, + * creating a multi-level hierarchy. *

- * Nesting levels: + * Nesting levels: *

    - *
  • Level 0: The finest granularity - individual data blocks - * containing actual pixel data + *
  • Level 0: The finest granularity - individual chunks (level-0 data + * blocks) containing actual pixel data *
  • Level 1: First-level shards, which contain multiple level-0 data * blocks *
  • Level 2+: Higher-level shards (if they exist), which contain @@ -62,7 +63,7 @@ * {@code NestedGrid} at a particular nesting level, providing coordinate * transformations between levels. *
- * + * */ public class Nesting { @@ -85,8 +86,8 @@ protected NestedPosition(final NestedGrid grid, final long[] position, final int /** * Get the nesting level of this position. *

- * Positions with {@code level=0} refer to DataBlocks, positions with - * {@code level=1} refer to first-level shards (containing DataBlocks), + * Positions with {@code level=0} refer to chunks, positions with + * {@code level=1} refer to first-level shards (containing chunks), * and so on. * * @return nesting level @@ -220,11 +221,11 @@ public int hashCode() { /** * A nested grid of blocks used to coordinate the relationships of shards - * and the blocks / chunks they contain. + * and the blocks (chunks / sub-shards) they contain. *

* The nesting depth ({@link #numLevels()}) of the {@code NestedGrid} is 1 * for non-sharded datasets, 2 for simple sharded datasets (where shards - * contain datablocks), and ≥3 for nested sharded datasets. + * contain chunks), and ≥3 for nested sharded datasets. *

* Positions with {@code level=0} refer to the DataBlock grid, positions * with {@code level=1} refer to first-level Shard grid, and so on. @@ -255,7 +256,7 @@ public static class NestedGrid { private final long[] datasetSize; /** - * dimensions of the dataset in (level-0) blocks. + * dimensions of the dataset in level-0 blocks. */ private final long[] datasetSizeInBlocks; @@ -337,7 +338,7 @@ public NestedGrid(int[][] blockSizes) { * level} grid {@code position}. *

* Note that {@code position} is in units of grid elements at {@code - * level}. Positions with {@code level=0} refer to the DataBlock grid, + * level}. Positions with {@code level=0} refer to the Chunk grid, * positions with {@code level=1} refer to first-level Shard grid, and * so on. *

@@ -356,18 +357,18 @@ public NestedPosition nestedPosition(final long[] position, final int level) { } /** - * Create a {@code NestedPosition} at the specified block grid {@code + * Create a {@code NestedPosition} at the specified chunk grid {@code * position} (that is, at nesting level 0). *

- * Note that {@code position} is in units of DataBlocks. + * Note that {@code position} is in units of chunks. *

* The returned {@code NestedPosition} will have * {@link NestedPosition#level() level()==0}. * * @param position - * position at level 0 (block grid) + * position at level 0 (chunk grid) * - * @return a NestedPosition representation of the specified block grid position + * @return a NestedPosition representation of the specified chunk grid position */ public NestedPosition nestedPosition(final long[] position) { return nestedPosition(position, 0); @@ -429,12 +430,12 @@ public long[] pixelPosition( } /** - * Get the maximum pixel position in the shard/block at the given {@code - * sourcePos} grid position at {@code sourceLevel}. + * Get the maximum pixel position in the block (chunk/shard) at the + * given {@code sourcePos} grid position at {@code sourceLevel}. *

* Note that this does not take into account {@link #getDatasetSize() * dataset dimensions}. That is, it is always assumed that the - * shard/block has the default size. + * chunk/shard has the default size. * * @param sourcePos * a grid position at {@code sourceLevel} @@ -454,12 +455,12 @@ public void maxPixelPosition( } /** - * Get the maximum pixel position in the shard/block at the given {@code - * sourcePos} grid position at {@code sourceLevel}. + * Get the maximum pixel position in the block (chunk/shard) at the + * given {@code sourcePos} grid position at {@code sourceLevel}. *

* Note that this does not take into account {@link #getDatasetSize() * dataset dimensions}. That is, it is always assumed that the - * shard/block has the default size. + * chunk/shard has the default size. * * @param sourcePos * a grid position at {@code sourceLevel} @@ -483,7 +484,7 @@ public long[] maxPixelPosition( *

* For example, this can be used to compute the coordinates on the shard * grid ({@code targetLevel==1}) of the shard containing a given - * datablock ({@code sourcePos} at {@code sourceLevel==0}). + * chunk ({@code sourcePos} at {@code sourceLevel==0}). * * @param sourcePos * a grid position at {@code sourceLevel} @@ -512,7 +513,7 @@ public void absolutePosition( *

* For example, this can be used to compute the coordinates on the shard * grid ({@code targetLevel==1}) of the shard containing a given - * datablock ({@code sourcePos} at {@code sourceLevel==0}). + * chunk ({@code sourcePos} at {@code sourceLevel==0}). * * @param sourcePos * the source position j @@ -537,11 +538,11 @@ public long[] absolutePosition( /** * Get the absolute grid position at {@code targetLevel} for the given - * {@code sourcePos} block grid position (level 0). + * {@code sourcePos} chunk grid position (level 0). *

* For example, this can be used to compute the coordinates on the shard * grid ({@code targetLevel==1}) of the shard containing a given - * datablock ({@code sourcePos}. + * chunk ({@code sourcePos}. * * @param sourcePos * the source position j @@ -564,8 +565,8 @@ public long[] absolutePosition( * numLevels} or the dataset for {@code targetLevel+1 == numLevels}.) *

* For example, this can be used to compute the grid coordinates {@code - * targetLevel==0} of a given datablock ({@code sourcePos} at {@code - * sourceLevel==0}) withing a shard (containing element at level {@code + * targetLevel==0} of a given chunk ({@code sourcePos} at {@code + * sourceLevel==0}) within a shard (containing element at level {@code * targetLevel+1==1}). *

* @@ -612,7 +613,7 @@ public long[] relativePosition( * level-1} (that is, in units of {@code level-1} blocks). *

* For example {@code relativeBlockSize(1)} returns the number of - * datablocks in a (non-nested) shard. + * chunks in a (non-nested) shard. */ public int[] relativeBlockSize(final int level) { return relativeToAdjacent[level]; @@ -623,7 +624,7 @@ public int[] relativeBlockSize(final int level) { * 0} (that is, in units of {@code level-0} blocks). *

* For example {@code relativeToBaseBlockSize(1)} returns the number of - * datablocks in a (non-nested) shard. + * chunks in a (non-nested) shard. */ public int[] relativeToBaseBlockSize(final int level) { return relativeToBase[level]; @@ -647,7 +648,7 @@ public long[] getDatasetSize() { * This might return {@code null}, if this {@code NestedGrid} was not * constructed with dataset dimensions. * - * @return size of the dataset in pixels + * @return size of the dataset in chunks */ public long[] getDatasetSizeInBlocks() { return datasetSizeInBlocks; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java index 3b763eda4..7e57c6ae0 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java @@ -46,7 +46,7 @@ public class DatasetAttributesTest { /** - * Test that validateBlockShardSizes method accepts valid shard and block size combinations. + * Test that validateBlockShardSizes method accepts valid shard and chunk size combinations. */ @Test public void testValidateBlockShardSizesValid() { @@ -97,10 +97,10 @@ public void testValidateBlockShardSizesValid() { } private static DatasetAttributes shardDatasetAttributes( - long[] dimensions, int[] shardSize, int[] blockSize, DataType dataType) { + long[] dimensions, int[] shardSize, int[] chunkSize, DataType dataType) { DefaultShardCodecInfo blockCodecInfo = new DefaultShardCodecInfo( - blockSize, + chunkSize, new N5BlockCodecInfo(), new DataCodecInfo[]{new RawCompression()}, new RawBlockCodecInfo(), From 59160b5aed82f07c1f6ae686cd1ffc64a202fcc4 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 25 Mar 2026 17:20:26 +0100 Subject: [PATCH 16/41] fix javadoc and argument names --- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 18 +++++----- .../org/janelia/saalfeldlab/n5/N5Writer.java | 36 ++++++++++--------- .../saalfeldlab/n5/shard/DatasetAccess.java | 5 ++- .../n5/shard/DefaultDatasetAccess.java | 14 ++++---- .../n5/http/HttpReaderFsWriter.java | 12 +++---- 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 538d33b19..96e164326 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -225,12 +225,12 @@ default void writeRegion( final DatasetAttributes datasetAttributes, final long[] min, final long[] size, - final DataBlockSupplier dataBlocks, + final DataBlockSupplier chunkSupplier, final boolean writeFully) throws N5Exception { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes); - convertedDatasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, writeFully); + convertedDatasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, chunkSupplier, writeFully); } catch (final UncheckedIOException e) { throw new N5IOException( "Failed to write blocks into dataset " + datasetPath, e); @@ -243,13 +243,13 @@ default void writeRegion( final DatasetAttributes datasetAttributes, final long[] min, final long[] size, - final DataBlockSupplier dataBlocks, + final DataBlockSupplier chunkSupplier, final boolean writeFully, final ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes); - convertedDatasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, writeFully, exec); + convertedDatasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, chunkSupplier, writeFully, exec); } catch (final UncheckedIOException e) { throw new N5IOException( "Failed to write blocks into dataset " + datasetPath, e); @@ -260,12 +260,12 @@ default void writeRegion( default void writeChunks( final String datasetPath, final DatasetAttributes datasetAttributes, - final DataBlock... dataBlocks) throws N5Exception { + final DataBlock... chunks) throws N5Exception { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes); - convertedDatasetAttributes.getDatasetAccess().writeChunks(posKva, Arrays.asList(dataBlocks)); + convertedDatasetAttributes.getDatasetAccess().writeChunks(posKva, Arrays.asList(chunks)); } catch (final UncheckedIOException e) { throw new N5IOException( "Failed to write chunks into dataset " + datasetPath, e); @@ -276,15 +276,15 @@ default void writeChunks( default void writeChunk( final String path, final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws N5Exception { + final DataBlock chunk) throws N5Exception { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes); - convertedDatasetAttributes. getDatasetAccess().writeChunk(posKva, dataBlock); + convertedDatasetAttributes. getDatasetAccess().writeChunk(posKva, chunk); } catch (final UncheckedIOException e) { throw new N5IOException( - "Failed to write chunk " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, + "Failed to write chunk " + Arrays.toString(chunk.getGridPosition()) + " into dataset " + path, e); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 60c5700b1..4100f6421 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -212,7 +212,7 @@ default DatasetAttributes createDataset( final String normalPath = N5URI.normalizeGroupPath(datasetPath); createGroup(normalPath); - DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); + final DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); setDatasetAttributes(normalPath, convertedDatasetAttributes); return convertedDatasetAttributes; } @@ -245,32 +245,32 @@ default DatasetAttributes createDataset( * * @param datasetPath dataset path * @param datasetAttributes the dataset attributes - * @param dataBlock the chunk as a DataBlock + * @param chunk the chunk as a DataBlock * @param the data block data type * @throws N5Exception the exception */ void writeChunk( final String datasetPath, final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws N5Exception; + final DataBlock chunk) throws N5Exception; /** * Write multiple chunks represented by {@link DataBlock}s, useful for aggregation. * * @param datasetPath dataset path * @param datasetAttributes the dataset attributes - * @param dataBlocks the chunks + * @param chunks the chunks * @param the data block data type * @throws N5Exception the exception */ default void writeChunks( final String datasetPath, final DatasetAttributes datasetAttributes, - final DataBlock... dataBlocks) throws N5Exception { + final DataBlock... chunks) throws N5Exception { // default method is naive DatasetAttributes convertedAttributes = getConvertedDatasetAttributes(datasetAttributes); - for (DataBlock block : dataBlocks) { + for (DataBlock block : chunks) { writeChunk(datasetPath, convertedAttributes, block); } } @@ -278,7 +278,9 @@ default void writeChunks( /** * Writes a block stored as a {@link DataBlock}. *

- * A block is the highest (coarsest) level of the dataset's {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid}. + * A block is the highest (coarsest) level of the dataset's {@link NestedGrid}, + * that is, a shard if the dataset is sharded, or a chunk otherwise. + *

* This method's behavior is identical to {@link #writeChunk} for un-sharded datasets. * * @param pathName dataset path @@ -313,25 +315,25 @@ interface DataBlockSupplier { * @param datasetAttributes the dataset attributes * @param min min pixel coordinate of region to write * @param size size in pixels of region to write - * @param dataBlocks is asked to create chunks within the given region + * @param chunkSupplier is asked to create chunks within the given region * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. * @throws N5Exception the exception */ void writeRegion( - final String datasetPath, - final DatasetAttributes datasetAttributes, - final long[] min, - final long[] size, - final DataBlockSupplier dataBlocks, - final boolean writeFully) throws N5Exception; + String datasetPath, + DatasetAttributes datasetAttributes, + long[] min, + long[] size, + DataBlockSupplier chunkSupplier, + boolean writeFully) throws N5Exception; /** * @param datasetPath the dataset path * @param datasetAttributes the dataset attributes * @param min min pixel coordinate of region to write * @param size size in pixels of region to write - * @param dataBlocks is asked to create blocks within the given region - * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * @param chunkSupplier is asked to create blocks within the given region + * @param writeFully if false, merge existing data in shards/chunks that overlap the region boundary. if true, override everything. * @param exec used to parallelize over blocks (chunks and shards) * @throws N5Exception the exception */ @@ -340,7 +342,7 @@ void writeRegion( DatasetAttributes datasetAttributes, long[] min, long[] size, - DataBlockSupplier dataBlocks, + DataBlockSupplier chunkSupplier, boolean writeFully, ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index f6d158261..d1afb53f6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -90,14 +90,13 @@ public interface DatasetAccess { /** * Writes a chunk to the given storage position. - * */ - void writeChunk(PositionValueAccess pva, DataBlock dataBlock) throws N5IOException; + void writeChunk(PositionValueAccess pva, DataBlock chunk) throws N5IOException; /** * Writes multiple chunks to the given storage positions. */ - void writeChunks(PositionValueAccess pva, List> blocks) throws N5IOException; + void writeChunks(PositionValueAccess pva, List> chunks) throws N5IOException; /** * Deletes the chunk at {@code gridPosition}. @return true if it existed and diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 3c652a353..aaded5014 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -171,18 +171,18 @@ private void readChunksRecursive( } @Override - public void writeChunk(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException { + public void writeChunk(final PositionValueAccess pva, final DataBlock chunk) throws N5IOException { if (grid.numLevels() == 1) { @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[0]; - pva.set(dataBlock.getGridPosition(), codec.encode(dataBlock)); + pva.set(chunk.getGridPosition(), codec.encode(chunk)); } else { - final NestedPosition position = grid.nestedPosition(dataBlock.getGridPosition()); + final NestedPosition position = grid.nestedPosition(chunk.getGridPosition()); final long[] key = position.key(); final ReadData modifiedData; try (final VolatileReadData existingData = pva.get(key)) { - modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1); + modifiedData = writeBlockRecursive(existingData, chunk, position, grid.numLevels() - 1); // Here, we are about to write the shard data, but with the new block modified. // Need to make sure that the read operations happen now before pva.set acquires a write lock modifiedData.materialize(); @@ -216,16 +216,16 @@ private ReadData writeBlockRecursive( } @Override - public void writeChunks(final PositionValueAccess pva, final List> dataBlocks) throws N5IOException { + public void writeChunks(final PositionValueAccess pva, final List> chunks) throws N5IOException { if (grid.numLevels() == 1) { @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[0]; - dataBlocks.forEach(dataBlock -> pva.set(dataBlock.getGridPosition(), codec.encode(dataBlock))); + chunks.forEach(dataBlock -> pva.set(dataBlock.getGridPosition(), codec.encode(dataBlock))); } else { // Create a list of DataBlockRequests, sorted such that requests from // the same (nested) shard are grouped contiguously. - final DataBlockRequests requests = createWriteRequests(dataBlocks); + final DataBlockRequests requests = createWriteRequests(chunks); requests.removeDuplicates(); final List> split = requests.split(); for (final DataBlockRequests subRequests : split) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index 661d556cf..4f217c097 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -56,7 +56,7 @@ public class HttpReaderFsWriter implements GsonKeyValueN5Writer { private final GsonKeyValueN5Reader reader; public HttpReaderFsWriter(final W writer, final R reader) { - + this.writer = writer; this.reader = reader; @@ -240,7 +240,7 @@ public HttpRead writer.setDatasetAttributes(datasetPath, datasetAttributes); } - + @Override public DatasetAttributes getConvertedDatasetAttributes(DatasetAttributes datasetAttributes) { return writer.getConvertedDatasetAttributes(datasetAttributes); @@ -272,8 +272,8 @@ public HttpRead return convertedDatasetAttributes; } - @Override public void writeChunk(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { - writer.writeChunk(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); + @Override public void writeChunk(String datasetPath, DatasetAttributes datasetAttributes, DataBlock chunk) throws N5Exception { + writer.writeChunk(datasetPath, getConvertedDatasetAttributes(datasetAttributes), chunk); } @Override public void writeBlock(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { @@ -315,8 +315,8 @@ public HttpRead writer.writeAttributes(normalGroupPath, attributes); } - @Override public void writeChunks(String datasetPath, DatasetAttributes datasetAttributes, DataBlock... dataBlocks) throws N5Exception { + @Override public void writeChunks(String datasetPath, DatasetAttributes datasetAttributes, DataBlock... chunks) throws N5Exception { - writer.writeChunks(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlocks); + writer.writeChunks(datasetPath, getConvertedDatasetAttributes(datasetAttributes), chunks); } } From d5e7510f38efcd06a695164e7b7e9200986dae01 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 25 Mar 2026 17:46:28 +0100 Subject: [PATCH 17/41] fix javadoc --- .../org/janelia/saalfeldlab/n5/N5Writer.java | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 4100f6421..713db1461 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -39,6 +39,7 @@ import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; /** * A simple structured container API for hierarchies of chunked @@ -55,7 +56,7 @@ public interface N5Writer extends N5Reader { * @param groupPath group path * @param attributePath the key * @param attribute the attribute - * @param the attribute type type + * @param the attribute type * @throws N5Exception the exception */ default void setAttribute( @@ -348,6 +349,13 @@ void writeRegion( /** * Deletes the block at {@code gridPosition}. + *

+ * A block is the highest (coarsest) level of the dataset's {@link NestedGrid}, + * that is, a shard if the dataset is sharded, or a chunk otherwise. + * Note that {@code gridPosition} is in units of blocks at the highest + * (coarsest) level, too. + *

+ * This method's behavior is identical to {@link #deleteChunk} for un-sharded datasets. * * @param datasetPath dataset path * @param gridPosition position of block to be deleted @@ -364,6 +372,13 @@ default boolean deleteBlock( /** * Deletes the block at {@code gridPosition}. + *

+ * A block is the highest (coarsest) level of the dataset's {@link NestedGrid}, + * that is, a shard if the dataset is sharded, or a chunk otherwise. + * Note that {@code gridPosition} is in units of blocks at the highest + * (coarsest) level, too. + *

+ * This method's behavior is identical to {@link #deleteChunk} for un-sharded datasets. * * @param datasetPath the dataset path * @param datasetAttributes the dataset attributes @@ -379,6 +394,8 @@ boolean deleteBlock( /** * Deletes the chunk at {@code gridPosition}. + *

+ * Note that {@code gridPosition} is in units of chunks. * * @param datasetPath dataset path * @param gridPosition position of chunk to be deleted @@ -395,6 +412,8 @@ default boolean deleteChunk( /** * Deletes the chunk at {@code gridPosition}. + *

+ * Note that {@code gridPosition} is in units of chunks. * * @param datasetPath the dataset path * @param datasetAttributes the dataset attributes @@ -410,6 +429,8 @@ boolean deleteChunk( /** * Deletes the chunks at the given {@code gridPositions}. + *

+ * Note that {@code gridPositions} are in units of chunks. * * @param datasetPath dataset path * @param gridPositions a list of grid positions @@ -429,8 +450,7 @@ default boolean deleteChunks( /** * Save a {@link Serializable} as an N5 {@link DataBlock} at a given offset. - * The - * offset is given in {@link DataBlock} grid coordinates. + * The offset is given in {@link DataBlock} grid coordinates. * * @param object the object to serialize * @param datasetPath the dataset path From 75535e2865e17a4cee54b37c6153e2c17881e5be Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 25 Mar 2026 17:53:35 +0100 Subject: [PATCH 18/41] clean up redundant modifiers --- .../org/janelia/saalfeldlab/n5/N5Reader.java | 46 +++++++++---------- .../org/janelia/saalfeldlab/n5/N5URI.java | 3 +- .../org/janelia/saalfeldlab/n5/N5Writer.java | 28 +++++------ 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 98e73af93..acd8a1de7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -56,7 +56,7 @@ */ public interface N5Reader extends AutoCloseable { - public static class Version { + class Version { private final int major; private final int minor; @@ -193,17 +193,17 @@ public boolean isCompatible(final Version version) { /** * SemVer version of this N5 spec. */ - public static final Version NO_VERSION = new Version(0, 0, 0); + Version NO_VERSION = new Version(0, 0, 0); /** * SemVer version of this N5 spec. */ - public static final Version VERSION = new Version(4, 0, 0); + Version VERSION = new Version(4, 0, 0); /** * Version attribute key. */ - public static final String VERSION_KEY = "n5"; + String VERSION_KEY = "n5"; /** * Get the SemVer version of this container as specified in the 'version' @@ -246,9 +246,9 @@ default Version getVersion() throws N5Exception { * the exception */ T getAttribute( - final String pathName, - final String key, - final Class clazz) throws N5Exception; + String pathName, + String key, + Class clazz) throws N5Exception; /** * Reads an attribute. @@ -266,9 +266,9 @@ T getAttribute( * the exception */ T getAttribute( - final String pathName, - final String key, - final Type type) throws N5Exception; + String pathName, + String key, + Type type) throws N5Exception; /** * Get mandatory dataset attributes. @@ -280,7 +280,7 @@ T getAttribute( * @throws N5Exception * the exception */ - DatasetAttributes getDatasetAttributes(final String pathName) throws N5Exception; + DatasetAttributes getDatasetAttributes(String pathName) throws N5Exception; /** * Some implementations may need to convert arbitrary DatasetAttributes to their specific equivalent variant. @@ -314,9 +314,9 @@ default DatasetAttributes getConvertedDatasetAttributes(final DatasetAttributes * the exception */ DataBlock readChunk( - final String pathName, - final DatasetAttributes datasetAttributes, - final long... gridPosition) throws N5Exception; + String pathName, + DatasetAttributes datasetAttributes, + long... gridPosition) throws N5Exception; /** * Reads multiple chunks as {@link DataBlock}s. @@ -370,9 +370,9 @@ default List> readChunks( * @see DatasetAttributes#getNestedBlockGrid() */ DataBlock readBlock( - final String pathName, - final DatasetAttributes datasetAttributes, - final long... gridPosition) throws N5Exception; + String pathName, + DatasetAttributes datasetAttributes, + long... gridPosition) throws N5Exception; /** * Checks if a block exists at the given grid position without reading the data. @@ -396,9 +396,9 @@ DataBlock readBlock( * the exception */ boolean blockExists( - final String pathName, - final DatasetAttributes datasetAttributes, - final long... gridPosition) throws N5Exception; + String pathName, + DatasetAttributes datasetAttributes, + long... gridPosition) throws N5Exception; /** * Load a {@link DataBlock} as a {@link Serializable}. The offset is given @@ -444,7 +444,7 @@ default T readSerializedBlock( * group path * @return true if the path exists */ - boolean exists(final String pathName); + boolean exists(String pathName); /** * Test whether a dataset exists at a given path. @@ -469,7 +469,7 @@ default boolean datasetExists(final String pathName) throws N5Exception { * @throws N5Exception * an exception is thrown if pathName is not a valid group */ - String[] list(final String pathName) throws N5Exception; + String[] list(String pathName) throws N5Exception; /** * Recursively list all groups and datasets in the given path. @@ -873,7 +873,7 @@ static void deepListHelper( * @return a map of attribute keys to their inferred class * @throws N5Exception if an error occurred during listing */ - Map> listAttributes(final String pathName) throws N5Exception; + Map> listAttributes(String pathName) throws N5Exception; /** * Returns the symbol that is used to separate nodes in a group path. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java index c086acc05..9bd9fa326 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java @@ -36,6 +36,7 @@ import java.nio.charset.CharsetDecoder; import java.nio.charset.CoderResult; import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -53,7 +54,7 @@ */ public class N5URI { - private static final Charset UTF8 = Charset.forName("UTF-8"); + private static final Charset UTF8 = StandardCharsets.UTF_8; public static final Pattern ARRAY_INDEX = Pattern.compile("\\[([0-9]+)]"); final URI uri; private final String scheme; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 713db1461..dbce32209 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -78,8 +78,8 @@ default void setAttribute( * @throws N5Exception the exception */ void setAttributes( - final String groupPath, - final Map attributes) throws N5Exception; + String groupPath, + Map attributes) throws N5Exception; /** * Remove the attribute from group {@code pathName} with key {@code key}. @@ -165,7 +165,7 @@ default void setVersion() throws N5Exception { * @param groupPath the path * @throws N5Exception the exception */ - void createGroup(final String groupPath) throws N5Exception; + void createGroup(String groupPath) throws N5Exception; /** * Removes a group or dataset (directory and all contained files). @@ -183,7 +183,7 @@ default void setVersion() throws N5Exception { * @return true if removal was successful, false otherwise * @throws N5Exception the exception */ - boolean remove(final String groupPath) throws N5Exception; + boolean remove(String groupPath) throws N5Exception; /** * Removes the N5 container. @@ -251,9 +251,9 @@ default DatasetAttributes createDataset( * @throws N5Exception the exception */ void writeChunk( - final String datasetPath, - final DatasetAttributes datasetAttributes, - final DataBlock chunk) throws N5Exception; + String datasetPath, + DatasetAttributes datasetAttributes, + DataBlock chunk) throws N5Exception; /** * Write multiple chunks represented by {@link DataBlock}s, useful for aggregation. @@ -293,9 +293,9 @@ default void writeChunks( * @see DatasetAttributes#getNestedBlockGrid() */ void writeBlock( - final String pathName, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws N5Exception; + String pathName, + DatasetAttributes datasetAttributes, + DataBlock dataBlock) throws N5Exception; @FunctionalInterface interface DataBlockSupplier { @@ -308,7 +308,7 @@ interface DataBlockSupplier { * * @return data block at the given gridPos */ - DataBlock get(long[] gridPos, final DataBlock existingDataBlock); + DataBlock get(long[] gridPos, DataBlock existingDataBlock); } /** @@ -438,9 +438,9 @@ boolean deleteChunk( * @throws N5Exception if any of the chunks did exist but could not be deleted */ default boolean deleteChunks( - String datasetPath, - DatasetAttributes datasetAttributes, - List gridPositions) throws N5Exception { + final String datasetPath, + final DatasetAttributes datasetAttributes, + final List gridPositions) throws N5Exception { boolean deleted = false; for (long[] pos : gridPositions) { deleted |= deleteChunk(datasetPath, datasetAttributes, pos); From d3eaf760141a051b9eb138188b111499c48a3c24 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 25 Mar 2026 18:10:39 +0100 Subject: [PATCH 19/41] fix javadoc and argument names --- .../org/janelia/saalfeldlab/n5/N5Writer.java | 2 +- .../saalfeldlab/n5/shard/DatasetAccess.java | 53 ++++++++++--------- .../n5/shard/DefaultDatasetAccess.java | 8 +-- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index dbce32209..6c4f35659 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -333,7 +333,7 @@ void writeRegion( * @param datasetAttributes the dataset attributes * @param min min pixel coordinate of region to write * @param size size in pixels of region to write - * @param chunkSupplier is asked to create blocks within the given region + * @param chunkSupplier is asked to create chunks within the given region * @param writeFully if false, merge existing data in shards/chunks that overlap the region boundary. if true, override everything. * @param exec used to parallelize over blocks (chunks and shards) * @throws N5Exception the exception diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index d1afb53f6..30441c3a9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -47,16 +47,16 @@ public interface DatasetAccess { /** - * Read the {@code DataBlock} at the given {@code gridPosition}. + * Read the chunk at the given {@code gridPosition}. *

- * If the requested block doesn't exist, then this method will return {@code + * If the requested chunk doesn't exist, then this method will return {@code * null}. {@code N5IOException} will be thrown if something goes wrong * reading or decoding data for an existing key. * * @param pva * dataset storage * @param gridPosition - * grid position of the DataBlock to read + * grid position of the chunks to read * * @return the DataBlock or {@code null} * @@ -66,13 +66,13 @@ public interface DatasetAccess { DataBlock readChunk(PositionValueAccess pva, long[] gridPosition) throws N5IOException; /** - * Read the {@code DataBlock}s at the given {@code gridPositions}. + * Read the chunks at the given {@code gridPositions}. *

* The returned {@code List>} is in the same order as the * requested {@code gridPositions}. That is, the {@code DataBlock} at index * {@code i} has grid coordinates {@code gridPositions.get(i)}. *

- * If a requested block doesn't exist, then the corresponding element in the + * If a requested chunk doesn't exist, then the corresponding element in the * result list will be {@code null}. ({@code N5IOException} will only be * thrown if something goes wrong reading or decoding data for an existing * key.) @@ -80,7 +80,7 @@ public interface DatasetAccess { * @param pva * dataset storage * @param gridPositions - * list of grid positions of the DataBlocks to read + * list of grid positions of the chunks to read * @return list of DataBlocks * * @throws N5IOException @@ -89,36 +89,39 @@ public interface DatasetAccess { List> readChunks(PositionValueAccess pva, List gridPositions) throws N5IOException; /** - * Writes a chunk to the given storage position. + * Writes a chunk to the {@link DataBlock#getGridPosition() grid position} + * specified by {@code chunk}. */ void writeChunk(PositionValueAccess pva, DataBlock chunk) throws N5IOException; /** - * Writes multiple chunks to the given storage positions. + * Writes multiple chunks to the {@link DataBlock#getGridPosition() grid + * positions} specified by the respective {@code chunks}. */ void writeChunks(PositionValueAccess pva, List> chunks) throws N5IOException; /** - * Deletes the chunk at {@code gridPosition}. @return true if it existed and - * was deleted. + * Deletes the chunk at {@code gridPosition}. + * + * @return true if it existed and was deleted. */ boolean deleteChunk(PositionValueAccess pva, long[] gridPosition) throws N5IOException; /** - * Deletes the chunks at the given {@code positions}. @return true if any - * existed and were deleted. + * Deletes the chunks at the given {@code positions}. + * + * @return true if any existed and were deleted. */ - boolean deleteChunks(PositionValueAccess pva, List positions) throws N5IOException; + boolean deleteChunks(PositionValueAccess pva, List gridPositions) throws N5IOException; /** - * * @param pva * @param min * min pixel coordinate of region to write * @param size * size in pixels of region to write - * @param blocks - * is asked to create blocks within the given region + * @param chunkSupplier + * is asked to create chunks within the given region * @param writeFully * if false, merge existing data in blocks/chunks that overlap the region boundary. if true, override everything. * @@ -128,7 +131,7 @@ void writeRegion( PositionValueAccess pva, long[] min, long[] size, - DataBlockSupplier blocks, + DataBlockSupplier chunkSupplier, boolean writeFully ) throws N5IOException; @@ -139,8 +142,8 @@ void writeRegion( * min pixel coordinate of region to write * @param size * size in pixels of region to write - * @param blocks - * is asked to create blocks within the given region. must be thread-safe. + * @param chunkSupplier + * is asked to create chunks within the given region. must be thread-safe. * @param writeFully * if false, merge existing data in blocks/chunks that overlap the region boundary. if true, override everything. * @param exec @@ -154,15 +157,15 @@ void writeRegion( PositionValueAccess pva, long[] min, long[] size, - DataBlockSupplier blocks, + DataBlockSupplier chunkSupplier, boolean writeFully, ExecutorService exec ) throws N5Exception, InterruptedException, ExecutionException; /** - * Read a block at {@code shardGridPosition} at the given nesting - * {@code level}. The shard data is rearranged and assembled into a (large) - * {@code DataBlock}. + * Read a block at {@code shardGridPosition} at the given nesting {@code + * level}. The data is read as chunks and then rearranged and assembled into + * a (large) {@code DataBlock}. *

* The {@code getGridPosition()} of the returned {@code DataBlock} is the * grid position with respect to {@code level}. For example, if {@code @@ -182,11 +185,11 @@ void writeRegion( /** * Write a full block at the given nesting {@code level}. The block data is * given as a (large) {@code DataBlock} that will be sliced, rearranged, and - * written as (level-0) DataBlocks. + * written as chunks (level-0 DataBlocks). *

* {@code dataBlock.getGridPosition()} is the grid position with respect to * {@code level}. For example, if {@code level==1}, then this refers to the - * position on the block grid. + * position on the level-1 shard grid. * * @param pva * dataset storage diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index aaded5014..9dc972f08 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -290,7 +290,7 @@ public void writeRegion( final PositionValueAccess pva, final long[] min, final long[] size, - final DataBlockSupplier blocks, + final DataBlockSupplier chunkSupplier, final boolean writeFully ) throws N5IOException { @@ -301,7 +301,7 @@ public void writeRegion( final boolean nestedWriteFully = writeFully || region.fullyContains(pos); final ReadData modifiedData; try (final VolatileReadData existingData = nestedWriteFully ? null : pva.get(key)) { - modifiedData = writeRegionRecursive(existingData, region, blocks, pos); + modifiedData = writeRegionRecursive(existingData, region, chunkSupplier, pos); // Here, we are about to write the shard data, but with the new block modified. // Need to make sure that the read operations happen now before pva.set acquires a write lock if (existingData != null && modifiedData != null) { @@ -317,7 +317,7 @@ public void writeRegion( final PositionValueAccess pva, final long[] min, final long[] size, - final DataBlockSupplier blocks, + final DataBlockSupplier chunkSupplier, final boolean writeFully, final ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException { @@ -329,7 +329,7 @@ public void writeRegion( final boolean nestedWriteFully = writeFully || region.fullyContains(pos); final ReadData modifiedData; try (final VolatileReadData existingData = nestedWriteFully ? null : pva.get(key)) { - modifiedData = writeRegionRecursive(existingData, region, blocks, pos); + modifiedData = writeRegionRecursive(existingData, region, chunkSupplier, pos); // Here, we are about to write the shard data, but with the new block modified. // Need to make sure that the read operations happen now before pva.set acquires a write lock if (existingData != null && modifiedData != null) { From 96cb3f5557e30b5f889ce67cc40b457b7fae00dd Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 25 Mar 2026 21:08:15 +0100 Subject: [PATCH 20/41] renaming in Nesting and DefaultDatasetAccess --- .../n5/shard/DefaultDatasetAccess.java | 317 +++++++++--------- .../janelia/saalfeldlab/n5/shard/Nesting.java | 22 +- .../saalfeldlab/n5/shard/NestedGridTest.java | 2 +- 3 files changed, 167 insertions(+), 174 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 9dc972f08..72f650686 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -102,18 +102,18 @@ private DataBlock readChunkRecursive( @Override public List> readChunks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { - // for non-sharded datasets, just read the blocks individually + // for non-sharded datasets, just read the chunks individually if (grid.numLevels() == 1) { return gridPositions.stream().map(pos -> readChunk(pva, pos)).collect(Collectors.toList()); } - // Create a list of DataBlockRequests and sort it such that requests + // Create a list of ChunkRequests and sort it such that requests // from the same (nested) shard are grouped contiguously. - final DataBlockRequests requests = createReadRequests(gridPositions); - final List> duplicates = requests.removeDuplicates(); + final ChunkRequests requests = createReadRequests(gridPositions); + final List> duplicates = requests.removeDuplicates(); - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { + final List> split = requests.split(); + for (final ChunkRequests subRequests : split) { final long[] key = subRequests.relativeGridPosition(); try (final VolatileReadData readData = pva.get(key)) { readChunksRecursive(readData, subRequests); @@ -124,7 +124,7 @@ public List> readChunks(final PositionValueAccess pva, final List> readChunks(final PositionValueAccess pva, final List requests + final ChunkRequests requests ) { assert !requests.requests.isEmpty(); assert requests.level > 0; @@ -150,19 +150,19 @@ private void readChunksRecursive( final RawShard shard = codec.decode(readData, requests.gridPosition()).getData(); if (level == 1 ) { - //Base case; read the blocks + //Base case; read the chunks // TODO: collect all the elementPos that we will need and prefetch // Probably best to add a prefetch method to RawShard? - for (final DataBlockRequest request : requests) { + for (final ChunkRequest request : requests) { final long[] elementPos = request.position.relative(0); final ReadData elementData = shard.getElementData(elementPos); - request.block = readChunkRecursive(elementData, request.position, 0); + request.chunk = readChunkRecursive(elementData, request.position, 0); } } else { // level > 1 - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { + final List> split = requests.split(); + for (final ChunkRequests subRequests : split) { final long[] subShardPosition = subRequests.relativeGridPosition(); final ReadData elementData = shard.getElementData(subShardPosition); readChunksRecursive(elementData, subRequests); @@ -182,7 +182,7 @@ public void writeChunk(final PositionValueAccess pva, final DataBlock chunk) final long[] key = position.key(); final ReadData modifiedData; try (final VolatileReadData existingData = pva.get(key)) { - modifiedData = writeBlockRecursive(existingData, chunk, position, grid.numLevels() - 1); + modifiedData = writeChunkRecursive(existingData, chunk, position, grid.numLevels() - 1); // Here, we are about to write the shard data, but with the new block modified. // Need to make sure that the read operations happen now before pva.set acquires a write lock modifiedData.materialize(); @@ -191,15 +191,15 @@ public void writeChunk(final PositionValueAccess pva, final DataBlock chunk) } } - private ReadData writeBlockRecursive( + private ReadData writeChunkRecursive( final ReadData existingReadData, - final DataBlock dataBlock, + final DataBlock chunk, final NestedPosition position, final int level) { if (level == 0) { @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[0]; - return codec.encode(dataBlock); + return codec.encode(chunk); } else { @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[level]; @@ -209,7 +209,7 @@ private ReadData writeBlockRecursive( final ReadData existingElementData = (level == 1) ? null // if level == 1, we don't need to extract the nested (DataBlock) ReadData because it will be overridden anyway : shard.getElementData(elementPos); - final ReadData modifiedElementData = writeBlockRecursive(existingElementData, dataBlock, position, level - 1); + final ReadData modifiedElementData = writeChunkRecursive(existingElementData, chunk, position, level - 1); shard.setElementData(modifiedElementData, elementPos); return codec.encode(new RawShardDataBlock(gridPos, shard)); } @@ -221,14 +221,14 @@ public void writeChunks(final PositionValueAccess pva, final List> if (grid.numLevels() == 1) { @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[0]; - chunks.forEach(dataBlock -> pva.set(dataBlock.getGridPosition(), codec.encode(dataBlock))); + chunks.forEach(chunk -> pva.set(chunk.getGridPosition(), codec.encode(chunk))); } else { - // Create a list of DataBlockRequests, sorted such that requests from + // Create a list of ChunkRequests, sorted such that requests from // the same (nested) shard are grouped contiguously. - final DataBlockRequests requests = createWriteRequests(chunks); + final ChunkRequests requests = createWriteRequests(chunks); requests.removeDuplicates(); - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { + final List> split = requests.split(); + for (final ChunkRequests subRequests : split) { final boolean writeFully = subRequests.coversShard(); final long[] shardKey = subRequests.relativeGridPosition(); final ReadData modifiedData; @@ -251,7 +251,7 @@ public void writeChunks(final PositionValueAccess pva, final List> */ private ReadData writeChunksRecursive( final ReadData existingReadData, // may be null - final DataBlockRequests requests + final ChunkRequests requests ) { assert !requests.requests.isEmpty(); assert requests.level > 0; @@ -266,14 +266,14 @@ private ReadData writeChunksRecursive( if ( level == 1 ) { // Base case, write the blocks - for (final DataBlockRequest request : requests) { - final ReadData elementData = writeBlockRecursive(null, request.block, request.position, 0); + for (final ChunkRequest request : requests) { + final ReadData elementData = writeChunkRecursive(null, request.chunk, request.position, 0); final long[] elementPos = request.position.relative(0); shard.setElementData(elementData, elementPos); } } else { // level > 1 - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { + final List> split = requests.split(); + for (final ChunkRequests subRequests : split) { final boolean nestedWriteFully = writeFully || subRequests.coversShard(); final long[] elementPos = subRequests.relativeGridPosition(); final ReadData existingElementData = nestedWriteFully ? null : shard.getElementData(elementPos); @@ -344,7 +344,7 @@ public void writeRegion( private ReadData writeRegionRecursive( final ReadData existingReadData, // may be null final Region region, - final DataBlockSupplier blocks, + final DataBlockSupplier chunkSupplier, final NestedPosition position ) { final boolean writeFully = existingReadData == null; @@ -360,21 +360,21 @@ private ReadData writeRegionRecursive( // existing DataBlock and pass it to the BlockSupplier for modification. // (This might fail with N5NoSuchKeyException if existingReadData // lazily points to non-existent data.) - DataBlock existingDataBlock = null; + DataBlock existingChunk = null; if (existingReadData != null) { try { - existingDataBlock = codec.decode(existingReadData, gridPosition); + existingChunk = codec.decode(existingReadData, gridPosition); } catch (N5NoSuchKeyException ignored) { } } - final DataBlock dataBlock = blocks.get(gridPosition, existingDataBlock); + final DataBlock chunk = chunkSupplier.get(gridPosition, existingChunk); // null blocks may be provided when they contain only the fill value // and only non-empty blocks should be written, for example - if (dataBlock == null) + if (chunk == null) return null; - return codec.encode(dataBlock); + return codec.encode(chunk); } else { @SuppressWarnings("unchecked") @@ -385,7 +385,7 @@ private ReadData writeRegionRecursive( final boolean nestedWriteFully = writeFully || region.fullyContains(pos); final long[] elementPos = pos.relative(); final ReadData existingElementData = nestedWriteFully ? null : shard.getElementData(elementPos); - final ReadData modifiedElementData = writeRegionRecursive(existingElementData, region, blocks, pos); + final ReadData modifiedElementData = writeRegionRecursive(existingElementData, region, chunkSupplier, pos); shard.setElementData(modifiedElementData, elementPos); } @@ -398,7 +398,7 @@ private ReadData writeRegionRecursive( } // - // -- deleteBlock --------------------------------------------------------- + // -- deleteChunk --------------------------------------------------------- @Override public boolean deleteChunk(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { @@ -472,7 +472,7 @@ private ReadData deleteChunkRecursive( } // - // -- deleteBlocks -------------------------------------------------------- + // -- deleteChunks -------------------------------------------------------- @Override public boolean deleteChunks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { @@ -486,25 +486,25 @@ public boolean deleteChunks(final PositionValueAccess pva, final List gr return deleted; } else { - // Create a list of DataBlockRequests and sort it such that requests + // Create a list of ChunkRequests and sort it such that requests // from the same (nested) shard are grouped contiguously. // Despite the name, createReadRequests() works for delete requests as well ... - final DataBlockRequests requests = createReadRequests(gridPositions); + final ChunkRequests requests = createReadRequests(gridPositions); requests.removeDuplicates(); boolean deleted = false; - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { + final List> split = requests.split(); + for (final ChunkRequests subRequests : split) { final boolean writeFully = subRequests.coversShard(); final long[] key = subRequests.relativeGridPosition(); final ReadData modifiedData; try (final VolatileReadData existingData = writeFully ? null : pva.get(key)) { - modifiedData = deleteBlocksRecursive(existingData, subRequests);; + modifiedData = deleteChunksRecursive(existingData, subRequests);; if (modifiedData == existingData) { - // nothing changed, the blocks we wanted to delete didn't exist anyway + // nothing changed, the chunks we wanted to delete didn't exist anyway continue; } else if (existingData != null && modifiedData != null) { - // Here, we are about to write the shard data, but without the block to be deleted. + // Here, we are about to write the shard data, but without the chunk to be deleted. // Need to make sure that the read operations happen now before pva.set acquires a write lock modifiedData.materialize(); } @@ -528,11 +528,11 @@ public boolean deleteChunks(final PositionValueAccess pva, final List gr * Bulk Delete operation on a shard. * * @param existingReadData encoded existing shard data (to decode and partially override) - * @param requests for blocks within the shard to be deleted + * @param requests for chunks within the shard to be deleted */ - private ReadData deleteBlocksRecursive( + private ReadData deleteChunksRecursive( final ReadData existingReadData, // may be null - final DataBlockRequests requests + final ChunkRequests requests ) { assert !requests.requests.isEmpty(); assert requests.level > 0; @@ -550,8 +550,8 @@ private ReadData deleteBlocksRecursive( boolean modified = false; boolean shardElementSetToNull = false; if ( level == 1 ) { - // Base case, delete the blocks - for (final DataBlockRequest request : requests) { + // Base case, delete the chunks + for (final ChunkRequest request : requests) { final long[] elementPos = request.position.relative(0); if (shard.getElementData(elementPos) != null) { shard.setElementData(null, elementPos); @@ -560,12 +560,12 @@ private ReadData deleteBlocksRecursive( } } } else { // level > 1 - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { + final List> split = requests.split(); + for (final ChunkRequests subRequests : split) { final boolean writeFully = subRequests.coversShard(); final long[] elementPos = subRequests.relativeGridPosition(); final ReadData existingElementData = writeFully ? null : shard.getElementData(elementPos); - final ReadData modifiedElementData = deleteBlocksRecursive(existingElementData, subRequests); + final ReadData modifiedElementData = deleteChunksRecursive(existingElementData, subRequests); if (modifiedElementData != existingElementData) { shard.setElementData(modifiedElementData, elementPos); modified = true; @@ -575,13 +575,13 @@ private ReadData deleteBlocksRecursive( } if (!modified) { - // No nested shard or DataBlock was modified. + // No nested shard or chunk was modified. // This shard remains unchanged. return existingReadData; } if (shardElementSetToNull) { - // At least one DataBlock or nested shard was removed. + // At least one chunk or nested shard was removed. // Check whether this shard becomes empty. if (shard.index().allElementsNull()) { // This shard is empty and should be removed. @@ -625,7 +625,7 @@ private ReadData deleteBlocksRecursive( // do not want to write DataBlocks that are completely outside the // dataset (even if the "big DataBlock" covers this area.) // - // This works for writing. For reading we'll have to decide what to return, + // This works for writing. For reading, we'll have to decide what to return, // though... Potentially, there is no valid block at the border, so we // cannot determine where to put the border just from the data. We need to // rely on external input or heuristics. @@ -674,50 +674,50 @@ private DataBlock readBlockInternal( } } - // level-0 block-grid position of the min block in the shard + // level-0 block-grid position of the min chunk in the shard final long[] gridMin = grid.absolutePosition(shardGridPosition, level, 0); - // level-0 block-grid position of the max block in the shard that we need to read. + // level-0 block-grid position of the max chunk in the shard that we need to read. // (the shard might go beyond the dataset border, and we don't need to read anything there) final long[] gridMax = new long[n]; - final long[] datasetSizeInBlocks = grid.getDatasetSizeInBlocks(); - final int[] blockSize = grid.getBlockSize(0); + final long[] datasetSizeInChunks = grid.getDatasetSizeInChunks(); + final int[] chunkSize = grid.getBlockSize(0); for (int d = 0; d < n; ++d) { - final int shardSizeInBlocks = (shardSizeInPixels[d] + blockSize[d] - 1) / blockSize[d]; - final int gridSize = Math.min(shardSizeInBlocks, (int) (datasetSizeInBlocks[d] - gridMin[d])); + final int shardSizeInChunks = (shardSizeInPixels[d] + chunkSize[d] - 1) / chunkSize[d]; + final int gridSize = Math.min(shardSizeInChunks, (int) (datasetSizeInChunks[d] - gridMin[d])); gridMax[d] = gridMin[d] + gridSize - 1; } - // read all blocks in (gridMin, gridMax) and filter out missing blocks - final List blockPositions = Region.gridPositions(gridMin, gridMax); - final List> blocks = readChunks(pva, blockPositions) + // read all chunks in (gridMin, gridMax) and filter out missing chunks + final List chunkPositions = Region.gridPositions(gridMin, gridMax); + final List> chunks = readChunks(pva, chunkPositions) .stream().filter(Objects::nonNull).collect(Collectors.toList()); - if (blocks.isEmpty()) { + if (chunks.isEmpty()) { return null; } - // allocate shard and copy data from blocks - final DataBlock shard = DataBlockFactory.of(blocks.get(0).getData()).createDataBlock(shardSizeInPixels, shardGridPosition); + // allocate shard and copy data from chunks + final DataBlock shard = DataBlockFactory.of(chunks.get(0).getData()).createDataBlock(shardSizeInPixels, shardGridPosition); final long[] shardPixelPos = grid.pixelPosition(shardGridPosition, level); - final long[] blockPixelPos = new long[n]; + final long[] chunkPixelPos = new long[n]; final int[] srcPos = new int[n]; final int[] destPos = new int[n]; final int[] size = new int[n]; - for (final DataBlock block : blocks) { - // copy block data that overlaps the shard - grid.pixelPosition(block.getGridPosition(), 0, blockPixelPos); - final int[] bsize = block.getSize(); + for (final DataBlock chunk : chunks) { + // copy chunk data that overlaps the shard + grid.pixelPosition(chunk.getGridPosition(), 0, chunkPixelPos); + final int[] bsize = chunk.getSize(); for (int d = 0; d < n; d++) { - destPos[d] = (int) (blockPixelPos[d] - shardPixelPos[d]); + destPos[d] = (int) (chunkPixelPos[d] - shardPixelPos[d]); size[d] = Math.min(bsize[d], shardSizeInPixels[d] - destPos[d]); } - SubArrayCopy.copy(block.getData(), bsize, srcPos, shard.getData(), shardSizeInPixels, destPos, size); + SubArrayCopy.copy(chunk.getData(), bsize, srcPos, shard.getData(), shardSizeInPixels, destPos, size); } return shard; } // - // -- writeShard ---------------------------------------------------------- + // -- writeBlock ---------------------------------------------------------- @Override public void writeBlock( @@ -735,49 +735,49 @@ public void writeBlock( final DataBlockFactory blockFactory = DataBlockFactory.of(shardData); final int n = grid.numDimensions(); - final int[] blockSize = grid.getBlockSize(0); // size of a standard (non-truncated) DataBlock - final long[] datasetBlockSize = grid.getDatasetSizeInBlocks(); + final int[] chunkSize = grid.getBlockSize(0); // size of a standard (non-truncated) DataBlock + final long[] datasetChunkSize = grid.getDatasetSizeInChunks(); final long[] shardPixelMin = grid.pixelPosition(dataBlock.getGridPosition(), level); final int[] shardPixelSize = dataBlock.getSize(); - // the max block + 1 in the shard, if it isn't truncated by the dataset border - final long[] shardBlockTo = new long[n]; - Arrays.setAll(shardBlockTo, d -> (shardPixelMin[d] + shardPixelSize[d] + blockSize[d] - 1) / blockSize[d]); + // the max chunk + 1 in the shard, if it isn't truncated by the dataset border + final long[] shardChunkTo = new long[n]; + Arrays.setAll(shardChunkTo, d -> (shardPixelMin[d] + shardPixelSize[d] + chunkSize[d] - 1) / chunkSize[d]); - // level 0 DataBlock positions of all DataBlocks we want to extract + // level 0 grid positions of all chunks we want to extract final long[] gridMin = grid.absolutePosition(dataBlock.getGridPosition(), level, 0); final long[] gridMax = new long[n]; - Arrays.setAll(gridMax, d -> Math.min(shardBlockTo[d], datasetBlockSize[d]) - 1); - final List blockPositions = Region.gridPositions(gridMin, gridMax); + Arrays.setAll(gridMax, d -> Math.min(shardChunkTo[d], datasetChunkSize[d]) - 1); + final List chunkPositions = Region.gridPositions(gridMin, gridMax); // Max pixel coordinates + 1, of the region we want to copy. This should // always be shardPixelMin + shardPixelSize, except at the dataset - // border, where we truncate to the smallest multiple of blockSize still + // border, where we truncate to the smallest multiple of chunkSize still // overlapping the dataset. final long[] regionBound = new long[n]; - Arrays.setAll(regionBound, d -> Math.min(shardPixelMin[d] + shardPixelSize[d], datasetBlockSize[d] * blockSize[d])); + Arrays.setAll(regionBound, d -> Math.min(shardPixelMin[d] + shardPixelSize[d], datasetChunkSize[d] * chunkSize[d])); - final List> blocks = new ArrayList<>(blockPositions.size()); + final List> chunks = new ArrayList<>(chunkPositions.size()); final int[] srcPos = new int[n]; final int[] destPos = new int[n]; final int[] destSize = new int[n]; - for ( long[] blockPos : blockPositions) { - final long[] pixelMin = grid.pixelPosition(blockPos, 0); + for ( long[] chunkPos : chunkPositions) { + final long[] pixelMin = grid.pixelPosition(chunkPos, 0); for (int d = 0; d < n; d++) { srcPos[d] = (int) (pixelMin[d] - shardPixelMin[d]); - destSize[d] = Math.min(blockSize[d], (int) (regionBound[d] - pixelMin[d])); + destSize[d] = Math.min(chunkSize[d], (int) (regionBound[d] - pixelMin[d])); } // This extracting DataBlocks will not work if num_array_elements != num_block_elements. // But we'll deal with that later if it becomes a problem... - final DataBlock block = blockFactory.createDataBlock(destSize, blockPos); - SubArrayCopy.copy(shardData, shardPixelSize, srcPos, block.getData(), destSize, destPos, destSize); - blocks.add(block); + final DataBlock chunk = blockFactory.createDataBlock(destSize, chunkPos); + SubArrayCopy.copy(shardData, shardPixelSize, srcPos, chunk.getData(), destSize, destPos, destSize); + chunks.add(chunk); } - writeChunks(pva, blocks); + writeChunks(pva, chunks); } // @@ -809,73 +809,72 @@ private RawShard getRawShard( } /** - * A request to read or write a (level-0) DataBlock at a given {@link #position}. + * A request to read or write a chunk (level-0 DataBlock) at a given {@link #position}. *

- * Write requests are constructed with {@link #position} and {@link #block}. + * Write requests are constructed with {@link #position} and {@link #chunk}. *

* Read requests are constructed with only a {@link #position}, and - * initially {@link #block block=null}. When the DataBlock is read, it will - * be put into {@link #block}. + * initially {@link #chunk chunk=null}. When the DataBlock is read, it will + * be put into {@link #chunk}. *

- * {@code DataBlockRequest} are used for reading/writing a list of blocks - * with {@link #readBlocks} and {@link #writeBlocks}. The {@link #index} - * field is the position in the list of positions/blocks to read/write. For + * {@code ChunkRequest} are used for reading/writing a list of chunks + * with {@link #readChunks} and {@link #writeChunks}. The {@link #index} + * field is the position in the list of positions/chunks to read/write. For * processing, requests are re-ordered such that all requests from the same * (sub-)shard are grouped together. The {@link #index} field is used to - * re-establish the order of results (only important for {@link - * #readBlocks}). + * re-establish the order of results (only important for {@link #readChunks}). * * @param * type of the data contained in the DataBlock */ - private static final class DataBlockRequest { + private static final class ChunkRequest { final NestedPosition position; final int index; - DataBlock block; + DataBlock chunk; // read request - DataBlockRequest(final NestedPosition position, final int index) { + ChunkRequest(final NestedPosition position, final int index) { this.position = position; this.index = index; - this.block = null; + this.chunk = null; } // write request - DataBlockRequest(final NestedPosition position, final DataBlock block) { + ChunkRequest(final NestedPosition position, final DataBlock chunk) { this.position = position; this.index = -1; - this.block = block; + this.chunk = chunk; } @Override public String toString() { - return "DataBlockRequest{position=" + position + ", index=" + index + '}'; + return "ChunkRequest{position=" + position + ", index=" + index + '}'; } } /** - * A list of {@code DataBlockRequest}, ordered by {@code NestedPosition}. + * A list of {@code ChunkRequest}, ordered by {@code NestedPosition}. * All requests lie in the same shard at the given {@code level}. *

- * {@code DataBlockRequests} should be constructed using {@link + * {@code ChunkRequests} should be constructed using {@link * #createReadRequests} (for reading) or {@link #createWriteRequests} (for * writing). *

- * When recursing into nested shard levels, {@code DataBlockRequests} should - * be {@link #split} to partition into sub-{@code DataBlockRequests} that + * When recursing into nested shard levels, {@code ChunkRequests} should + * be {@link #split} to partition into sub-{@code ChunkRequests} that * each cover one shard. * * @param * type of the data contained in the DataBlocks */ - private static final class DataBlockRequests implements Iterable> { + private static final class ChunkRequests implements Iterable> { private final NestedGrid grid; - private final List> requests; + private final List> requests; private final int level; - private DataBlockRequests(final List> requests, final int level, final NestedGrid grid) { + private ChunkRequests(final List> requests, final int level, final NestedGrid grid) { this.requests = requests; this.level = level; this.grid = grid; @@ -892,12 +891,12 @@ private DataBlockRequests(final List> requests, final int le * If {@code n} duplicates occur in {@code requests}, the resulting list * will have {@code 2*n} elements. */ - public List> removeDuplicates() { - List> duplicates = new ArrayList<>(); - DataBlockRequest previous = null; - final ListIterator> iter = requests.listIterator(); + public List> removeDuplicates() { + List> duplicates = new ArrayList<>(); + ChunkRequest previous = null; + final ListIterator> iter = requests.listIterator(); while (iter.hasNext()) { - final DataBlockRequest current = iter.next(); + final ChunkRequest current = iter.next(); if (previous != null) { if (previous.position.equals(current.position)) { iter.remove(); @@ -912,15 +911,15 @@ public List> removeDuplicates() { } @Override - public Iterator> iterator() { + public Iterator> iterator() { return requests.iterator(); } /** - * All blocks contained in this {@code DataBlockRequests} are in the + * All chunks contained in this {@code ChunkRequests} are in the * same shard at this nesting level. *

- * Use {@link #split()} to partition into {@code DataBlockRequests} with + * Use {@link #split()} to partition into {@code ChunkRequests} with * nesting level {@link #level()}{@code -1}. * * @return nesting level @@ -930,7 +929,7 @@ public int level() { } /** - * Position on the shard grid at the level of this DataBlockRequests + * Position on the shard grid at the level of this ChunkRequests * (of the one shard containing all the requested blocks). */ public long[] gridPosition() { @@ -938,7 +937,7 @@ public long[] gridPosition() { } /** - * Relative grid position at the level of this DataBlockRequests, + * Relative grid position at the level of this ChunkRequests, * that is, relative offset within containing the (level+1) element. */ public long[] relativeGridPosition() { @@ -954,9 +953,9 @@ private NestedPosition position() { /** * Split into sub-requests, grouping by same position at nesting level {@link #level()}{@code -1}. */ - public List> split() { + public List> split() { final int subLevel = level - 1; - final List> subRequests = new ArrayList<>(); + final List> subRequests = new ArrayList<>(); for (int i = 0; i < requests.size(); ) { final long[] ilpos = requests.get(i).position.absolute(subLevel); int j = i + 1; @@ -966,14 +965,14 @@ public List> split() { break; } } - subRequests.add(new DataBlockRequests<>(requests.subList(i, j), subLevel, grid)); + subRequests.add(new ChunkRequests<>(requests.subList(i, j), subLevel, grid)); i = j; } return subRequests; } /** - * Returns {@code true} if this {@code DataBlockRequests} completely + * Returns {@code true} if this {@code ChunkRequests} completely * fills its containing shard at nesting level {@link #level()}. * (This can be used to avoid reading a shard that will be completely * overwritten). @@ -983,7 +982,7 @@ public List> split() { */ public boolean coversShard() { final long[] gridMin = grid.absolutePosition(position().absolute(level), level, 0); - final long[] datasetSize = grid.getDatasetSizeInBlocks(); // in units of DataBlocks + final long[] datasetSize = grid.getDatasetSizeInChunks(); // in units of DataBlocks final int[] defaultShardSize = grid.relativeToBaseBlockSize(level); // in units of DataBlocks int numElements = 1; for (int d = 0; d < defaultShardSize.length; d++) { @@ -993,65 +992,65 @@ public boolean coversShard() { } /** - * Extract {@link DataBlockRequest#block DataBlock}s from the requests, - * in the order of {@link DataBlockRequest#index indices}. + * Extract {@link ChunkRequest#chunk chunk}s from the requests, + * in the order of {@link ChunkRequest#index indices}. *

- * (This is used in {@link #readBlocks} to collect DataBlocks in the + * (This is used in {@link #readChunks} to collect chunks in the * requested order.) */ - public List> blocks(final List> duplicates) { + public List> chunks(final List> duplicates) { final int size = requests.size() + duplicates.size() / 2; final DataBlock[] blocks = new DataBlock[size]; - requests.forEach(r -> blocks[r.index] = r.block); + requests.forEach(r -> blocks[r.index] = r.chunk); for (int i = 0; i < duplicates.size(); i += 2) { - final DataBlockRequest a = duplicates.get(i * 2); - final DataBlockRequest b = duplicates.get(i * 2 + 1); - blocks[a.index] = b.block; + final ChunkRequest a = duplicates.get(i * 2); + final ChunkRequest b = duplicates.get(i * 2 + 1); + blocks[a.index] = b.chunk; } return Arrays.asList(blocks); } } /** - * Construct {@code DataBlockRequests} from a list of level-0 grid positions + * Construct {@code ChunkRequests} from a list of level-0 grid positions * for reading. *

- * The nesting level ot the returned {@code DataBlockRequests} is {@code + * The nesting level ot the returned {@code ChunkRequests} is {@code * grid.numLevels()}, that is level of the highest-order shard + 1. This * implies that the requests are not guaranteed to be in the same shard (at - * any level. {@link DataBlockRequests#split() Splitting} the {@code - * DataBlockRequests} once will return a list of {@code DataBlockRequests} - * that each contain blocks from one highest-order shard. + * any level. {@link ChunkRequests#split() Splitting} the {@code + * ChunkRequests} once will return a list of {@code ChunkRequests} + * that each contain chunks from one highest-order shard. */ - private DataBlockRequests createReadRequests(final List gridPositions) { - final List> requests = new ArrayList<>(gridPositions.size()); + private ChunkRequests createReadRequests(final List gridPositions) { + final List> requests = new ArrayList<>(gridPositions.size()); for (int i = 0; i < gridPositions.size(); i++) { final NestedPosition pos = grid.nestedPosition(gridPositions.get(i)); - requests.add(new DataBlockRequest<>(pos, i)); + requests.add(new ChunkRequest<>(pos, i)); } requests.sort(Comparator.comparing(r -> r.position)); - return new DataBlockRequests<>(requests, grid.numLevels(), grid); + return new ChunkRequests<>(requests, grid.numLevels(), grid); } /** - * Construct {@code DataBlockRequests} from a list of level-0 DataBlocks for - * writing. + * Construct {@code ChunkRequests} from a list of chunks (level-0 + * DataBlocks) for writing. *

- * The nesting level ot the returned {@code DataBlockRequests} is {@code + * The nesting level ot the returned {@code ChunkRequests} is {@code * grid.numLevels()}, that is level of the highest-order shard + 1. This * implies that the requests are not guaranteed to be in the same shard (at - * any level. {@link DataBlockRequests#split() Splitting} the {@code - * DataBlockRequests} once will return a list of {@code DataBlockRequests} - * that each contain blocks from one highest-order shard. + * any level. {@link ChunkRequests#split() Splitting} the {@code + * ChunkRequests} once will return a list of {@code ChunkRequests} + * that each contain chunks from one highest-order shard. */ - private DataBlockRequests createWriteRequests(final List> dataBlocks) { - final List> requests = new ArrayList<>(dataBlocks.size()); + private ChunkRequests createWriteRequests(final List> dataBlocks) { + final List> requests = new ArrayList<>(dataBlocks.size()); for (final DataBlock dataBlock : dataBlocks) { final NestedPosition pos = grid.nestedPosition(dataBlock.getGridPosition()); - requests.add(new DataBlockRequest<>(pos, dataBlock)); + requests.add(new ChunkRequest<>(pos, dataBlock)); } requests.sort(Comparator.comparing(r -> r.position)); - return new DataBlockRequests<>(requests, grid.numLevels(), grid); + return new ChunkRequests<>(requests, grid.numLevels(), grid); } /** @@ -1059,7 +1058,7 @@ private DataBlockRequests createWriteRequests(final List> dataBl * array type and the number of elements in a block corresponds to the * {@link DataBlock#getSize()}. *

- * This is used by {@link #readShard} and {@link #writeShard} which + * This is used by {@link #readBlock} and {@link #writeBlock} which * internally need to allocate new DataBlocks to split or merge a shard. */ private interface DataBlockFactory { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java index 29106b09c..feaa91cce 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java @@ -258,7 +258,7 @@ public static class NestedGrid { /** * dimensions of the dataset in level-0 blocks. */ - private final long[] datasetSizeInBlocks; + private final long[] datasetSizeInChunks; /** * {@code blockSizes[l][d]} is the block size at level {@code l} in dimension {@code d}. @@ -322,10 +322,10 @@ public NestedGrid(int[][] blockSizes, long[] datasetSize) { } if (datasetSize == null) { - datasetSizeInBlocks = null; + datasetSizeInChunks = null; } else { - datasetSizeInBlocks = new long[numDimensions]; - Arrays.setAll(datasetSizeInBlocks, d -> (datasetSize[d] + blockSizes[0][d] - 1) / blockSizes[0][d]); + datasetSizeInChunks = new long[numDimensions]; + Arrays.setAll(datasetSizeInChunks, d -> (datasetSize[d] + blockSizes[0][d] - 1) / blockSizes[0][d]); } } @@ -602,12 +602,6 @@ public long[] relativePosition( return targetPos; } - public long[] relativePosition( - final long[] sourcePos, - final int targetLevel) { - return relativePosition(sourcePos, 0, targetLevel); - } - /** * Get size of a block at the given {@code level} relative to {@code * level-1} (that is, in units of {@code level-1} blocks). @@ -621,7 +615,7 @@ public int[] relativeBlockSize(final int level) { /** * Get size of a block at the given {@code level} relative to level {@code - * 0} (that is, in units of {@code level-0} blocks). + * 0} (that is, in units of chunks). *

* For example {@code relativeToBaseBlockSize(1)} returns the number of * chunks in a (non-nested) shard. @@ -643,15 +637,15 @@ public long[] getDatasetSize() { } /** - * Get the size of the dataset in units of DataBlocks. + * Get the size of the dataset in units of chunks. *

* This might return {@code null}, if this {@code NestedGrid} was not * constructed with dataset dimensions. * * @return size of the dataset in chunks */ - public long[] getDatasetSizeInBlocks() { - return datasetSizeInBlocks; + public long[] getDatasetSizeInChunks() { + return datasetSizeInChunks; } } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java index 8d1f4411d..18cdbaf10 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java @@ -78,7 +78,7 @@ public void testAbsolutePositionChunkSize() { } private static long relPosition1D(final NestedGrid grid, final int sourcePos, final int targetLevel) { - return grid.relativePosition(new long[] {sourcePos}, targetLevel)[0]; + return grid.relativePosition(new long[] {sourcePos}, 0, targetLevel)[0]; } @Test From 2430d0332755f8d48b2876292a50d3585eaa98ef Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 25 Mar 2026 22:01:39 +0100 Subject: [PATCH 21/41] variable renaming and javadoc fixes --- .../n5/shard/PositionValueAccess.java | 20 +++++------ .../saalfeldlab/n5/shard/RawShardCodec.java | 2 +- .../n5/shard/RawShardDataBlock.java | 2 +- .../janelia/saalfeldlab/n5/shard/Region.java | 35 ++++++++++--------- .../saalfeldlab/n5/shard/ShardCodecInfo.java | 8 ++--- 5 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java index 80035b958..a0e87b1ad 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java @@ -39,33 +39,33 @@ /** * Wrap a KeyValueAccess and a dataset URI to be able to get/set values (ReadData) by {@code long[]} key - * indicating the position of a block or shard. + * indicating the position of a top-level DataBlock (a top-level shard for sharded datasets, + * a chunk for non-sharded datasets). */ public interface PositionValueAccess { /** - * Gets the {@link VolatileReadData} for the DataBlock (or shard) at the - * given position in the block (or shard) grid. + * Gets the {@link VolatileReadData} for the DataBlock at the given position + * in the block grid. *

* If the requested key does not exist, either {@code null} is returned or a * lazy {@code VolatileReadData} that will throw {@code N5NoSuchKeyException} * when trying to materialize. * * @param key - * The position of the data block or shard - * @return ReadData for the given key or {@code null} if the key doesn't - * exist + * The position of the block + * @return ReadData for the given key or {@code null} if the key doesn't exist * @throws N5Exception.N5IOException * if an error occurs while reading */ VolatileReadData get(long[] key) throws N5Exception.N5IOException; /** - * Write the {@code data} for a DataBlock (or shard) to the given position - * in the block grid. + * Write the {@code data} for a DataBlock to the given position in the block + * grid. * * @param key - * The grid position of the DataBlock (or shard) to write + * The grid position of the DataBlock to write * @param data * The data to write * @@ -106,7 +106,7 @@ class KvaPositionValueAccess implements PositionValueAccess { } /** - * Constructs the absolute path for a data block (or shard) at a given grid + * Constructs the absolute path for a DataBlock at a given grid * position. * * @param gridPosition diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java index e23640333..21c7f63f1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java @@ -45,7 +45,7 @@ public class RawShardCodec implements BlockCodec { /** - * Number of elements (DataBlocks, nested shards) in each dimension per shard. + * Number of elements (chunks, nested shards) in each dimension per shard. */ private final int[] size; private final IndexLocation indexLocation; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java index f129c9791..b2124851f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java @@ -46,7 +46,7 @@ public class RawShardDataBlock implements DataBlock { } // TODO: should this be the number of elements in the Shard (number of - // sub-shards / datablock) along each dimension, or the number of + // sub-shards / chunks) along each dimension, or the number of // pixels alon each dimension? @Override public int[] getSize() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java index 38a267ba1..ecfaa9570 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java @@ -36,7 +36,7 @@ /** * Bounds, in pixel coordinates, of a region in a dataset. *

- * Provides methods to find which blocks and shards are contained in the + * Provides methods to find which chunks and shards are contained in the * region, iterate sub-NestedPositions, etc. */ public class Region { @@ -63,12 +63,12 @@ public class Region { private final long[] size; /** - * {@code NestedPosition} of the block containing the min pixel position. + * {@code NestedPosition} of the chunk containing the min pixel position. */ private final Nesting.NestedPosition minPos; /** - * {@code NestedPosition} of the block containing the max pixel position. + * {@code NestedPosition} of the chunk containing the max pixel position. */ private final Nesting.NestedPosition maxPos; @@ -79,37 +79,38 @@ public Region(final long[] min, final long[] size, final NestedGrid grid) { this.datasetDimensions = grid.getDatasetSize(); final int n = min.length; - final int[] blockSize = grid.getBlockSize(0); + final int[] chunkSize = grid.getBlockSize(0); - final long[] minBlock = new long[n]; - Arrays.setAll(minBlock, d -> min[d] / blockSize[d]); - minPos = grid.nestedPosition(minBlock); + final long[] minChunk = new long[n]; + Arrays.setAll(minChunk, d -> min[d] / chunkSize[d]); + minPos = grid.nestedPosition(minChunk); - final long[] maxBlock = new long[n]; - Arrays.setAll(maxBlock, d -> (min[d] + size[d] - 1) / blockSize[d]); - maxPos = grid.nestedPosition(maxBlock); + final long[] maxChunk = new long[n]; + Arrays.setAll(maxChunk, d -> (min[d] + size[d] - 1) / chunkSize[d]); + maxPos = grid.nestedPosition(maxChunk); } /** - * Get the {@code NestedPosition} of the minimum DataBlock touched by the region. + * Get the {@code NestedPosition} of the minimum chunk touched by the region. */ public Nesting.NestedPosition minPos() { return minPos; } /** - * Get the {@code NestedPosition} of the maximum DataBlock touched by the region. + * Get the {@code NestedPosition} of the maximum chunk touched by the region. */ public Nesting.NestedPosition maxPos() { return maxPos; } /** - * Check whether the Shard or DataBlock corresponding to the given - * position is fully contained inside the region. - * (The {@link Nesting.NestedPosition#level() level} of {@code position} is used - * to determine whether it refers to a DataBlock or a (potentially - * nested) Shard. + * Check whether the shard or chunk corresponding to the given position is + * fully contained inside the region. + *

+ * The {@link Nesting.NestedPosition#level() level} of {@code position} is + * used to determine whether it refers to a chunk or a (potentially nested) + * shard. * * @param position * the NestedPosition to check diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java index fb324881a..8fa019c20 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java @@ -37,8 +37,8 @@ public interface ShardCodecInfo extends BlockCodecInfo { /** - * Chunk size of each shard element (either nested shard or DataBlock) - * + * Size in pixels of each shard element (either nested shard or chunk) + * * @return the size of each shard element */ int[] getInnerBlockSize(); @@ -64,14 +64,14 @@ public interface ShardCodecInfo extends BlockCodecInfo { /** * BlockCodec for shard index - * + * * @return the BlockCodecInfo for this shard's index */ BlockCodecInfo getIndexBlockCodecInfo(); /** * Deterministic-size DataCodecs for index BlockCodec - * + * * @return the collection of DataCodecInfos for this shard's index */ DataCodecInfo[] getIndexDataCodecInfos(); From 1b0cb8293bd0cd126c601229e6febd412e60768a Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 26 Mar 2026 11:42:15 +0100 Subject: [PATCH 22/41] rename variables, comments, minor fixes --- .../saalfeldlab/n5/codec/DataCodecInfo.java | 2 +- .../n5/codec/DeterministicSizeCodecInfo.java | 9 +- .../n5/benchmarks/N5BlockWriteBenchmarks.java | 7 +- .../saalfeldlab/n5/shard/ShardTest.java | 197 +++++++++--------- .../n5/shard/TestPositionValueAccess.java | 6 + .../saalfeldlab/n5/shard/WriteRegionTest.java | 36 ++-- .../saalfeldlab/n5/shard/WriteShardTest.java | 102 ++++----- .../saalfeldlab/n5/shard/WriteShardTest2.java | 10 +- .../n5/shard/WriteShardTestTruncate.java | 10 +- 9 files changed, 193 insertions(+), 186 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java index a863683cf..6a37247f8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java @@ -32,7 +32,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; /** - * Used to Create {@code DataCodec}s, which transform one {@link ReadData} into another, + * Used to create {@code DataCodec}s, which transform one {@link ReadData} into another, * for example, applying compression. */ @NameConfig.Prefix("data-codec") diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java index 06618ef6f..60f92762b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java @@ -29,13 +29,14 @@ package org.janelia.saalfeldlab.n5.codec; /** - * A {@link CodecInfo} that can deterministically determine the size of encoded data from the size of the raw data and vice versa from the data length alone (i.e. encoding is data - * independent). + * A {@link CodecInfo} that can deterministically determine the size of encoded + * data from the size of the raw data and vice versa from the data length alone + * (i.e. encoding is data independent). */ public interface DeterministicSizeCodecInfo extends CodecInfo { - public abstract long encodedSize(long size); + long encodedSize(long size); - public abstract long decodedSize(long size); + long decodedSize(long size); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java index 9961906fe..ec4dad3a2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java @@ -30,7 +30,6 @@ import java.io.File; import java.io.IOException; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; @@ -138,7 +137,7 @@ public void setup() { blocks.add(blk); // write data into the read group - n5.writeChunk(readGroup, dsetAttrs, blk); + n5.writeBlock(readGroup, dsetAttrs, blk); } } catch (final IOException e) { @@ -151,7 +150,7 @@ public void setup() { public void writeBenchmark() throws IOException { blocks.forEach(blk -> { - n5.writeChunk(writeGroup, dsetAttrs, blk); + n5.writeBlock(writeGroup, dsetAttrs, blk); }); } @@ -161,7 +160,7 @@ public void readBenchmark(Blackhole hole) throws IOException { final long[] p = new long[numDimensions]; for (int i = 0; i < numBlocks; i++) { p[0] = i; - hole.consume(n5.readChunk(readGroup, dsetAttrs, p)); + hole.consume(n5.readBlock(readGroup, dsetAttrs, p)); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index f4032bd7a..7d30db739 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -40,6 +40,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -128,15 +129,15 @@ public void removeTempWriters() { tempN5Factory.removeTempWriters(); } - private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] blockSize) { + private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] chunkSize) { - return getTestAttributes(DataType.UINT8, dimensions, shardSize, blockSize); + return getTestAttributes(DataType.UINT8, dimensions, shardSize, chunkSize); } - private DatasetAttributes getTestAttributes(DataType dataType, long[] dimensions, int[] shardSize, int[] blockSize) { + private DatasetAttributes getTestAttributes(DataType dataType, long[] dimensions, int[] shardSize, int[] chunkSize) { DefaultShardCodecInfo blockCodec = new DefaultShardCodecInfo( - blockSize, + chunkSize, new N5BlockCodecInfo(), new DataCodecInfo[]{new RawCompression()}, new RawBlockCodecInfo(), @@ -156,7 +157,7 @@ protected DatasetAttributes getTestAttributes() { } @Test - public void writeReadBlocksTest() { + public void writeReadChunksTest() { final N5Writer writer = tempN5Factory.createTempN5Writer(); @@ -170,8 +171,8 @@ public void writeReadBlocksTest() { writer.remove(dataset); writer.createDataset(dataset, datasetAttributes); - final int[] blockSize = datasetAttributes.getBlockSize(); - final int numElements = blockSize[0] * blockSize[1]; + final int[] chunkSize = datasetAttributes.getBlockSize(); + final int numElements = chunkSize[0] * chunkSize[1]; final byte[] data = new byte[numElements]; for (int i = 0; i < data.length; i++) { @@ -182,17 +183,17 @@ public void writeReadBlocksTest() { dataset, datasetAttributes, /* shard (0, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), - new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + new ByteArrayDataBlock(chunkSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{0, 1}, data), + new ByteArrayDataBlock(chunkSize, new long[]{1, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{1, 1}, data), /* shard (1, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{4, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{5, 0}, data), /* shard (2, 2) */ - new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) + new ByteArrayDataBlock(chunkSize, new long[]{11, 11}, data) ); final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); @@ -213,10 +214,10 @@ public void writeReadBlocksTest() { ensureKeysExist(kva, writer.getURI(), dataset, datasetAttributes, keys); ensureKeysDoNotExist(kva, writer.getURI(), dataset, datasetAttributes, someUnusedKeys); - final long[][] blockIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; - for (long[] blockIndex : blockIndices) { - final DataBlock block = writer.readChunk(dataset, datasetAttributes, blockIndex); - Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + final long[][] chunkIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; + for (long[] chunkIndex : chunkIndices) { + final DataBlock chunk = writer.readChunk(dataset, datasetAttributes, chunkIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])chunk.getData()); } final byte[] data2 = new byte[numElements]; @@ -228,15 +229,15 @@ public void writeReadBlocksTest() { dataset, datasetAttributes, /* shard (0, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data2), - new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data2), + new ByteArrayDataBlock(chunkSize, new long[]{0, 0}, data2), + new ByteArrayDataBlock(chunkSize, new long[]{1, 1}, data2), /* shard (0, 1) */ - new ByteArrayDataBlock(blockSize, new long[]{0, 4}, data2), - new ByteArrayDataBlock(blockSize, new long[]{0, 5}, data2), + new ByteArrayDataBlock(chunkSize, new long[]{0, 4}, data2), + new ByteArrayDataBlock(chunkSize, new long[]{0, 5}, data2), /* shard (2, 2) */ - new ByteArrayDataBlock(blockSize, new long[]{10, 10}, data2)); + new ByteArrayDataBlock(chunkSize, new long[]{10, 10}, data2)); long[][] keys2 = new long[][]{ {0, 0}, @@ -254,21 +255,21 @@ public void writeReadBlocksTest() { ensureKeysExist(kva, writer.getURI(), dataset, datasetAttributes, keys2); ensureKeysDoNotExist(kva, writer.getURI(), dataset, datasetAttributes, someUnusedKeys2); - final long[][] oldBlockIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; - for (long[] blockIndex : oldBlockIndices) { - final DataBlock block = writer.readChunk(dataset, datasetAttributes, blockIndex); - Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + final long[][] oldChunkIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; + for (long[] chunkIndex : oldChunkIndices) { + final DataBlock chunk = writer.readChunk(dataset, datasetAttributes, chunkIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])chunk.getData()); } - final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; - final List newBlockIndexList = Arrays.asList(newBlockIndices); - final List> readBlocks = writer.readChunks(dataset, datasetAttributes, newBlockIndexList); - for (int i = 0; i < newBlockIndices.length; i++) { - final long[] blockIndex = newBlockIndices[i]; - final DataBlock block = writer.readChunk(dataset, datasetAttributes, blockIndex); - Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])block.getData()); - final DataBlock blockFromReadBlocks = readBlocks.get(i); - Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])blockFromReadBlocks.getData()); + final long[][] newChunkIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; + final List newChunkIndexList = Arrays.asList(newChunkIndices); + final List> readChunks = writer.readChunks(dataset, datasetAttributes, newChunkIndexList); + for (int i = 0; i < newChunkIndices.length; i++) { + final long[] chunkIndex = newChunkIndices[i]; + final DataBlock chunk = writer.readChunk(dataset, datasetAttributes, chunkIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])chunk.getData()); + final DataBlock chunkFromReadChunks = readChunks.get(i); + Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])chunkFromReadChunks.getData()); } } @@ -296,7 +297,7 @@ public void writeShardDataSizeTest() { // note: this test depends on the use of raw compression final N5Writer writer = tempN5Factory.createTempN5Writer(); - int numBlocksPerShard = 16; + int numChunksPerShard = 16; final int n5HeaderSizeBytes = 12; // 2 + 2 + 4*2 final DatasetAttributes attrs = getTestAttributes( new long[]{24, 24}, @@ -311,8 +312,8 @@ public void writeShardDataSizeTest() { final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); - final int[] blockSize = datasetAttributes.getBlockSize(); - final int numElements = blockSize[0] * blockSize[1]; + final int[] chunkSize = datasetAttributes.getChunkSize(); + final int numElements = chunkSize[0] * chunkSize[1]; final byte[] data = new byte[numElements]; for (int i = 0; i < data.length; i++) { @@ -320,42 +321,42 @@ public void writeShardDataSizeTest() { } /* - * No blocks or shards exist. - * Calling readBlocks should return a list that is the same length as the requested grid positions, + * No chunks or shards exist. + * Calling readChunks should return a list that is the same length as the requested grid positions, * and every entry should be null. */ - final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; - final List> readBlocks = writer.readChunks(dataset, datasetAttributes, Arrays.asList(newBlockIndices)); - assertEquals(newBlockIndices.length, readBlocks.size()); - assertTrue("readBlocks for empty shard: all blocks null", readBlocks.stream().allMatch(blk -> blk == null)); + final long[][] newChunkIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; + final List> readChunks = writer.readChunks(dataset, datasetAttributes, Arrays.asList(newChunkIndices)); + assertEquals(newChunkIndices.length, readChunks.size()); + assertTrue("readChunks for empty shard: all chunks null", readChunks.stream().allMatch(Objects::isNull)); /* - * Now write blocks + * Now write chunks */ writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), - new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + new ByteArrayDataBlock(chunkSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{0, 1}, data), + new ByteArrayDataBlock(chunkSize, new long[]{1, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{1, 1}, data), /* shard (1, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{4, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{5, 0}, data), /* shard (2, 2) */ - new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) + new ByteArrayDataBlock(chunkSize, new long[]{11, 11}, data) ); - final int indexSizeBytes = numBlocksPerShard * 16; // 8 bytes per long * - final int blockDataSizeBytes = numElements + n5HeaderSizeBytes; + final int indexSizeBytes = numChunksPerShard * 16; // 8 bytes per long * + final int chunkDataSizeBytes = numElements + n5HeaderSizeBytes; - // shard 0,0 has 4 blocks so should be this size: - long shard00SizeBytes = indexSizeBytes + 4 * blockDataSizeBytes; - long shard10SizeBytes = indexSizeBytes + 2 * blockDataSizeBytes; - long shard22SizeBytes = indexSizeBytes + 1 * blockDataSizeBytes; + // shard 0,0 has 4 chunks so should be this size: + long shard00SizeBytes = indexSizeBytes + 4 * chunkDataSizeBytes; + long shard10SizeBytes = indexSizeBytes + 2 * chunkDataSizeBytes; + long shard22SizeBytes = indexSizeBytes + 1 * chunkDataSizeBytes; final String[][] keys = new String[][]{ {dataset, "0", "0"}, @@ -378,7 +379,7 @@ public void writeShardDataSizeTest() { } @Test - public void writeReadBlockTest() { + public void writeReadChunkTest() { final GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)tempN5Factory.createTempN5Writer(); final DatasetAttributes datasetAttributes = getTestAttributes(); @@ -391,29 +392,29 @@ public void writeReadBlockTest() { final DataType dataType = datasetAttributes.getDataType(); final int numElements = 2 * 2; - final HashMap writtenBlocks = new HashMap<>(); + final HashMap writtenChunks = new HashMap<>(); for (int idx1 = 1; idx1 >= 0; idx1--) { for (int idx2 = 1; idx2 >= 0; idx2--) { final long[] gridPosition = {idx1, idx2}; - final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(chunkSize, gridPosition, numElements); - byte[] data = dataBlock.getData(); + final DataBlock chunk = (DataBlock)dataType.createDataBlock(chunkSize, gridPosition, numElements); + byte[] data = chunk.getData(); for (int i = 0; i < data.length; i++) { data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); } - writer.writeChunk(dataset, datasetAttributes, dataBlock); + writer.writeChunk(dataset, datasetAttributes, chunk); - final DataBlock block = writer.readChunk(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); - Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + final DataBlock readChunk = writer.readChunk(dataset, datasetAttributes, chunk.getGridPosition().clone()); + Assert.assertArrayEquals("Read from shard doesn't match", data, readChunk.getData()); - for (Map.Entry entry : writtenBlocks.entrySet()) { + for (Map.Entry entry : writtenChunks.entrySet()) { final long[] otherGridPosition = entry.getKey(); final byte[] otherData = entry.getValue(); - final DataBlock otherBlock = writer.readChunk(dataset, datasetAttributes, otherGridPosition); - Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); + final DataBlock otherChunk = writer.readChunk(dataset, datasetAttributes, otherGridPosition); + Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, otherChunk.getData()); } - writtenBlocks.put(gridPosition, data); + writtenChunks.put(gridPosition, data); } } } @@ -441,7 +442,7 @@ public void writeReadShardTest() { /** * The 4x4 shard at (0,0) - * and the 2x2 blocks it contains + * and the 2x2 chunks it contains * * * 0 1 | 2 3 @@ -459,7 +460,7 @@ public void writeReadShardTest() { n5.deleteChunk(dataset, attrs, new long[]{1, 1}); /** - * After deleting block (1,1) + * After deleting chunk (1,1) * * 0 1 | 2 3 * 4 5 | 6 7 @@ -471,7 +472,7 @@ public void writeReadShardTest() { assertArrayEquals(new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 12, 13, 0, 0}, partlyEmptyShard.getData()); - // Delete the rest of the blocks + // Delete the rest of the chunks n5.deleteChunks(dataset, attrs, Stream.of( new long[] {0,0}, new long[] {1,0}, new long[] {0,1}).collect(Collectors.toList())); @@ -481,20 +482,20 @@ public void writeReadShardTest() { // write the shard again n5.writeBlock(dataset, attrs, shard); - // delete the block - // ensure it returns true because the block exists + // delete the chard + // ensure it returns true because the shard exists assertTrue(n5.deleteBlock(dataset, attrs, shard.getGridPosition())); - // ensure it returns false when the block does not exist + // ensure it returns false when the shard does not exist assertFalse(n5.deleteBlock(dataset, attrs, shard.getGridPosition())); - // readBlock must return null for the deleted block + // readBlock must return null for the deleted shard assertNull(n5.readBlock(dataset, attrs, shard.getGridPosition())); } } /** - * Checks how many read calls to the backend are performed for a particular readBlocks + * Checks how many read calls to the backend are performed for a particular readChunks * call. At this time (Nov 4 2025), one read for the index, and one read per block are performed. */ public void numReadsTest() { @@ -511,8 +512,8 @@ public void numReadsTest() { writer.remove(dataset); writer.createDataset(dataset, datasetAttributes); - final int[] blockSize = datasetAttributes.getBlockSize(); - final int numElements = blockSize[0] * blockSize[1]; + final int[] chunkSize = datasetAttributes.getChunkSize(); + final int numElements = chunkSize[0] * chunkSize[1]; final byte[] data = new byte[numElements]; for (int i = 0; i < data.length; i++) { @@ -523,17 +524,17 @@ public void numReadsTest() { dataset, datasetAttributes, /* shard (0, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), - new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + new ByteArrayDataBlock(chunkSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{0, 1}, data), + new ByteArrayDataBlock(chunkSize, new long[]{1, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{1, 1}, data), /* shard (1, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{4, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{5, 0}, data), /* shard (2, 2) */ - new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) + new ByteArrayDataBlock(chunkSize, new long[]{11, 11}, data) ); writer.resetNumMaterializeCalls(); @@ -567,8 +568,8 @@ public void shardExistsTest() { writer.remove(dataset); DatasetAttributes attrs = writer.createDataset(dataset, datasetAttributes); - final int[] blockSize = datasetAttributes.getBlockSize(); - final int numElements = blockSize[0] * blockSize[1]; + final int[] chunkSize = datasetAttributes.getChunkSize(); + final int numElements = chunkSize[0] * chunkSize[1]; final byte[] data = new byte[numElements]; for (int i = 0; i < data.length; i++) { @@ -579,9 +580,9 @@ public void shardExistsTest() { writer.writeChunks( dataset, attrs, - new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), /* shard (0, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), /* shard (1, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) /* shard (2, 2) */ + new ByteArrayDataBlock(chunkSize, new long[]{0, 0}, data), /* shard (0, 0) */ + new ByteArrayDataBlock(chunkSize, new long[]{4, 0}, data), /* shard (1, 0) */ + new ByteArrayDataBlock(chunkSize, new long[]{11, 11}, data) /* shard (2, 2) */ ); TrackingN5Writer trackingWriter = ((TrackingN5Writer) writer); @@ -627,8 +628,8 @@ public void testPartialReadAggregationBehavior() { writer.remove(dataset); DatasetAttributes attrs = writer.createDataset(dataset, datasetAttributes); - final int[] blockSize = attrs.getBlockSize(); - final int numElements = blockSize[0] * blockSize[1]; + final int[] chunkSize = attrs.getChunkSize(); + final int numElements = chunkSize[0] * chunkSize[1]; final byte[] data = new byte[numElements]; for (int i = 0; i < data.length; i++) { @@ -646,10 +647,10 @@ public void testPartialReadAggregationBehavior() { writer.writeChunks( dataset, datasetAttributes, - new ByteArrayDataBlock(blockSize, ptList.get(0), data), - new ByteArrayDataBlock(blockSize, ptList.get(1), data), - new ByteArrayDataBlock(blockSize, ptList.get(2), data), - new ByteArrayDataBlock(blockSize, ptList.get(3), data) + new ByteArrayDataBlock(chunkSize, ptList.get(0), data), + new ByteArrayDataBlock(chunkSize, ptList.get(1), data), + new ByteArrayDataBlock(chunkSize, ptList.get(2), data), + new ByteArrayDataBlock(chunkSize, ptList.get(3), data) ); writer.resetNumMaterializeCalls(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java index 957b8e5dd..7ac8f7db4 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java @@ -41,6 +41,12 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; +/** + * Implementation of {@link PositionValueAccess} for tests. + *

+ * Instead of writing to a {@code KeyValueAccess} backend, here, data is just + * stored in a {@code Map} as {@code byte[]} arrays. + */ public class TestPositionValueAccess implements PositionValueAccess { private final Map map = new HashMap<>(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java index 89e9a4990..e9e92b8eb 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java @@ -52,20 +52,20 @@ public class WriteRegionTest { public static void main(String[] args) { -// int[] datablockSize = {3, 3, 3}; +// int[] chunkSize = {3, 3, 3}; // int[] level1ShardSize = {6, 6, 6}; // int[] level2ShardSize = {24, 24, 24}; - int[] datablockSize = {3}; + int[] chunkSize = {3}; int[] level1ShardSize = {6}; int[] level2ShardSize = {24}; final long[] datasetDimensions = {96}; - // DataBlocks are 3x3x3 + // Chunks are 3x3x3 // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -121,46 +121,46 @@ public static void main(String[] args) { false); // verify that the written blocks can be read back with the correct values -// checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), true, 1); -// checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); -// checkBlock(datasetAccess.readBlock(store, new long[] {0, 1, 0}), true, 3); -// checkBlock(datasetAccess.readBlock(store, new long[] {1, 1, 0}), true, 4); -// checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5); -// checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6); +// checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), true, 1); +// checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); +// checkBlock(datasetAccess.readChunk(store, new long[] {0, 1, 0}), true, 3); +// checkBlock(datasetAccess.readChunk(store, new long[] {1, 1, 0}), true, 4); +// checkBlock(datasetAccess.readChunk(store, new long[] {3, 2, 1}), true, 5); +// checkBlock(datasetAccess.readChunk(store, new long[] {8, 4, 1}), true, 6); // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) // datasetAccess.deleteBlock(store, new long[] {0, 0, 0}); -// checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); -// checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); +// checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), false, 1); +// checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); // if a shard becomes empty the corresponding key should be deleted // if ( store.get(new long[] {1, 0, 0}) == null ) { // throw new IllegalStateException("expected non-null readData"); // } -// datasetAccess.deleteBlock(store, new long[] {8, 4, 1}); +// datasetAccess.deleteChunk(store, new long[] {8, 4, 1}); // if ( store.get(new long[] {1, 0, 0}) != null ) { // throw new IllegalStateException("expected null readData"); // } // deleting a non-existent block should not fail -// datasetAccess.deleteBlock(store, new long[] {0, 0, 8}); +// datasetAccess.deleteChunk(store, new long[] {0, 0, 8}); System.out.println("all good"); } - + @Test public void testWriteRegionSharded() { - int[] datablockSize = {3}; + int[] chunkSize = {3}; int[] level2ShardSize = {24}; final long[] datasetDimensions = {96}; - // DataBlocks are 3x3x3 + // Chunks are 3x3x3 // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java index 02737334f..904eae533 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java @@ -58,15 +58,15 @@ public class WriteShardTest { public static void main(String[] args) { - final int[] datablockSize = {3}; + final int[] chunkSize = {3}; final int[] level1ShardSize = {6}; final long[] datasetDimensions = {36}; - // DataBlocks are 3 + // Chunks are 3 // Level 1 shards are 6 final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -119,19 +119,19 @@ public static void main(String[] args) { System.out.println("all good"); } - + @Test public void testShardDatasetAccess() { - final int[] datablockSize = {3}; + final int[] chunkSize = {3}; final int[] level1ShardSize = {6}; final long[] datasetDimensions = {36}; - // DataBlocks are 3 + // Chunks are 3 // Level 1 shards are 6 final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -163,14 +163,14 @@ public void testShardDatasetAccess() { @Test public void testWriteNullBlockRemovesShard() throws Exception { - final int[] datablockSize = {3}; + final int[] chunkSize = {3}; final int[] level1ShardSize = {6}; final long[] datasetDimensions = {36}; // Level 1 shards have size 6 (each containing two datablocks of size 3) final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[]{new RawCompression()}, new RawBlockCodecInfo(), @@ -194,58 +194,58 @@ public void testWriteNullBlockRemovesShard() throws Exception { assertFalse("Shard should not exist at the start of the test", store.exists(shardKey)); - // Write a single block at grid position [3] - // This block is in shard [1] - // ( shard 0 contains blocks 0-1, - // shard 1 contains blocks 2-3 ) - final long[] blockGridPosition = {3}; - final DataBlock block = createDataBlock(datablockSize, blockGridPosition, 100); + // Write a single chunk at grid position [3] + // This chunk is in shard [1] + // ( shard 0 contains chunks 0-1, + // shard 1 contains chunks 2-3 ) + final long[] chunkGridPosition = {3}; + final DataBlock chunk = createDataBlock(chunkSize, chunkGridPosition, 100); - datasetAccess.writeChunk(store, block); + datasetAccess.writeChunk(store, chunk); // Verify the shard exists - assertTrue("Shard should exist after writing block", store.exists(shardKey)); + assertTrue("Shard should exist after writing chunk", store.exists(shardKey)); - // Write a null block at the same location using writeRegion - // This should remove the block and delete the now-empty shard - final long[] regionMin = {9}; // pixel position of block [3] + // Write a null chunk at the same location using writeRegion + // This should remove the chunk and delete the now-empty shard + final long[] regionMin = {9}; // pixel position of chunk [3] final long[] regionSize = {3}; // size of one block datasetAccess.writeRegion( store, regionMin, regionSize, - (gridPosition, existingBlock) -> null, // block supplier returns - // null to indicate - // removal + (gridPosition, + // block supplier returns null to indicate removal + existingBlock) -> null, false); // Verify the shard has been removed - assertFalse("Shard should be removed after writing null block", store.exists(shardKey)); - + assertFalse("Shard should be removed after writing null chunk", store.exists(shardKey)); + /** - * THREE BLOCKS, TWO SHARDS + * THREE CHUNKS, TWO SHARDS */ - // Write two blocks into the same shard, and one block into a second shard - // Shard [1] contains blocks [2] and [3] - // Shard [2] contains block [4] + // Write two chunks into the same shard, and one chunk into a second shard + // Shard [1] contains chunks [2] and [3] + // Shard [2] contains chunk [4] final long[] shardKey1 = {1}; final long[] shardKey2 = {2}; - final DataBlock block1 = createDataBlock(datablockSize, new long[]{2}, 100); - final DataBlock block2 = createDataBlock(datablockSize, new long[]{3}, 200); - final DataBlock block3 = createDataBlock(datablockSize, new long[]{4}, 300); + final DataBlock chunk1 = createDataBlock(chunkSize, new long[]{2}, 100); + final DataBlock chunk2 = createDataBlock(chunkSize, new long[]{3}, 200); + final DataBlock chunk3 = createDataBlock(chunkSize, new long[]{4}, 300); assertFalse("Shard should not exist at the start of the test", store.exists(shardKey1)); assertFalse("Shard should not exist at the start of the test", store.exists(shardKey2)); // write blocks - datasetAccess.writeChunks(store, Streams.of(block1, block2, block3).collect(Collectors.toList())); + datasetAccess.writeChunks(store, Streams.of(chunk1, chunk2, chunk3).collect(Collectors.toList())); // Verify the shard exists - assertTrue("Shard should exist after writing blocks", store.exists(shardKey1)); - assertTrue("Shard should exist after writing blocks", store.exists(shardKey2)); + assertTrue("Shard should exist after writing chunks", store.exists(shardKey1)); + assertTrue("Shard should exist after writing chunks", store.exists(shardKey2)); // Write a null block at block [3]'s location datasetAccess.writeRegion( @@ -255,26 +255,26 @@ public void testWriteNullBlockRemovesShard() throws Exception { (gridPosition, existingBlock) -> null, false); - // Verify the shard still exists (because block [2] is still there) - assertTrue("Shard should still exist because it contains another block", store.exists(shardKey1)); + // Verify the shard still exists (because chunk [2] is still there) + assertTrue("Shard should still exist because it contains another chunk", store.exists(shardKey1)); assertTrue("Shard should still exist because was not affected", store.exists(shardKey2)); - // Verify we can still read block [2] - final DataBlock readBlock = datasetAccess.readChunk(store, new long[]{2}); - assertTrue("Block [2] should still be readable", readBlock != null); - assertTrue("Block [2] data should match", Arrays.equals(block1.getData(), readBlock.getData())); + // Verify we can still read chunk [2] + final DataBlock readChunk = datasetAccess.readChunk(store, new long[]{2}); + assertTrue("Block [2] should still be readable", readChunk != null); + assertTrue("Block [2] data should match", Arrays.equals(chunk1.getData(), readChunk.getData())); - // Verify block [3] is gone - final DataBlock readBlock2 = datasetAccess.readChunk(store, new long[]{3}); - assertNull("Block [3] should be null after removal", readBlock2); + // Verify chunk [3] is gone + final DataBlock readChunk2 = datasetAccess.readChunk(store, new long[]{3}); + assertNull("Block [3] should be null after removal", readChunk2); - // Verify block [4] exists - final DataBlock readBlock3 = datasetAccess.readChunk(store, new long[]{4}); - assertTrue("Block [4] should still be readable", readBlock3 != null); - assertTrue("Block [4] data should match", Arrays.equals(block3.getData(), readBlock3.getData())); + // Verify chunk [4] exists + final DataBlock readChunk3 = datasetAccess.readChunk(store, new long[]{4}); + assertTrue("Block [4] should still be readable", readChunk3 != null); + assertTrue("Block [4] data should match", Arrays.equals(chunk3.getData(), readChunk3.getData())); - // Write a null block at block [2]'s location - final long[] regionMin2 = {6}; // pixel position of block [3] + // Write a null chunk at chunk [2]'s location + final long[] regionMin2 = {6}; // pixel position of chunk [3] final long[] regionSize2 = {3}; datasetAccess.writeRegion( @@ -284,7 +284,7 @@ public void testWriteNullBlockRemovesShard() throws Exception { (gridPosition, existingBlock) -> null, false); - assertFalse("Shard should not exist after deleting all blocks", store.exists(shardKey1)); + assertFalse("Shard should not exist after deleting all chunks", store.exists(shardKey1)); assertTrue("Shard should still exist because was not affected", store.exists(shardKey2)); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java index 452bfce23..92c16ae98 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java @@ -50,15 +50,15 @@ public class WriteShardTest2 { public static void main(String[] args) { - final int[] datablockSize = {3,3}; + final int[] chunkSize = {3,3}; final int[] level1ShardSize = {6,6}; final long[] datasetDimensions = {36, 18}; - // DataBlocks are 3 - // Level 1 shards are 6 + // Chunks are 3x3 + // Level 1 shards are 6x6 final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -92,7 +92,7 @@ public static void main(String[] args) { System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); datasetAccess.writeBlock(store, shard, 1); - // we should get these DataBlock values + // we should get these Chunk values // 1, 2, 3, | 4, 5, 6, // 7, 8, 9, | 10, 11, 12, diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java index 73f239d50..3ad759371 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java @@ -50,15 +50,15 @@ public class WriteShardTestTruncate { public static void main(String[] args) { - final int[] datablockSize = {3,3}; + final int[] chunkSize = {3,3}; final int[] level1ShardSize = {6,6}; final long[] datasetDimensions = {34, 17}; // 6 x 3 shards, border (2, 1) - // DataBlocks are 3 - // Level 1 shards are 6 + // Chunks are 3x3 + // Level 1 shards are 6x6 final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -92,7 +92,7 @@ public static void main(String[] args) { System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); datasetAccess.writeBlock(store, shard, 1); - // we should get these DataBlock values + // we should get these chunk values // 1, 2, 3, | 4, 5, 6, // 7, 8, 9, | 10, 11, 12, From a0bca79dbc9d2e783d06733b8c5dc6cef7c4d68e Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 26 Mar 2026 15:08:34 +0100 Subject: [PATCH 23/41] Make getDatasetAccess() protected and createDatasetAccess() private createDatasetAccess() can be private because we don't override it anywhere. (Maybe we did previously in n5-zarr?) getDatasetAccess() has to be protected for tests only. It is better for tests to use the canonical DatasetAccess instead of creating a separate one, as they did before using createDatasetAccess() --- .../saalfeldlab/n5/DatasetAttributes.java | 8 ++++---- .../saalfeldlab/n5/codec/BlockCodecTests.java | 18 +++++++++++++++--- .../n5/shard/DatasetAccessTest.java | 10 ++++------ .../saalfeldlab/n5/shard/WriteRegionTest.java | 12 +++++------- .../saalfeldlab/n5/shard/WriteShardTest.java | 13 ++++++------- .../saalfeldlab/n5/shard/WriteShardTest2.java | 9 ++++----- .../n5/shard/WriteShardTestTruncate.java | 9 ++++----- 7 files changed, 42 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 71a62298c..cbc602763 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -189,7 +189,7 @@ public DatasetAttributes( this(dimensions, blockSize, dataType, new DataCodecInfo[0]); } - protected DatasetAccess createDatasetAccess() { + private DatasetAccess createDatasetAccess() { final int m = nestingDepth(blockCodecInfo); @@ -210,7 +210,7 @@ protected DatasetAccess createDatasetAccess() { BlockCodecInfo currentBlockCodecInfo = blockCodecInfo; DataCodecInfo[] currentDataCodecInfos = dataCodecInfos; - + DatasetCodecInfo[] datasetCodecInfos = this.datasetCodecInfos; final NestedGrid grid = new NestedGrid(blockSizes, dimensions); @@ -323,9 +323,9 @@ public DataType getDataType() { * * @return the {@code DatasetAccess} for this dataset */ - DatasetAccess getDatasetAccess() { + protected DatasetAccess getDatasetAccess() { - return (DatasetAccess)access; + return (DatasetAccess) access; } /** diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java index c0d40b10f..f8b8555dc 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java @@ -49,7 +49,6 @@ import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; import org.janelia.saalfeldlab.n5.shard.DatasetAccess; -import org.janelia.saalfeldlab.n5.shard.DatasetAccessTest; import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; import org.janelia.saalfeldlab.n5.shard.TestPositionValueAccess; import org.junit.Test; @@ -146,7 +145,7 @@ public void testEmptyBlock() throws Exception { final int[] blockSize = {0, 0}; final long[] gridPosition = {0, 0}; final N5BlockCodecInfo blockCodecInfo = new N5BlockCodecInfo(); - final DatasetAccessTest.TestDatasetAttributes attributes = new DatasetAccessTest.TestDatasetAttributes( + final TestDatasetAttributes attributes = new TestDatasetAttributes( new long[]{64, 64}, new int[]{8, 8}, DataType.UINT8, @@ -154,7 +153,7 @@ public void testEmptyBlock() throws Exception { new RawCompression()); final PositionValueAccess store = new TestPositionValueAccess(); - DatasetAccess access = attributes.datasetAccess(); + DatasetAccess access = attributes.getDatasetAccess(); // Test encode/decode final ByteArrayDataBlock emptyBlock = new ByteArrayDataBlock(blockSize, gridPosition, new byte[0]); @@ -315,5 +314,18 @@ private static void assertDataEquals(DataBlock expected, DataBlock actual) } } + public static class TestDatasetAttributes extends DatasetAttributes { + + public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType dataType, BlockCodecInfo blockCodecInfo, + DataCodecInfo... dataCodecInfos) { + + super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); + } + + @Override // to make this accessible for the test + protected DatasetAccess getDatasetAccess() { + return super.getDatasetAccess(); + } + } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java index 9d54874c0..29c587c9e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java @@ -93,7 +93,7 @@ public void setup() { new RawCompression() ); - datasetAccess = attributes.datasetAccess(); + datasetAccess = attributes.getDatasetAccess(); } @@ -243,12 +243,10 @@ public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType d super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); } - public DatasetAccess datasetAccess() { - - // to make this accessible for the test - return createDatasetAccess(); + @Override // to make this accessible for the test + protected DatasetAccess getDatasetAccess() { + return super.getDatasetAccess(); } - } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java index e9e92b8eb..00b356fae 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java @@ -88,7 +88,7 @@ public static void main(String[] args) { c2, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); // --------------------------------------------------------------- @@ -175,7 +175,7 @@ public void testWriteRegionSharded() { c1, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); final int[] dataBlockSize = c1.getInnerBlockSize(); @@ -227,12 +227,10 @@ public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType d super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); } - public DatasetAccess datasetAccess() { - - // to make this accessible for the test - return createDatasetAccess(); + @Override // to make this accessible for the test + protected DatasetAccess getDatasetAccess() { + return super.getDatasetAccess(); } - } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java index 904eae533..d03de5a4a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java @@ -81,7 +81,7 @@ public static void main(String[] args) { c1, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); // 0 1 2 3 4 5 6 @@ -146,7 +146,7 @@ public void testShardDatasetAccess() { c1, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); // 0 1 2 3 4 5 6 @@ -184,7 +184,7 @@ public void testWriteNullBlockRemovesShard() throws Exception { c1, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); final long[] shardKey = {1}; @@ -302,10 +302,9 @@ public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType d super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); } - public DatasetAccess datasetAccess() { - - // to make this accessible for the test - return createDatasetAccess(); + @Override // to make this accessible for the test + protected DatasetAccess getDatasetAccess() { + return super.getDatasetAccess(); } } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java index 92c16ae98..be1d973dd 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java @@ -73,7 +73,7 @@ public static void main(String[] args) { c1, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); // 0 1 2 3 4 5 6 @@ -129,10 +129,9 @@ public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType d super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); } - public DatasetAccess datasetAccess() { - - // to make this accessible for the test - return createDatasetAccess(); + @Override // to make this accessible for the test + protected DatasetAccess getDatasetAccess() { + return super.getDatasetAccess(); } } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java index 3ad759371..d3e537a63 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java @@ -73,7 +73,7 @@ public static void main(String[] args) { c1, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); // 0 1 2 3 4 5 6 @@ -137,10 +137,9 @@ public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType d super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); } - public DatasetAccess datasetAccess() { - - // to make this accessible for the test - return createDatasetAccess(); + @Override // to make this accessible for the test + protected DatasetAccess getDatasetAccess() { + return super.getDatasetAccess(); } } } From ede0a371654aecd618e6c922986b643d90d75817 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 26 Mar 2026 17:26:30 +0100 Subject: [PATCH 24/41] more renaming in comments --- .../n5/shard/DefaultDatasetAccess.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 72f650686..630c30033 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -131,7 +131,7 @@ public List> readChunks(final PositionValueAccess pva, final List> * Bulk Write operation on a shard. * * @param existingReadData encoded existing shard data (to decode and partially override) - * @param requests for blocks within the shard to be written + * @param requests for chunks within the shard to be written */ private ReadData writeChunksRecursive( final ReadData existingReadData, // may be null @@ -302,7 +302,7 @@ public void writeRegion( final ReadData modifiedData; try (final VolatileReadData existingData = nestedWriteFully ? null : pva.get(key)) { modifiedData = writeRegionRecursive(existingData, region, chunkSupplier, pos); - // Here, we are about to write the shard data, but with the new block modified. + // Here, we are about to write the shard data, but with the new shard modified. // Need to make sure that the read operations happen now before pva.set acquires a write lock if (existingData != null && modifiedData != null) { modifiedData.materialize(); @@ -369,8 +369,8 @@ private ReadData writeRegionRecursive( } final DataBlock chunk = chunkSupplier.get(gridPosition, existingChunk); - // null blocks may be provided when they contain only the fill value - // and only non-empty blocks should be written, for example + // null chunks may be provided when they contain only the fill value + // and only non-empty chunks should be written, for example if (chunk == null) return null; @@ -415,13 +415,13 @@ public boolean deleteChunk(final PositionValueAccess pva, final long[] gridPosit // nothing changed, the blocks we wanted to delete didn't exist anyway return false; } else if (modifiedData != null) { - // Here, we are about to write the shard data, but without the block to be deleted. + // Here, we are about to write the shard data, but without the chunk to be deleted. // Need to make sure that the read operations happen now before pva.set acquires a write lock modifiedData.materialize(); } } catch (final N5NoSuchKeyException e) { // the key didn't exist (as we found out when lazy-reading the index) - // so nothing changed, the blocks we wanted to delete didn't exist anyway + // so nothing changed, the chunks we wanted to delete didn't exist anyway return false; } if (modifiedData == null) { @@ -447,7 +447,7 @@ private ReadData deleteChunkRecursive( final long[] elementPos = position.relative(level - 1); final ReadData existingElementData = shard.getElementData(elementPos); if (existingElementData == null) { - // The DataBlock (or the whole nested shard containing it) does not exist. + // The chunk (or the whole nested shard containing it) does not exist. // This shard remains unchanged. return existingReadData; } else { @@ -459,7 +459,7 @@ private ReadData deleteChunkRecursive( } shard.setElementData(modifiedElementData, elementPos); if (modifiedElementData == null) { - // The DataBlock or nested shard was removed. + // The chunk or nested shard was removed. // Check whether this shard becomes empty. if (shard.isEmpty()) { // This shard is empty and should be removed. @@ -477,7 +477,7 @@ private ReadData deleteChunkRecursive( @Override public boolean deleteChunks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { - // for non-sharded datasets, just delete the blocks individually + // for non-sharded datasets, just delete the chunks individually if (grid.numLevels() == 1) { boolean deleted = false; for (long[] pos : gridPositions) { @@ -510,7 +510,7 @@ public boolean deleteChunks(final PositionValueAccess pva, final List gr } } catch (final N5NoSuchKeyException e) { // the key didn't exist (as we found out when lazy-reading the index) - // so nothing changed, the blocks we wanted to delete didn't exist anyway + // so nothing changed, the chunks we wanted to delete didn't exist anyway continue; } if (modifiedData == null) { @@ -735,7 +735,7 @@ public void writeBlock( final DataBlockFactory blockFactory = DataBlockFactory.of(shardData); final int n = grid.numDimensions(); - final int[] chunkSize = grid.getBlockSize(0); // size of a standard (non-truncated) DataBlock + final int[] chunkSize = grid.getBlockSize(0); // size of a standard (non-truncated) chunk final long[] datasetChunkSize = grid.getDatasetSizeInChunks(); final long[] shardPixelMin = grid.pixelPosition(dataBlock.getGridPosition(), level); @@ -770,7 +770,7 @@ public void writeBlock( destSize[d] = Math.min(chunkSize[d], (int) (regionBound[d] - pixelMin[d])); } - // This extracting DataBlocks will not work if num_array_elements != num_block_elements. + // This extracting chunks will not work if num_array_elements != num_block_elements. // But we'll deal with that later if it becomes a problem... final DataBlock chunk = blockFactory.createDataBlock(destSize, chunkPos); SubArrayCopy.copy(shardData, shardPixelSize, srcPos, chunk.getData(), destSize, destPos, destSize); From f9c0ea33be0faf56a8893225046ef51af6997460 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 30 Mar 2026 17:40:45 -0400 Subject: [PATCH 25/41] test: flesh out WriteRegionTest --- .../saalfeldlab/n5/shard/WriteRegionTest.java | 271 +++++++++++------- 1 file changed, 173 insertions(+), 98 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java index 00b356fae..90f979e61 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java @@ -38,6 +38,7 @@ import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.janelia.saalfeldlab.n5.N5Writer.DataBlockSupplier; import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; import org.junit.Test; @@ -45,119 +46,94 @@ public class WriteRegionTest { -// #...............................#...............................#...............................#...............................# -// $.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$ -// |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...| -// 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96 - - public static void main(String[] args) { + @Test + public void testWriteRegion() { -// int[] chunkSize = {3, 3, 3}; -// int[] level1ShardSize = {6, 6, 6}; -// int[] level2ShardSize = {24, 24, 24}; int[] chunkSize = {3}; - int[] level1ShardSize = {6}; - int[] level2ShardSize = {24}; - final long[] datasetDimensions = {96}; - - // Chunks are 3x3x3 - // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) - // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) + final long[] datasetDimensions = {15}; final BlockCodecInfo c0 = new N5BlockCodecInfo(); - final ShardCodecInfo c1 = new DefaultShardCodecInfo( - chunkSize, - c0, - new DataCodecInfo[] {new RawCompression()}, - new RawBlockCodecInfo(), - new DataCodecInfo[] {new RawCompression()}, - IndexLocation.END - ); - final ShardCodecInfo c2 = new DefaultShardCodecInfo( - level1ShardSize, - c1, - new DataCodecInfo[] {new RawCompression()}, - new RawBlockCodecInfo(), - new DataCodecInfo[] {new RawCompression()}, - IndexLocation.START - ); TestDatasetAttributes attributes = new TestDatasetAttributes( datasetDimensions, - level2ShardSize, + chunkSize, DataType.INT8, - c2, + c0, new RawCompression()); final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); + DataBlockSupplier chunks = (gridPos, existing) -> { + return createDataBlock(chunkSize, gridPos.clone(), (byte) gridPos[0]); + }; + + DataBlockSupplier chunks255 = (gridPos, existing) -> { + return createDataBlock(chunkSize, gridPos.clone(), (byte)255); + }; + + DataBlockSupplier chunksDelete = (gridPos, existing) -> { + return null; + }; - // --------------------------------------------------------------- - // Some "tests" - // TODO: Turn into unit tests - // --------------------------------------------------------------- - // write some blocks, filled with constant values - final int[] dataBlockSize = c1.getInnerBlockSize(); -// final long[] datasetDimensions = {100, 100, 100}; -// final long[] regionMin = {9,9,9}; -// final long[] regionSize = {15,15,15}; + // write one chunk at grid Position 1 + datasetAccess.writeRegion(store, + new long[] {3}, + new long[] {3}, + chunks, + false); - // #...............................#...............................#...............................#...............................# - // $.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$ - // |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...| - // 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96 + // Chunks + // |...|...|...|...|...| + // Pixels indexes + // 0 3 6 9 12 15- + + checkChunk(datasetAccess.readChunk(store, new long[] {0}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {1}), true, 1); + checkChunk(datasetAccess.readChunk(store, new long[] {2}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {3}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {4}), false, 0); + + // write two chunks at grid positions 2 and 3 + datasetAccess.writeRegion(store, + new long[]{6}, + new long[]{6}, + chunks, + false); - final long[] regionMin = {93}; - final long[] regionSize = {1}; - DataBlockSupplier blocks = (gridPos, existing) -> { -// System.out.println("BlockSupplier.get(" + Arrays.toString(gridPos) + ", " + existing + ")"); - return createDataBlock(dataBlockSize, gridPos.clone(), (byte) gridPos[0]); - }; + checkChunk(datasetAccess.readChunk(store, new long[] {0}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {1}), true, 1); + checkChunk(datasetAccess.readChunk(store, new long[] {2}), true, 2); + checkChunk(datasetAccess.readChunk(store, new long[] {3}), true, 3); + checkChunk(datasetAccess.readChunk(store, new long[] {4}), false, 0); + + // delete two chunks at grid positions 3 and 4 datasetAccess.writeRegion(store, - regionMin, - regionSize, - blocks, + new long[]{9}, + new long[]{6}, + chunksDelete, false); - // verify that the written blocks can be read back with the correct values -// checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), true, 1); -// checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); -// checkBlock(datasetAccess.readChunk(store, new long[] {0, 1, 0}), true, 3); -// checkBlock(datasetAccess.readChunk(store, new long[] {1, 1, 0}), true, 4); -// checkBlock(datasetAccess.readChunk(store, new long[] {3, 2, 1}), true, 5); -// checkBlock(datasetAccess.readChunk(store, new long[] {8, 4, 1}), true, 6); - - // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) -// datasetAccess.deleteBlock(store, new long[] {0, 0, 0}); -// checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), false, 1); -// checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); - - // if a shard becomes empty the corresponding key should be deleted -// if ( store.get(new long[] {1, 0, 0}) == null ) { -// throw new IllegalStateException("expected non-null readData"); -// } -// datasetAccess.deleteChunk(store, new long[] {8, 4, 1}); -// if ( store.get(new long[] {1, 0, 0}) != null ) { -// throw new IllegalStateException("expected null readData"); -// } - - // deleting a non-existent block should not fail -// datasetAccess.deleteChunk(store, new long[] {0, 0, 8}); - - System.out.println("all good"); } @Test public void testWriteRegionSharded() { + // Shards + // #...............................#...............................#...............................#...............................# + // Chunks + // |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...| + // Pixels indexes + // 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96 + int[] chunkSize = {3}; - int[] level2ShardSize = {24}; + int[] shardSize = {24}; final long[] datasetDimensions = {96}; + int numChunks = (int)(datasetDimensions[0] / chunkSize[0]); + + // Chunks are size 3 + // Shards are size 24 (contain 8 chunks) - // Chunks are 3x3x3 - // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) - // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( chunkSize, @@ -170,7 +146,7 @@ public void testWriteRegionSharded() { TestDatasetAttributes attributes = new TestDatasetAttributes( datasetDimensions, - level2ShardSize, + shardSize, DataType.INT8, c1, new RawCompression()); @@ -178,25 +154,113 @@ public void testWriteRegionSharded() { final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); - final int[] dataBlockSize = c1.getInnerBlockSize(); - final long[] regionMin = {93}; - final long[] regionSize = {1}; + DataBlockSupplier chunks = (gridPos, existing) -> { + return createDataBlock(chunkSize, gridPos.clone(), (byte) gridPos[0]); + }; + + DataBlockSupplier chunks255 = (gridPos, existing) -> { + return createDataBlock(chunkSize, gridPos.clone(), (byte)255); + }; - DataBlockSupplier blocks = (gridPos, existing) -> { - return createDataBlock(dataBlockSize, gridPos.clone(), (byte) gridPos[0]); + DataBlockSupplier chunksDelete = (gridPos, existing) -> { + return null; }; + // write one chunk at grid Position 1 + datasetAccess.writeRegion(store, + new long[] {3}, + new long[] {3}, + chunks, + false); + + checkChunk(datasetAccess.readChunk(store, new long[] {0}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {1}), true, 1); + checkChunk(datasetAccess.readChunk(store, new long[] {2}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {3}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {4}), false, 0); + + // only the first shard should exist + checkKey(store, new long[]{0}, true); + checkKey(store, new long[]{1}, false); + checkKey(store, new long[]{2}, false); + checkKey(store, new long[]{3}, false); + + // write two chunks at grid positions 2 and 3 + datasetAccess.writeRegion(store, + new long[]{6}, + new long[]{6}, + chunks, + false); + + checkChunk(datasetAccess.readChunk(store, new long[]{0}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[]{1}), true, 1); + checkChunk(datasetAccess.readChunk(store, new long[]{2}), true, 2); + checkChunk(datasetAccess.readChunk(store, new long[]{3}), true, 3); + checkChunk(datasetAccess.readChunk(store, new long[]{4}), false, 0); + + // delete two chunks at grid positions 3 and 4 + datasetAccess.writeRegion(store, + new long[]{9}, + new long[]{6}, + chunksDelete, + false); + + checkChunk(datasetAccess.readChunk(store, new long[]{0}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[]{1}), true, 1); + checkChunk(datasetAccess.readChunk(store, new long[]{2}), true, 2); + checkChunk(datasetAccess.readChunk(store, new long[]{3}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[]{4}), false, 0); + + // overwrite all chunks to hold 255 datasetAccess.writeRegion(store, - regionMin, - regionSize, - blocks, + new long[]{0}, + new long[]{96}, + chunks255, false); + for (int i = 0; i < numChunks; i++) { + checkChunk(datasetAccess.readChunk(store, new long[]{i}), true, 255); + } + + // all shards should exist + checkKey(store, new long[]{0}, true); + checkKey(store, new long[]{1}, true); + checkKey(store, new long[]{2}, true); + checkKey(store, new long[]{3}, true); + + // delete some chunks + datasetAccess.writeRegion(store, + new long[]{18}, + new long[]{18}, + chunksDelete, + false); + + checkChunk(datasetAccess.readChunk(store, new long[]{5}), true, 255); + checkChunk(datasetAccess.readChunk(store, new long[]{6}), false, 0); + + // all shards should exist + checkKey(store, new long[]{0}, true); + checkKey(store, new long[]{1}, true); + checkKey(store, new long[]{2}, true); + checkKey(store, new long[]{3}, true); + + // delete more chunks so that shard 1 is empty + datasetAccess.writeRegion(store, + new long[]{36}, + new long[]{15}, + chunksDelete, + false); + + // shard 1 should be gone + checkKey(store, new long[]{0}, true); + checkKey(store, new long[]{1}, false); + checkKey(store, new long[]{2}, true); + checkKey(store, new long[]{3}, true); } - private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { + private static void checkChunk(final DataBlock chunk, final boolean expectedNonNull, final int expectedFillValue) { - if (dataBlock == null) { + if (chunk == null) { if (expectedNonNull) { throw new IllegalStateException("expected non-null dataBlock"); } @@ -204,7 +268,7 @@ private static void checkBlock(final DataBlock dataBlock, final boolean if (!expectedNonNull) { throw new IllegalStateException("expected null dataBlock"); } - final byte[] bytes = dataBlock.getData(); + final byte[] bytes = chunk.getData(); for (byte b : bytes) { if (b != (byte) expectedFillValue) { throw new IllegalStateException("expected all values to be " + expectedFillValue); @@ -213,6 +277,17 @@ private static void checkBlock(final DataBlock dataBlock, final boolean } } + private static void checkKey(PositionValueAccess pva, long[] position, final boolean expectedNonNull) { + + try (VolatileReadData val = pva.get(position)) { + + if (expectedNonNull && val == null) + throw new IllegalStateException("expected non-null value"); + else if (!expectedNonNull && val != null) + throw new IllegalStateException("expected null value"); + } + } + private static DataBlock createDataBlock(int[] size, long[] gridPosition, int fillValue) { final byte[] bytes = new byte[DataBlock.getNumElements(size)]; Arrays.fill(bytes, (byte) fillValue); From c72e4679cbe56f7136deb8d9d25be49c40fb9b5b Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 3 Mar 2026 11:13:26 -0500 Subject: [PATCH 26/41] perf: Don't use `Files.copy` for Unsafe. It's much slower than writing via FileChannel. fix: ensure parent directories exist before writing file for Unsafe Signed-off-by: Caleb Hulbert --- .../janelia/saalfeldlab/n5/ChannelLock.java | 26 ++--------------- .../janelia/saalfeldlab/n5/FsIoPolicy.java | 29 ++++++++++++++++++- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java b/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java index c8c6fc691..4267845c8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java @@ -6,9 +6,7 @@ import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; /** * Holds a channel and system-level file lock (shared for writing, non-shared @@ -56,7 +54,7 @@ FileChannel getChannel() { */ static ChannelLock lock(final Path path, final boolean forWriting) throws IOException { - final FileChannel channel = openFileChannel(path, forWriting); + final FileChannel channel = FsIoPolicy.openFileChannel(path, forWriting); try { while (true) { try { @@ -94,7 +92,7 @@ static ChannelLock tryLock(final Path path, final boolean forWriting) throws IOE FileChannel channel = null; try { - channel = openFileChannel(path, forWriting); + channel = FsIoPolicy.openFileChannel(path, forWriting); final FileLock lock = channel.tryLock(0, Long.MAX_VALUE, !forWriting); return lock == null ? null : new ChannelLock(channel, lock); } catch (Exception e) { @@ -103,26 +101,6 @@ static ChannelLock tryLock(final Path path, final boolean forWriting) throws IOE } } - /** - * Opens a file channel. If the channel is opened {@code forWriting}, - * then this may create the file and the parent directories as needed. - * - * @throws IOException - * if the channel cannot be opened - */ - private static FileChannel openFileChannel(final Path path, final boolean forWriting) throws IOException { - - if (forWriting) { - final Path parent = path.getParent(); - if (parent != null) { - Files.createDirectories(parent); - } - return FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE); - } else { - return FileChannel.open(path, StandardOpenOption.READ); - } - } - private static void closeQuietly(final FileChannel fileChannel) { if (fileChannel != null) { try { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java index 196cbc2ab..ab237e3f1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java @@ -6,8 +6,10 @@ import java.io.Closeable; import java.io.IOException; +import java.io.OutputStream; import java.io.UncheckedIOException; import java.nio.ByteBuffer; +import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.file.*; @@ -29,11 +31,36 @@ else if (length >= 0 && offset + length > channelSize) return true; } + /** + * Opens a file channel. If the channel is opened {@code forWriting}, + * then this may create the file and the parent directories as needed. + * + * @throws IOException + * if the channel cannot be opened + */ + static FileChannel openFileChannel(final Path path, final boolean forWriting) throws IOException { + + if (forWriting) { + final Path parent = path.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + return FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE); + } else { + return FileChannel.open(path, StandardOpenOption.READ); + } + } + public static class Unsafe implements IoPolicy { @Override public void write(String key, ReadData readData) throws IOException { final Path path = Paths.get(key); - Files.copy(readData.inputStream(), path, StandardCopyOption.REPLACE_EXISTING); + try ( + FileChannel channel = openFileChannel(path, true); + OutputStream os = Channels.newOutputStream(channel); + ) { + readData.writeTo(os); + } } @Override From b39bd0bc164eb15acc1bc58db801b760596c63e7 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 3 Mar 2026 11:17:22 -0500 Subject: [PATCH 27/41] perf: don't truncate separately over the file channel when writing, just specify the TRUNCATE_EXISTING OpenOption. It is much faster Signed-off-by: Caleb Hulbert --- .../java/org/janelia/saalfeldlab/n5/FsIoPolicy.java | 2 +- .../org/janelia/saalfeldlab/n5/LockedFileChannel.java | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java index ab237e3f1..37b137686 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java @@ -45,7 +45,7 @@ static FileChannel openFileChannel(final Path path, final boolean forWriting) th if (parent != null) { Files.createDirectories(parent); } - return FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE); + return FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); } else { return FileChannel.open(path, StandardOpenOption.READ); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java index b678dfe41..9f8a82823 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java @@ -45,14 +45,12 @@ public InputStream newInputStream() throws N5Exception.N5IOException { @Override public Writer newWriter() throws N5Exception.N5IOException { - truncateChannel(); return Channels.newWriter(channel, StandardCharsets.UTF_8.name()); } @Override public OutputStream newOutputStream() throws N5Exception.N5IOException { - truncateChannel(); return Channels.newOutputStream(channel); } @@ -62,15 +60,6 @@ public FileChannel getFileChannel() { return channel; } - private void truncateChannel() throws N5Exception.N5IOException { - - try { - channel.truncate(0); - } catch (final IOException e) { - throw new N5Exception.N5IOException("Failed to truncate channel", e); - } - } - @Override public void close() throws IOException { From 61631c160053cff23cf5bb09c98be104bbc5be8e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 3 Mar 2026 11:20:24 -0500 Subject: [PATCH 28/41] perf: only try to create directory if parent is not already a directory (isFile -> throws, null -> creates) Signed-off-by: Caleb Hulbert --- src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java index 37b137686..2eafda6cd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java @@ -42,7 +42,8 @@ static FileChannel openFileChannel(final Path path, final boolean forWriting) th if (forWriting) { final Path parent = path.getParent(); - if (parent != null) { + /* if not null and not directory, it will call `createDirectories` but we expect it to throw an IOException */ + if (parent != null && !parent.toFile().isDirectory()) { Files.createDirectories(parent); } return FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); From fdf451ef7c31bcdcc0c3f2f5f237136d948c94af Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 3 Mar 2026 11:21:12 -0500 Subject: [PATCH 29/41] refactor: make explicit the use of the FileLock in ChannelLock Signed-off-by: Caleb Hulbert --- src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java b/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java index 4267845c8..643d43bd0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java @@ -16,11 +16,14 @@ class ChannelLock implements Closeable { private final FileChannel channel; - private final FileLock lock; + + //Used to ensure a hard reference exists until we close + @SuppressWarnings({"unused", "FieldCanBeLocal"}) + private final FileLock unusedHardRef; private ChannelLock(final FileChannel channel, final FileLock lock) { this.channel = channel; - this.lock = lock; + this.unusedHardRef = lock; } public void close() throws IOException { From d175b543de42ead1e2601d948bb7ebf598b54bf7 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 3 Apr 2026 11:43:19 -0400 Subject: [PATCH 30/41] fix: continue-on-close-failure behavior needs to work for successful writes that also fail on closing the file. It's a bit more involved, since we need to flush the write operation, but the logic is the same as for read Signed-off-by: Caleb Hulbert --- .../janelia/saalfeldlab/n5/FsIoPolicy.java | 46 +++++++++++++++---- .../saalfeldlab/n5/LockedFileChannel.java | 9 +--- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java index 2eafda6cd..cbd9cc734 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java @@ -52,16 +52,44 @@ static FileChannel openFileChannel(final Path path, final boolean forWriting) th } } + + /** + * This method is necessary to handle the situtation where writing is successful, but `close` fails on the file channel. + * This has been observed to happen fairly consistently on MacOS when writing to a file mounted over SMB. + * + * @param readData to write to the {@code Path} + * @param path to write to + * @throws IOException if writing failed. + */ + private static void writeToPathIgnoreCloseException(ReadData readData, Path path) throws IOException { + + FileChannel channel = openFileChannel(path, true); + OutputStream os = Channels.newOutputStream(channel); + + try { + readData.writeTo(os); + os.flush(); + channel.force(true); + } catch (Throwable e) { + os.close(); + channel.close(); + throw e; + } + + /* if we get here, the write succeeded, and the os/channel may not be closed yet */ + try { + os.close(); + channel.close(); + } catch (IOException | UncheckedIOException ignore) { + /* Ignore; we know the data was written already. */ + } + } + public static class Unsafe implements IoPolicy { @Override public void write(String key, ReadData readData) throws IOException { final Path path = Paths.get(key); - try ( - FileChannel channel = openFileChannel(path, true); - OutputStream os = Channels.newOutputStream(channel); - ) { - readData.writeTo(os); - } + writeToPathIgnoreCloseException(readData, path); } @Override @@ -164,9 +192,9 @@ public ReadData materialize(final long offset, final long length) { throw new N5Exception.N5NoSuchKeyException("No such file", e); } catch (IOException | UncheckedIOException e) { /* Occasionally (frequently for some source remote mounted file systems) this can throw exceptions during - * `channel.close()` which is called automatically in the try-with-resources block. In this case, we have - * successfully read the data, and we can return it, and ignore the exception. - * */ + * `channel.close()` which is called automatically in the try-with-resources block. In this case, we have + * successfully read the data, and we can return it, and ignore the exception. + * */ if (readData == null) throw new N5Exception.N5IOException(e); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java index 9f8a82823..0e84071ab 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java @@ -12,8 +12,7 @@ /** * LockedFileChannel implementation for both read and write operations. */ -// TODO: This only has to be public because of a test in another package. Fix that -public class LockedFileChannel implements LockedChannel { +class LockedFileChannel implements LockedChannel { private final FileChannel channel; private ReleaseLock releaseLock; @@ -54,12 +53,6 @@ public OutputStream newOutputStream() throws N5Exception.N5IOException { return Channels.newOutputStream(channel); } - // TODO: This only has to be public because of a test in another package. Fix that - public FileChannel getFileChannel() { - - return channel; - } - @Override public void close() throws IOException { From c3a815dab5f4501a9dbd79dc6da80de970cb3a09 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 2 Feb 2026 21:16:29 +0100 Subject: [PATCH 31/41] Add SliceTrackingLazyRead This is in preparation for implementing prefetch() for cloud storage backends... --- .../prefetch/SliceTrackingLazyRead.java | 114 +++++++++++++++ .../n5/readdata/prefetch/Slices.java | 116 +++++++++++++++ .../n5/readdata/prefetch/SlicesTest.java | 134 ++++++++++++++++++ 3 files changed, 364 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java new file mode 100644 index 000000000..3ffd852ea --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java @@ -0,0 +1,114 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.Range; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +public class SliceTrackingLazyRead implements LazyRead { + + private static class Slice implements Range { + + // Offset and length in the delegate + private final long offset; + private final long length; + + // Data of this slice + private final ReadData data; + + Slice(final long offset, final long length, final ReadData data) { + this.offset = offset; + this.length = length; + this.data = data; + } + + @Override + public long offset() { + return offset; + } + + @Override + public long length() { + return length; + } + + @Override + public String toString() { + return "{" + offset + ", " + length + '}'; + } + } + + private final List slices = new ArrayList<>(); + + /** + * The {@code LazyRead} providing our data. + */ + private final LazyRead delegate; + + public SliceTrackingLazyRead(final LazyRead delegate) { + this.delegate = delegate; + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + @Override + public ReadData materialize(final long offset, final long length) throws N5IOException { + final Slice containing = Slices.findContainingSlice(slices, offset, length); + if (containing != null) { + return containing.data.slice(offset - containing.offset, length); + } else { + final ReadData data = delegate.materialize(offset, length); + Slices.addSlice(slices, new Slice(offset, length, data)); + return data; + } + } + + @Override + public long size() throws N5IOException { + return delegate.size(); + } + + /** + * Indicates that the given slices will be subsequently read. + * {@code LazyRead} implementations (optionally) may take steps to prepare + * for these subsequent slices. + *

+ * Minimal implementation: Find offset and length covering all ranges that + * are not yet fully covered by existing slices. Then materialize the slice + * covering that range. + * + * @param ranges + * slice ranges to prefetch + * + * @throws N5IOException + * if any I/O error occurs + */ + @Override + public void prefetch(final Collection ranges) throws N5IOException { + + long fromIndex = Long.MAX_VALUE; + long toIndex = Long.MIN_VALUE; + for (final Range slice : ranges) { + if (!isCovered(slice)) { + fromIndex = Math.min(fromIndex, slice.offset()); + toIndex = Math.max(toIndex, slice.end()); + } + } + + if (fromIndex < toIndex) { + materialize(fromIndex, toIndex - fromIndex); + } + } + + private boolean isCovered(final Range slice) { + + return Slices.findContainingSlice(slices, slice) != null; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java new file mode 100644 index 000000000..d3d0dfb11 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java @@ -0,0 +1,116 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import org.janelia.saalfeldlab.n5.readdata.Range; + +class Slices { + + private Slices() { + // utility class. should not be instantiated. + } + + /** + * In an ordered list of {@code slices}, find a slice that completely contains the given range. + *

+ * Pre-conditions: + *

    + *
  1. Slices are ordered by offset.
    2. + *
  2. If two slices overlap, no slice is fully contained within the other. + * (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
    3. + *
+ * + * @param slices + * ordered list of slices + * @param offset + * start of the range to cover + * @param length + * length of the range to cover + * + * @return a slice that completely contains the requested range, or {@code null} if no such slice exists + */ + static T findContainingSlice(final List slices, final long offset, final long length) { + + // Find the slice with the largest slice.offset <= offset. + final int i = Collections.binarySearch(slices, Range.at(offset, 0), Comparator.comparingLong(Range::offset)); + + // Largest index of a slice with slice.offset <= offset. + final int index = i < 0 ? -i - 2 : i; + if (index < 0) { + // We find no overlapping slice, because + // slices[0].offset is already too large. + return null; + } + + final T slice = slices.get(index); + if (slice.end() < offset + length) { + return null; + } + + return slice; + } + + /** + * In an ordered list of {@code slices}, find a slice that completely contains the given range. + *

+ * Pre-conditions: + *

    + *
  1. Slices are ordered by offset.
    2. + *
  2. If two slices overlap, no slice is fully contained within the other. + * (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
    3. + *
+ * + * @param slices + * ordered list of slices + * @param range + * range to cover + * + * @return a slice that completely contains the requested range, or {@code null} if no such slice exists + */ + static T findContainingSlice(final List slices, final Range range) { + return findContainingSlice(slices, range.offset(), range.length()); + } + + /** + * Add a new {@code slice} to the {@code slice} list. + *

+ * Note, that the new {@code slice} is expected to not be fully contained in + * an existing slice! + *

+ * Pre/post-conditions: + *

    + *
  1. Slices are ordered by offset.
    2. + *
  2. If two slices overlap, no slice is fully contained within the other. + * (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
    3. + *
+ *

+ * The new {@code slice} will be inserted into the list at the correct position + * (such that {@code slices} remains ordered by slice offset), and all existing + * slices that are fully contained in the new {@code slice} will be removed. + * + * @param slices + * ordered list of slices + * @param slice + * slice to be inserted + */ + static void addSlice(final List slices, final T slice) { + + final int i = Collections.binarySearch(slices, slice, Comparator.comparingLong(Range::offset)); + final int from = i < 0 ? -i - 1 : i; + + int to = from; + while (to < slices.size() && slices.get(to).end() <= slice.end()) { + ++to; + } + + if (from == to) { + // empty range: just insert + slices.add(from, slice); + } else { + // overwrite the first element in range, remove the rest + slices.set(from, slice); + slices.subList(from + 1, to).clear(); + } + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java new file mode 100644 index 000000000..0ce2b7442 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java @@ -0,0 +1,134 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.util.ArrayList; +import java.util.List; +import org.janelia.saalfeldlab.n5.readdata.Range; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + + +public class SlicesTest { + + private List createSlices(final long[] offsets, final long[] lengths) { + final List slices = new ArrayList<>(); + for (int i = 0; i < offsets.length; ++i) { + slices.add(Range.at(offsets[i], lengths[i])); + } + return slices; + } + + @Test + public void testFindContaining() { + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,6) [-----------] + // (6,4) [---------] + // (8,6) [-----------] + + final List slices = createSlices( + new long[] {2, 6, 8}, + new long[] {6, 4, 6}); + Range slice; + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (1,1) [-] + slice = Slices.findContainingSlice(slices, 1, 1); + assertEquals(null, slice); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,1) [-] + slice = Slices.findContainingSlice(slices, 2, 1); + assertEquals(2, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,6) [-----------] + slice = Slices.findContainingSlice(slices, 2, 6); + assertEquals(2, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,7) [-------------] + slice = Slices.findContainingSlice(slices, 2, 7); + assertEquals(null, slice); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (6,4) [-------] + slice = Slices.findContainingSlice(slices, 6, 4); + assertEquals(6, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (8,2) [---] + slice = Slices.findContainingSlice(slices, 8, 2); + assertEquals(8, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (12,2) [---] + slice = Slices.findContainingSlice(slices, 12, 2); + assertEquals(8, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (12,3) [-----] + slice = Slices.findContainingSlice(slices, 12, 3); + assertEquals(null, slice); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (14,1) [-] + slice = Slices.findContainingSlice(slices, 14, 1); + assertEquals(null, slice); + } + + + @Test + public void testAddSlice() { + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,6) [-----------] + // (6,4) [---------] + // (8,6) [-----------] + final List initial = createSlices( + new long[] {2, 6, 8}, + new long[] {6, 4, 6}); + List slices; + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, Range.at(0, 1)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (0,1) [-] + // (2,6) [-----------] + // (6,4) [---------] + // (8,6) [-----------] + assertEquals(createSlices( + new long[] {0, 2, 6, 8}, + new long[] {1, 6, 4, 6}), slices); + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, Range.at(0, 16)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (0,16)[-------------------------------] + assertEquals(createSlices( + new long[] {0}, + new long[] {16}), slices); + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, Range.at(2, 8)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,8) [-----------------] + // (8,6) [-----------] + assertEquals(createSlices( + new long[] {2, 8}, + new long[] {8, 6}), slices); + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, Range.at(1, 10)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (1,10) [---------------------] + // (8,6) [-----------] + assertEquals(createSlices( + new long[] {1, 8}, + new long[] {10, 6}), slices); + } +} From 639f6cccab185a9a5ba0c3fed65c4478b06578f5 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 2 Feb 2026 17:00:34 -0500 Subject: [PATCH 32/41] feat: add Range.aggregate * and a test --- .../saalfeldlab/n5/readdata/Range.java | 50 +++++++++++++++- .../saalfeldlab/n5/readdata/RangeTests.java | 58 +++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java index eff501ba2..a42c18ba4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java @@ -28,6 +28,9 @@ */ package org.janelia.saalfeldlab.n5.readdata; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.Comparator; /** @@ -61,7 +64,7 @@ default long end() { } static boolean equals(final Range r0, final Range r1) { - if (r0 == null && r1==null) { + if (r0 == null && r1 == null) { return true; } else if (r0 == null || r1 == null) { return false; @@ -119,4 +122,49 @@ public String toString() { return new DefaultRange(offset, length); } + + /** + * Returns a potentially new collection of {@link Range}s such that + * adjacent or overlapping Ranges are combined. + *

+ * If the input ranges are non-adjacent the input instance is returned, but if aggregation + * occurs, the result will be a new, sorted List. + * + * @param ranges + * a collection of Ranges + * @return + */ + static Collection aggregate(final Collection ranges) { + + if (ranges.size() == 0) + return ranges; + + ArrayList sortedRanges = new ArrayList<>(ranges); + Collections.sort(sortedRanges, Range.COMPARATOR); + + ArrayList result = new ArrayList<>(); + boolean wereMerges = false; + Range lo = null; + for (Range hi : sortedRanges) { + + if (lo == null) + lo = hi; + else if (lo.end() >= hi.offset()) { + // merge + final Range mergedLo = Range.at(lo.offset(), Math.max(lo.end(), hi.end()) - lo.offset()); + lo = mergedLo; + wereMerges = true; + } else { + result.add(lo); + lo = hi; + } + } + result.add(lo); + + if (!wereMerges) + return ranges; + + return result; + } + } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java new file mode 100644 index 000000000..3499220b9 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java @@ -0,0 +1,58 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.junit.Test; + +public class RangeTests { + + @Test + public void testAggregate() { + + List nonOverlapping = Stream.of(Range.at(0, 5), Range.at(12, 2)).collect(Collectors.toList()); + assertSame(nonOverlapping, Range.aggregate(nonOverlapping)); + + List nonOverlappingRev = Stream.of(Range.at(12, 2), Range.at(0, 5)).collect(Collectors.toList()); + assertSame(nonOverlappingRev, Range.aggregate(nonOverlappingRev)); + + /** + * 0 1 2 3 4 5 + * x x x x x - + * - x - - - - + */ + List containing = Stream.of(Range.at(0, 5), Range.at(1, 1)).collect(Collectors.toList()); + assertEquals(Collections.singletonList(Range.at(0, 5)), Range.aggregate(containing)); + + /** + * 0 1 2 3 4 5 6 + * x x x x x - - + * - - x x x x x + */ + List overlapping = Stream.of(Range.at(0, 5), Range.at(2, 5)).collect(Collectors.toList()); + assertEquals(Collections.singletonList(Range.at(0, 7)), Range.aggregate(overlapping)); + + /** + * 0 1 2 3 4 5 + * x x x x x - + * - - - - - x + */ + List adjacent = Stream.of(Range.at(0, 5), Range.at(5, 1)).collect(Collectors.toList()); + assertEquals(Collections.singletonList(Range.at(0, 6)), Range.aggregate(adjacent)); + + /** + * 0 1 2 3 4 5 + * - - - x x - + * x x - - - - + * - - - - - x + */ + List three = Stream.of(Range.at(3, 2), Range.at(0, 2), Range.at(5, 1)).collect(Collectors.toList()); + assertEquals(Stream.of(Range.at(0, 2), Range.at(3, 3)).collect(Collectors.toList()), Range.aggregate(three)); + } + +} From 861dfa1181b875c83815ff1e320f908b53a9e235 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 3 Apr 2026 21:40:56 +0200 Subject: [PATCH 33/41] simplify --- .../saalfeldlab/n5/readdata/Range.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java index a42c18ba4..06e753366 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java @@ -30,7 +30,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; /** @@ -131,19 +130,19 @@ public String toString() { * occurs, the result will be a new, sorted List. * * @param ranges - * a collection of Ranges - * @return + * a collection of Ranges + * + * @return collection with adjacent or overlapping ranges merged */ static Collection aggregate(final Collection ranges) { if (ranges.size() == 0) return ranges; - ArrayList sortedRanges = new ArrayList<>(ranges); - Collections.sort(sortedRanges, Range.COMPARATOR); + final ArrayList sortedRanges = new ArrayList<>(ranges); + sortedRanges.sort(Range.COMPARATOR); - ArrayList result = new ArrayList<>(); - boolean wereMerges = false; + final ArrayList result = new ArrayList<>(); Range lo = null; for (Range hi : sortedRanges) { @@ -151,9 +150,7 @@ static Collection aggregate(final Collection r lo = hi; else if (lo.end() >= hi.offset()) { // merge - final Range mergedLo = Range.at(lo.offset(), Math.max(lo.end(), hi.end()) - lo.offset()); - lo = mergedLo; - wereMerges = true; + lo = Range.at(lo.offset(), Math.max(lo.end(), hi.end()) - lo.offset()); } else { result.add(lo); lo = hi; @@ -161,10 +158,8 @@ else if (lo.end() >= hi.offset()) { } result.add(lo); - if (!wereMerges) - return ranges; - - return result; - } + final boolean wereMerges = ranges.size() != result.size(); + return wereMerges ? result : ranges; + } } From f702c7104c3e9949463a9b012bdbd2e6064e5a15 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 2 Feb 2026 17:01:52 -0500 Subject: [PATCH 34/41] feat: make SliceTrackingLazyRead abstract, add two implementations * Default- and Aggregating- * add tests --- .../AggregatingSliceTrackingLazyRead.java | 35 +++ .../DefaultSliceTrackingLazyRead.java | 50 ++++ .../prefetch/SliceTrackingLazyRead.java | 6 +- .../prefetch/SliceTrackingLazyReadTests.java | 233 ++++++++++++++++++ 4 files changed, 321 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingSliceTrackingLazyRead.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/DefaultSliceTrackingLazyRead.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingSliceTrackingLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingSliceTrackingLazyRead.java new file mode 100644 index 000000000..fc4fb660d --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingSliceTrackingLazyRead.java @@ -0,0 +1,35 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.util.Collection; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.Range; + +public class AggregatingSliceTrackingLazyRead extends SliceTrackingLazyRead { + + public AggregatingSliceTrackingLazyRead(final LazyRead delegate) { + super(delegate); + } + + /** + * Indicates that the given slices will be subsequently read. + *

+ * This implementation groups overlapping / adjacent {@link Range}s into single read requests. + * + * @param ranges + * slice ranges to prefetch + * + * @throws N5IOException + * if any I/O error occurs + */ + @Override + public void prefetch(final Collection ranges) throws N5IOException { + + final Collection aggregatedRanges = Range.aggregate(ranges); + for (final Range slice : aggregatedRanges) { + materialize(slice.offset(), slice.length()); + } + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/DefaultSliceTrackingLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/DefaultSliceTrackingLazyRead.java new file mode 100644 index 000000000..1e607ddc1 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/DefaultSliceTrackingLazyRead.java @@ -0,0 +1,50 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.util.Collection; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.Range; + +public class DefaultSliceTrackingLazyRead extends SliceTrackingLazyRead { + + public DefaultSliceTrackingLazyRead(final LazyRead delegate) { + super(delegate); + } + + /** + * Indicates that the given slices will be subsequently read. + * {@code LazyRead} implementations (optionally) may take steps to prepare + * for these subsequent slices. + *

+ * Minimal implementation: Find offset and length covering all ranges that + * are not yet fully covered by existing slices. Then materialize the slice + * covering that range. + * + * @param ranges + * slice ranges to prefetch + * + * @throws N5IOException + * if any I/O error occurs + */ + @Override + public void prefetch(final Collection ranges) throws N5IOException { + + long fromIndex = Long.MAX_VALUE; + long toIndex = Long.MIN_VALUE; + for (final Range slice : ranges) { + if (!isCovered(slice)) { + fromIndex = Math.min(fromIndex, slice.offset()); + toIndex = Math.max(toIndex, slice.end()); + } + } + + if (fromIndex < toIndex) { + materialize(fromIndex, toIndex - fromIndex); + } + } + + private boolean isCovered(final Range slice) { + + return Slices.findContainingSlice(slices, slice) != null; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java index 3ffd852ea..de20f2c39 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java @@ -9,9 +9,9 @@ import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; -public class SliceTrackingLazyRead implements LazyRead { +public abstract class SliceTrackingLazyRead implements LazyRead { - private static class Slice implements Range { + protected static class Slice implements Range { // Offset and length in the delegate private final long offset; @@ -42,7 +42,7 @@ public String toString() { } } - private final List slices = new ArrayList<>(); + protected final List slices = new ArrayList<>(); /** * The {@code LazyRead} providing our data. diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java new file mode 100644 index 000000000..044e6c1a3 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java @@ -0,0 +1,233 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.Range; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.junit.Test; + +public class SliceTrackingLazyReadTests { + + @Test + public void testDefaultSliceTracking() throws N5IOException { + + /** + * 1. Create sample ReadData from byte[] + * 2. Create a DummyLazy Read + * 3. Make a DefaultSliceTrackingLazyRead + * 4. Create a list of ranges + * 5. Call prefetch + * 6. Ensure the correct number of materialize calls were made (always 1 for DefaultSliceTrackingLazyRead) + * 7. Verify the stored slices contain the correct range + */ + + // 1-2. Create a DummyLazyRead with 64 bytes + DummyLazyRead dummyLazyRead = createDummyLazyRead(64); + + // 3. Make a testable DefaultSliceTrackingLazyRead + TestableDefaultSliceTracker sliceTracking = new TestableDefaultSliceTracker(dummyLazyRead); + + // 4. Create a list of ranges (two non-overlapping ranges with a gap) + List ranges = Arrays.asList( + Range.at(10, 5), // offset 10, length 5 (bytes 10-14) + Range.at(50, 10) // offset 50, length 10 (bytes 50-59) + ); + + // 5. Call prefetch + sliceTracking.prefetch(ranges); + + // 6. Ensure exactly 1 materialize call was made + // DefaultSliceTrackingLazyRead creates a single large slice covering all ranges + assertEquals("DefaultSliceTrackingLazyRead should make exactly 1 materialize call", + 1, dummyLazyRead.getNumMaterializeCalls()); + + // 7. Verify the stored slice covers the entire range from offset 10 with length 50 + assertStoredSlices(sliceTracking, Arrays.asList( + Range.at(10, 50) // Single slice covering offset 10-59 + )); + + } + + @Test + public void testAggregatingSliceTracking() throws N5IOException { + + /** + * 1. Create sample ReadData from byte[] + * 2. Create a DummyLazyRead + * 3. Make an AggregatingSliceTrackingLazyRead + * 4. Create a list of ranges + * 5. Call prefetch + * 6. Ensure the correct number of materialize calls were made (one per aggregated range) + * 7. Verify the stored slices contain the correct ranges + */ + + // 1-2. Create a DummyLazyRead with 64 bytes + DummyLazyRead dummyLazyRead = createDummyLazyRead(64); + + // 3. Make a testable AggregatingSliceTrackingLazyRead + TestableAggregatingSliceTracker sliceTracking = new TestableAggregatingSliceTracker(dummyLazyRead); + + /* + * Non-adjacent ranges + */ + // 4. Create a list of ranges (two non-overlapping ranges with a gap) + List ranges = Arrays.asList( + Range.at(10, 5), // offset 10, length 5 (bytes 10-14) + Range.at(50, 10) // offset 50, length 10 (bytes 50-59) + ); + + // 5. Call prefetch + sliceTracking.prefetch(ranges); + + // 6. Ensure exactly 2 materialize calls were made + // AggregatingSliceTrackingLazyRead aggregates overlapping/adjacent ranges + // Since these ranges are not adjacent or overlapping, it makes 2 separate calls + assertEquals("AggregatingSliceTrackingLazyRead should make 2 materialize calls for non-adjacent ranges", + 2, dummyLazyRead.getNumMaterializeCalls()); + + // 7. Verify the stored slices contain two separate ranges + assertStoredSlices(sliceTracking, Arrays.asList( + Range.at(10, 5), // First slice + Range.at(50, 10) // Second slice + )); + + /* + * Adjacent ranges + */ + + // new sliceTracking instance to clear slices + sliceTracking = new TestableAggregatingSliceTracker(dummyLazyRead); + dummyLazyRead.resetNumMaterializeCalls(); + + // 4. Create a list of three contiguous ranges + List adjacentRanges = Arrays.asList( + Range.at(10, 5), // offset 10, length 5 (bytes 10-14) + Range.at(15, 10), // offset 15, length 10 (bytes 15-24) + Range.at(25, 5) // offset 25, length 5 (bytes 25-29) + ); + + // 5. Call prefetch + sliceTracking.prefetch(adjacentRanges); + + // 6. Ensure exactly 1 materialize call was made + // AggregatingSliceTrackingLazyRead should aggregate these three contiguous ranges + // into a single range from offset 10 to 30, with length 20 + assertEquals("AggregatingSliceTrackingLazyRead should make 1 materialize call for contiguous ranges", + 1, dummyLazyRead.getNumMaterializeCalls()); + + // 7. Verify the stored slices now contain three ranges total: + // the two from the first prefetch plus one aggregated range from the second prefetch + assertStoredSlices(sliceTracking, Arrays.asList( + Range.at(10, 20) // Aggregated range + )); + + } + + private static DummyLazyRead createDummyLazyRead(int size) { + byte[] data = new byte[size]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)i; + } + return new DummyLazyRead(ReadData.from(data)); + } + + /** + * Helper method to verify that stored slices match expected ranges. + * + * @param sliceTracking the SliceTrackingLazyRead instance + * @param expectedRanges the expected ranges stored in slices + */ + private static void assertStoredSlices(TestableSliceTracker sliceTracking, List expectedRanges) { + // Access protected slices field via a test helper + List actualSlices = sliceTracking.getSlices(); + + assertEquals("Number of stored slices should match", expectedRanges.size(), actualSlices.size()); + + for (int i = 0; i < expectedRanges.size(); i++) { + Range expected = expectedRanges.get(i); + Range actual = actualSlices.get(i); + assertEquals("Slice " + i + " offset should match", expected.offset(), actual.offset()); + assertEquals("Slice " + i + " length should match", expected.length(), actual.length()); + } + } + + /** + * Testable wrapper for DefaultSliceTrackingLazyRead that exposes slices. + */ + static class TestableDefaultSliceTracker extends DefaultSliceTrackingLazyRead implements TestableSliceTracker { + public TestableDefaultSliceTracker(LazyRead delegate) { + super(delegate); + } + + @Override + public List getSlices() { + return java.util.Collections.unmodifiableList(slices); + } + } + + /** + * Testable wrapper for AggregatingSliceTrackingLazyRead that exposes slices. + */ + static class TestableAggregatingSliceTracker extends AggregatingSliceTrackingLazyRead implements TestableSliceTracker { + public TestableAggregatingSliceTracker(LazyRead delegate) { + super(delegate); + } + + @Override + public List getSlices() { + return java.util.Collections.unmodifiableList(slices); + } + } + + /** + * Interface for testable slice trackers. + */ + interface TestableSliceTracker { + List getSlices(); + } + + static class DummyLazyRead implements LazyRead { + + private ReadData data; + private int numMaterializeCalls = 0; + + public DummyLazyRead( ReadData data ) { + this.data = data; + } + + @Override + public void close() throws IOException { + // no op + } + + @Override + public ReadData materialize(long offset, long length) throws N5IOException { + + numMaterializeCalls++; + return data.slice(offset, length).materialize(); + } + + @Override + public long size() throws N5IOException { + + return data.length(); + } + + public int getNumMaterializeCalls() { + + return numMaterializeCalls; + } + + public void resetNumMaterializeCalls() { + + numMaterializeCalls = 0; + } + + } +} From d79dec380db71581bbaf576f48cf3c216828fcc6 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 9 Feb 2026 17:27:41 -0500 Subject: [PATCH 35/41] wip: toward using prefetching --- .../n5/shard/DefaultDatasetAccess.java | 11 ++++++++++- .../janelia/saalfeldlab/n5/shard/RawShard.java | 17 +++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 630c30033..66e0dd548 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -155,6 +155,15 @@ private void readChunksRecursive( // TODO: collect all the elementPos that we will need and prefetch // Probably best to add a prefetch method to RawShard? + // Here's an attempt at that. + // Don't love that we have to build a list of positions + // Consider making DataBlockRequest package private and passing them directly + final ArrayList positions = new ArrayList<>(); + for (final ChunkRequest request : requests) { + positions.add(request.position.relative(0)); + } + shard.prefetch(positions); + for (final ChunkRequest request : requests) { final long[] elementPos = request.position.relative(0); final ReadData elementData = shard.getElementData(elementPos); @@ -1015,7 +1024,7 @@ public List> chunks(final List> duplicates) { * Construct {@code ChunkRequests} from a list of level-0 grid positions * for reading. *

- * The nesting level ot the returned {@code ChunkRequests} is {@code + * The nesting level of the returned {@code ChunkRequests} is {@code * grid.numLevels()}, that is level of the highest-order shard + 1. This * implies that the requests are not guaranteed to be in the same shard (at * any level. {@link ChunkRequests#split() Splitting} the {@code diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java index 01d7e961b..419ee1ccb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java @@ -28,6 +28,11 @@ */ package org.janelia.saalfeldlab.n5.shard; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.segment.Segment; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; @@ -86,4 +91,16 @@ public void setElementData(final ReadData data, final long[] pos) { final Segment segment = data == null ? null : SegmentedReadData.wrap(data).segments().get(0); index.set(segment, pos); } + + public void prefetch(List positions) { + + final List ranges = new ArrayList<>(positions.size()); + for (long[] pos : positions) { + final Segment seg = index.get(pos); + if (seg != null) + ranges.add(sourceData.location(seg)); + } + sourceData.prefetch(ranges); + } + } From ede576d508ab9c269d0e67f522b005698420089a Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 26 Mar 2026 11:08:11 -0400 Subject: [PATCH 36/41] refactor: migrate previous batching/aggregate work to current development feat: use the aggregate/prefetch logic with VolatileReadData. By default, `VolatileReadData.from` now uses AggregatingSliceTrackingLazyRead. This is useful to ensure the default VolatileReadData behaves reasonably even for reading many chunks from the same shard. --- .../n5/readdata/VolatileReadData.java | 5 +- .../n5/backward/CompatibilityTest.java | 1 - .../n5/benchmarks/ReadDataBenchmarks.java | 3 - .../n5/http/HttpKeyValueAccessTest.java | 3 - .../n5/kva/TrackingKeyValueAccess.java | 24 ++++-- .../readdata/DelegatingVolatileReadData.java | 82 +++++++++++++++++++ .../n5/readdata/ReadDataTests.java | 1 - .../saalfeldlab/n5/shard/ShardTest.java | 36 +++++--- 8 files changed, 128 insertions(+), 27 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/readdata/DelegatingVolatileReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java index 122a399ef..7dba9e324 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java @@ -1,7 +1,7 @@ package org.janelia.saalfeldlab.n5.readdata; -import java.io.InputStream; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.prefetch.AggregatingSliceTrackingLazyRead; /** * During its life-time, the content of a {@code VolatileReadData} should not be @@ -29,7 +29,8 @@ public interface VolatileReadData extends ReadData, AutoCloseable { * @return a new VolatileReadData */ static VolatileReadData from(final LazyRead lazyRead) { - return new LazyReadData(lazyRead); + final LazyRead aggregatingLazyRead = new AggregatingSliceTrackingLazyRead(lazyRead); + return new LazyReadData(aggregatingLazyRead); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java index e5a705422..17d421750 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java @@ -35,7 +35,6 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.file.Files; import java.util.Arrays; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java index 4ce148c50..73b6a2f46 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java @@ -29,8 +29,6 @@ package org.janelia.saalfeldlab.n5.benchmarks; import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -40,7 +38,6 @@ import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java index 8572e1cee..fd151a265 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java @@ -30,17 +30,14 @@ import org.apache.commons.io.IOUtils; import org.janelia.saalfeldlab.n5.HttpKeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.junit.Test; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.charset.Charset; -import java.util.function.Function; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java index 0c4f4a876..a72608363 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java @@ -2,16 +2,18 @@ import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.readdata.DelegatingVolatileReadData; import org.janelia.saalfeldlab.n5.readdata.LazyRead; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; -import org.janelia.saalfeldlab.n5.shard.ShardTest; +import org.janelia.saalfeldlab.n5.readdata.prefetch.AggregatingSliceTrackingLazyRead; public class TrackingKeyValueAccess extends DelegateKeyValueAccess { public int numMaterializeCalls = 0; public int numIsFileCalls = 0; public long totalBytesRead = 0; + public boolean aggregate = false; public TrackingKeyValueAccess(final KeyValueAccess kva) { super(kva); @@ -25,15 +27,27 @@ public boolean isFile(String normalPath) { @Override public VolatileReadData createReadData(final String normalPath) { -// throw new N5NoSuchKeyException("Test No Such Key"); - return VolatileReadData.from(new TrackingVolatileReadData(kva.createReadData(normalPath))); + + final VolatileReadData volatileReadData = kva.createReadData(normalPath); + final TrackingLazyRead trackingLazyRead = new TrackingLazyRead(volatileReadData); + LazyRead lazyRead = trackingLazyRead; + if (aggregate) + lazyRead = new AggregatingSliceTrackingLazyRead(trackingLazyRead); + VolatileReadData delegate = VolatileReadData.from( lazyRead ); + return new DelegatingVolatileReadData(delegate) { + @Override + public void close() throws N5Exception.N5IOException { + super.close(); //closes delegate + volatileReadData.close(); + } + }; } - private class TrackingVolatileReadData implements LazyRead { + private class TrackingLazyRead implements LazyRead { private final VolatileReadData readData; - TrackingVolatileReadData(final VolatileReadData readData) { + TrackingLazyRead(final VolatileReadData readData) { this.readData = readData; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/DelegatingVolatileReadData.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/DelegatingVolatileReadData.java new file mode 100644 index 000000000..d2f633f1c --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/DelegatingVolatileReadData.java @@ -0,0 +1,82 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import org.janelia.saalfeldlab.n5.N5Exception; + +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Collection; + +public class DelegatingVolatileReadData implements VolatileReadData { + + private final VolatileReadData delegate; + + public DelegatingVolatileReadData(VolatileReadData delegate) { + this.delegate = delegate; + } + + @Override + public void close() throws N5Exception.N5IOException { + delegate.close(); + } + + @Override + public long length() { + return delegate.length(); + } + + @Override + public long requireLength() throws N5Exception.N5IOException { + return delegate.requireLength(); + } + + @Override + public ReadData limit(long length) throws N5Exception.N5IOException { + return delegate.limit(length); + } + + @Override + public ReadData slice(long offset, long length) throws N5Exception.N5IOException { + return delegate.slice(offset, length); + } + + @Override + public ReadData slice(Range range) throws N5Exception.N5IOException { + return delegate.slice(range); + } + + @Override + public InputStream inputStream() throws N5Exception.N5IOException, IllegalStateException { + return delegate.inputStream(); + } + + @Override + public byte[] allBytes() throws N5Exception.N5IOException, IllegalStateException { + return delegate.allBytes(); + } + + @Override + public ByteBuffer toByteBuffer() throws N5Exception.N5IOException, IllegalStateException { + return delegate.toByteBuffer(); + } + + @Override + public ReadData materialize() throws N5Exception.N5IOException { + return delegate.materialize(); + } + + @Override + public void writeTo(OutputStream outputStream) throws N5Exception.N5IOException, IllegalStateException { + delegate.writeTo(outputStream); + } + + @Override + public void prefetch(Collection ranges) throws N5Exception.N5IOException { + delegate.prefetch(ranges); + } + + @Override + public ReadData encode(OutputStreamOperator encoder) { + return delegate.encode(encoder); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index c4f3fa51b..78522ff3e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -38,7 +38,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.file.FileSystems; import java.util.Arrays; import java.util.function.IntUnaryOperator; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 7d30db739..36bf19240 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -537,9 +537,8 @@ public void numReadsTest() { new ByteArrayDataBlock(chunkSize, new long[]{11, 11}, data) ); - writer.resetNumMaterializeCalls(); - writer.readChunks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0})); - System.out.println(writer.getNumMaterializeCalls()); + writer.resetNumMaterializeCalls(); + writer.readChunks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0})); ArrayList ptList = new ArrayList<>(); ptList.add(new long[] {0, 0}); @@ -547,10 +546,8 @@ public void numReadsTest() { ptList.add(new long[] {1, 0}); ptList.add(new long[] {1, 1}); - writer.resetNumMaterializeCalls(); - writer.readChunks(dataset, datasetAttributes, ptList); - System.out.println(writer.getNumMaterializeCalls()); - System.out.println(""); + writer.resetNumMaterializeCalls(); + writer.readChunks(dataset, datasetAttributes, ptList); } @Test @@ -656,14 +653,29 @@ public void testPartialReadAggregationBehavior() { writer.resetNumMaterializeCalls(); writer.readChunks(dataset, datasetAttributes, ptList); - // TODO change this if and when we implement aggregation of read calls - // one for the index, one for each of the four blocks - assertEquals(5, writer.getNumMaterializeCalls()); + // one for the index, one for the four blocks (aggregated) + assertEquals(2, writer.getNumMaterializeCalls()); writer.resetNumMaterializeCalls(); writer.readBlock(dataset, datasetAttributes, new long[] {0,0}); - // one for the index, one for each of the four blocks - assertEquals(5, writer.getNumMaterializeCalls()); + // one for the index, one for the four blocks (aggregated) + assertEquals(2, writer.getNumMaterializeCalls()); + + + /** + * Aggregate read calls + */ + writer.tkva.aggregate = true; + writer.resetNumMaterializeCalls(); + writer.readChunks(dataset, datasetAttributes, ptList); + + // one for the index, one that covers ALL the blocks) + assertEquals(2, writer.getNumMaterializeCalls()); + + writer.resetNumMaterializeCalls(); + writer.readBlock(dataset, datasetAttributes, new long[] {0,0}); + // one for the index, one that covers ALL the blocks + assertEquals(2, writer.getNumMaterializeCalls()); } } From 8d963193411ca6ba662e63152c6b77d96e86739e Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 3 Apr 2026 22:45:07 +0200 Subject: [PATCH 37/41] clean up The wrapped volatileReadData is closed via the wrapper chain LazyReadData -> AggregatingSliceTrackingLazyRead -> TrackingLazyRead -> volatileReadData No need for DelegatingVolatileReadData. --- .../n5/kva/TrackingKeyValueAccess.java | 10 +-- .../readdata/DelegatingVolatileReadData.java | 82 ------------------- 2 files changed, 1 insertion(+), 91 deletions(-) delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/readdata/DelegatingVolatileReadData.java diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java index a72608363..3f747b8ec 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java @@ -2,7 +2,6 @@ import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.readdata.DelegatingVolatileReadData; import org.janelia.saalfeldlab.n5.readdata.LazyRead; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; @@ -33,14 +32,7 @@ public VolatileReadData createReadData(final String normalPath) { LazyRead lazyRead = trackingLazyRead; if (aggregate) lazyRead = new AggregatingSliceTrackingLazyRead(trackingLazyRead); - VolatileReadData delegate = VolatileReadData.from( lazyRead ); - return new DelegatingVolatileReadData(delegate) { - @Override - public void close() throws N5Exception.N5IOException { - super.close(); //closes delegate - volatileReadData.close(); - } - }; + return VolatileReadData.from( lazyRead ); } private class TrackingLazyRead implements LazyRead { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/DelegatingVolatileReadData.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/DelegatingVolatileReadData.java deleted file mode 100644 index d2f633f1c..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/DelegatingVolatileReadData.java +++ /dev/null @@ -1,82 +0,0 @@ -package org.janelia.saalfeldlab.n5.readdata; - -import org.janelia.saalfeldlab.n5.N5Exception; - -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Collection; - -public class DelegatingVolatileReadData implements VolatileReadData { - - private final VolatileReadData delegate; - - public DelegatingVolatileReadData(VolatileReadData delegate) { - this.delegate = delegate; - } - - @Override - public void close() throws N5Exception.N5IOException { - delegate.close(); - } - - @Override - public long length() { - return delegate.length(); - } - - @Override - public long requireLength() throws N5Exception.N5IOException { - return delegate.requireLength(); - } - - @Override - public ReadData limit(long length) throws N5Exception.N5IOException { - return delegate.limit(length); - } - - @Override - public ReadData slice(long offset, long length) throws N5Exception.N5IOException { - return delegate.slice(offset, length); - } - - @Override - public ReadData slice(Range range) throws N5Exception.N5IOException { - return delegate.slice(range); - } - - @Override - public InputStream inputStream() throws N5Exception.N5IOException, IllegalStateException { - return delegate.inputStream(); - } - - @Override - public byte[] allBytes() throws N5Exception.N5IOException, IllegalStateException { - return delegate.allBytes(); - } - - @Override - public ByteBuffer toByteBuffer() throws N5Exception.N5IOException, IllegalStateException { - return delegate.toByteBuffer(); - } - - @Override - public ReadData materialize() throws N5Exception.N5IOException { - return delegate.materialize(); - } - - @Override - public void writeTo(OutputStream outputStream) throws N5Exception.N5IOException, IllegalStateException { - delegate.writeTo(outputStream); - } - - @Override - public void prefetch(Collection ranges) throws N5Exception.N5IOException { - delegate.prefetch(ranges); - } - - @Override - public ReadData encode(OutputStreamOperator encoder) { - return delegate.encode(encoder); - } -} From a7ef2f1238fa4da0c95707c058308814d92a0f2a Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 3 Apr 2026 23:02:15 +0200 Subject: [PATCH 38/41] Make SliceTrackingLazyRead non-abstract It doesn't do any prefetching, just keeps track of what has been materialized (and uses that to avoid materializing slices that are already fully covered). --- .../DefaultSliceTrackingLazyRead.java | 5 -- .../prefetch/SliceTrackingLazyRead.java | 46 +++++-------------- 2 files changed, 11 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/DefaultSliceTrackingLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/DefaultSliceTrackingLazyRead.java index 1e607ddc1..f35a8733c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/DefaultSliceTrackingLazyRead.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/DefaultSliceTrackingLazyRead.java @@ -42,9 +42,4 @@ public void prefetch(final Collection ranges) throws N5IOExcept materialize(fromIndex, toIndex - fromIndex); } } - - private boolean isCovered(final Range slice) { - - return Slices.findContainingSlice(slices, slice) != null; - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java index de20f2c39..5835c473e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java @@ -2,14 +2,22 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.LazyRead; import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; -public abstract class SliceTrackingLazyRead implements LazyRead { +/** + * A {@link LazyRead} that wraps a delegate {@code LazyRead} and keeps track of + * all slices that have been {@link #materialize materialized}. + *

+ * When materializing a new slice, we first check whether it is completely + * covered by a materialized slice that we already track. If so, then we just + * return a slice on the existing materialized slice. If not, we materialize the + * slice from the delegate track it. + */ +public class SliceTrackingLazyRead implements LazyRead { protected static class Slice implements Range { @@ -75,39 +83,7 @@ public long size() throws N5IOException { return delegate.size(); } - /** - * Indicates that the given slices will be subsequently read. - * {@code LazyRead} implementations (optionally) may take steps to prepare - * for these subsequent slices. - *

- * Minimal implementation: Find offset and length covering all ranges that - * are not yet fully covered by existing slices. Then materialize the slice - * covering that range. - * - * @param ranges - * slice ranges to prefetch - * - * @throws N5IOException - * if any I/O error occurs - */ - @Override - public void prefetch(final Collection ranges) throws N5IOException { - - long fromIndex = Long.MAX_VALUE; - long toIndex = Long.MIN_VALUE; - for (final Range slice : ranges) { - if (!isCovered(slice)) { - fromIndex = Math.min(fromIndex, slice.offset()); - toIndex = Math.max(toIndex, slice.end()); - } - } - - if (fromIndex < toIndex) { - materialize(fromIndex, toIndex - fromIndex); - } - } - - private boolean isCovered(final Range slice) { + protected boolean isCovered(final Range slice) { return Slices.findContainingSlice(slices, slice) != null; } From ee8612cc839650ff150791d66aafbbb7d277030e Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 3 Apr 2026 23:12:47 +0200 Subject: [PATCH 39/41] rename prefetching LazyReads to "...PrefetchLazyRead" instead of "...SliceTracking..." which is not their main feature --- .../saalfeldlab/n5/readdata/VolatileReadData.java | 4 ++-- ...ingLazyRead.java => AggregatingPrefetchLazyRead.java} | 9 +++++++-- ...ckingLazyRead.java => EnclosingPrefetchLazyRead.java} | 8 ++++++-- .../saalfeldlab/n5/kva/TrackingKeyValueAccess.java | 4 ++-- .../n5/readdata/prefetch/SliceTrackingLazyReadTests.java | 4 ++-- 5 files changed, 19 insertions(+), 10 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/{AggregatingSliceTrackingLazyRead.java => AggregatingPrefetchLazyRead.java} (73%) rename src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/{DefaultSliceTrackingLazyRead.java => EnclosingPrefetchLazyRead.java} (81%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java index 7dba9e324..25da8b0b8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java @@ -1,7 +1,7 @@ package org.janelia.saalfeldlab.n5.readdata; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.readdata.prefetch.AggregatingSliceTrackingLazyRead; +import org.janelia.saalfeldlab.n5.readdata.prefetch.AggregatingPrefetchLazyRead; /** * During its life-time, the content of a {@code VolatileReadData} should not be @@ -29,7 +29,7 @@ public interface VolatileReadData extends ReadData, AutoCloseable { * @return a new VolatileReadData */ static VolatileReadData from(final LazyRead lazyRead) { - final LazyRead aggregatingLazyRead = new AggregatingSliceTrackingLazyRead(lazyRead); + final LazyRead aggregatingLazyRead = new AggregatingPrefetchLazyRead(lazyRead); return new LazyReadData(aggregatingLazyRead); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingSliceTrackingLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java similarity index 73% rename from src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingSliceTrackingLazyRead.java rename to src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java index fc4fb660d..3a5145359 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingSliceTrackingLazyRead.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java @@ -6,9 +6,14 @@ import org.janelia.saalfeldlab.n5.readdata.LazyRead; import org.janelia.saalfeldlab.n5.readdata.Range; -public class AggregatingSliceTrackingLazyRead extends SliceTrackingLazyRead { +/** + * A {@link SliceTrackingLazyRead} that implements {@link #prefetch} to + * aggregate overlapping / adjacent ranges and then materialize each aggregated + * range. + */ +public class AggregatingPrefetchLazyRead extends SliceTrackingLazyRead { - public AggregatingSliceTrackingLazyRead(final LazyRead delegate) { + public AggregatingPrefetchLazyRead(final LazyRead delegate) { super(delegate); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/DefaultSliceTrackingLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/EnclosingPrefetchLazyRead.java similarity index 81% rename from src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/DefaultSliceTrackingLazyRead.java rename to src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/EnclosingPrefetchLazyRead.java index f35a8733c..b0d6fdc7a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/DefaultSliceTrackingLazyRead.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/EnclosingPrefetchLazyRead.java @@ -5,9 +5,13 @@ import org.janelia.saalfeldlab.n5.readdata.LazyRead; import org.janelia.saalfeldlab.n5.readdata.Range; -public class DefaultSliceTrackingLazyRead extends SliceTrackingLazyRead { +/** + * A {@link SliceTrackingLazyRead} that implements {@link #prefetch} to + * materialize the bounding range of all requested ranges. + */ +public class EnclosingPrefetchLazyRead extends SliceTrackingLazyRead { - public DefaultSliceTrackingLazyRead(final LazyRead delegate) { + public EnclosingPrefetchLazyRead(final LazyRead delegate) { super(delegate); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java index 3f747b8ec..a2e18b17d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java @@ -5,7 +5,7 @@ import org.janelia.saalfeldlab.n5.readdata.LazyRead; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; -import org.janelia.saalfeldlab.n5.readdata.prefetch.AggregatingSliceTrackingLazyRead; +import org.janelia.saalfeldlab.n5.readdata.prefetch.AggregatingPrefetchLazyRead; public class TrackingKeyValueAccess extends DelegateKeyValueAccess { @@ -31,7 +31,7 @@ public VolatileReadData createReadData(final String normalPath) { final TrackingLazyRead trackingLazyRead = new TrackingLazyRead(volatileReadData); LazyRead lazyRead = trackingLazyRead; if (aggregate) - lazyRead = new AggregatingSliceTrackingLazyRead(trackingLazyRead); + lazyRead = new AggregatingPrefetchLazyRead(trackingLazyRead); return VolatileReadData.from( lazyRead ); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java index 044e6c1a3..fcac9a5e3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java @@ -160,7 +160,7 @@ private static void assertStoredSlices(TestableSliceTracker sliceTracking, List< /** * Testable wrapper for DefaultSliceTrackingLazyRead that exposes slices. */ - static class TestableDefaultSliceTracker extends DefaultSliceTrackingLazyRead implements TestableSliceTracker { + static class TestableDefaultSliceTracker extends EnclosingPrefetchLazyRead implements TestableSliceTracker { public TestableDefaultSliceTracker(LazyRead delegate) { super(delegate); } @@ -174,7 +174,7 @@ public List getSlices() { /** * Testable wrapper for AggregatingSliceTrackingLazyRead that exposes slices. */ - static class TestableAggregatingSliceTracker extends AggregatingSliceTrackingLazyRead implements TestableSliceTracker { + static class TestableAggregatingSliceTracker extends AggregatingPrefetchLazyRead implements TestableSliceTracker { public TestableAggregatingSliceTracker(LazyRead delegate) { super(delegate); } From a3e707bbd74ca8637928e4d73002006f43efefd2 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 3 Apr 2026 23:18:37 +0200 Subject: [PATCH 40/41] Don't include ranges that are already materialized in the aggregation --- .../n5/readdata/prefetch/AggregatingPrefetchLazyRead.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java index 3a5145359..dcc0097a1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java @@ -1,7 +1,9 @@ package org.janelia.saalfeldlab.n5.readdata.prefetch; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.LazyRead; import org.janelia.saalfeldlab.n5.readdata.Range; @@ -31,7 +33,9 @@ public AggregatingPrefetchLazyRead(final LazyRead delegate) { @Override public void prefetch(final Collection ranges) throws N5IOException { - final Collection aggregatedRanges = Range.aggregate(ranges); + final List filteredRanges = new ArrayList<>(ranges); + filteredRanges.removeIf(this::isCovered); + final Collection aggregatedRanges = Range.aggregate(filteredRanges); for (final Range slice : aggregatedRanges) { materialize(slice.offset(), slice.length()); } From 90ddf5ddc93863738e8ea8095c1886bb757aec73 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 7 Apr 2026 13:55:42 -0400 Subject: [PATCH 41/41] fix: setDatasetAttributes no long overwrites user attributes * add a test --- .../java/org/janelia/saalfeldlab/n5/N5Writer.java | 2 +- .../janelia/saalfeldlab/n5/AbstractN5Test.java | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 6c4f35659..d0faf18f8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -142,7 +142,7 @@ default void setDatasetAttributes( final String datasetPath, final DatasetAttributes datasetAttributes) throws N5Exception { - setAttribute(datasetPath, "/", getConvertedDatasetAttributes(datasetAttributes)); + setAttributes(datasetPath, getConvertedDatasetAttributes(datasetAttributes).asMap()); } /** diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index bc3510ddd..c240cac43 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -771,6 +771,21 @@ public void testAttributes() { } } + @Test + public void testDatasetAttributes() { + + final String dset = ""; + final String key = "user-attr"; + final String value = "value"; + try (final N5Writer n5 = createTempN5Writer()) { + + n5.setAttribute(dset, key, value); + n5.setDatasetAttributes(dset, new DatasetAttributes(new long[]{5}, new int[]{5}, DataType.INT32)); + assertNotNull(n5.getDatasetAttributes(dset)); + assertEquals(value, n5.getAttribute(dset, key, String.class)); + } + } + @Test public void testNullAttributes() throws URISyntaxException, IOException {