diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index af25c729d..71a62298c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -89,13 +89,13 @@ public class DatasetAttributes implements Serializable { private final long[] dimensions; + // number of samples per chunk per dimension + private final int[] chunkSize; + // number of samples per block per dimension + // identical to chunkSize for non-sharded datasets private final int[] blockSize; - // TODO add a getter? - // the shard size - private final int[] outerBlockSize; - private final DataType dataType; private final JsonElement defaultValue; @@ -108,7 +108,7 @@ public class DatasetAttributes implements Serializable { public DatasetAttributes( final long[] dimensions, - final int[] outerBlockSize, + final int[] blockSize, final DataType dataType, final JsonElement defaultValue, final BlockCodecInfo blockCodecInfo, @@ -117,7 +117,7 @@ public DatasetAttributes( this.dimensions = dimensions; this.dataType = dataType; - this.outerBlockSize = outerBlockSize; + this.blockSize = blockSize; this.defaultValue = defaultValue == null ? JsonNull.INSTANCE : defaultValue; this.blockCodecInfo = blockCodecInfo == null ? defaultBlockCodecInfo() : blockCodecInfo; @@ -131,7 +131,7 @@ public DatasetAttributes( .toArray(DataCodecInfo[]::new); access = createDatasetAccess(); - blockSize = access.getGrid().getBlockSize(0); + chunkSize = access.getGrid().getBlockSize(0); } public DatasetAttributes( @@ -200,7 +200,7 @@ protected DatasetAccess createDatasetAccess() { // NestedGrid validates block sizes, so instantiate it before creating the blockCodecs // blockCodecInfo.create below could fail unexpecedly with invalid // blockSizes so validate first - blockSizes[m - 1] = outerBlockSize; + blockSizes[m - 1] = blockSize; BlockCodecInfo tmpInfo = blockCodecInfo; for (int l = m - 1; l > 0; --l) { final ShardCodecInfo info = (ShardCodecInfo)tmpInfo; @@ -274,6 +274,11 @@ public int getNumDimensions() { return dimensions.length; } + public int[] getChunkSize() { + + return chunkSize; + } + public int[] getBlockSize() { return blockSize; @@ -334,7 +339,6 @@ public NestedGrid getNestedBlockGrid() { return getDatasetAccess().getGrid(); } - public BlockCodecInfo getBlockCodecInfo() { return blockCodecInfo; @@ -359,7 +363,7 @@ public HashMap asMap() { final HashMap map = new HashMap<>(); map.put(DIMENSIONS_KEY, dimensions); - map.put(BLOCK_SIZE_KEY, blockSize); + map.put(BLOCK_SIZE_KEY, chunkSize); map.put(DATA_TYPE_KEY, dataType); map.put(COMPRESSION_KEY, getCompression()); return map; @@ -425,7 +429,7 @@ public static class DatasetAttributesAdapter implements JsonSerializer DataBlock readBlock( + default DataBlock readChunk( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { @@ -103,7 +103,7 @@ default DataBlock readBlock( try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), convertedDatasetAttributes); - return convertedDatasetAttributes. getDatasetAccess().readBlock(posKva, gridPosition); + return convertedDatasetAttributes. getDatasetAccess().readChunk(posKva, gridPosition); } catch (N5Exception.N5NoSuchKeyException e) { return null; @@ -111,18 +111,18 @@ default DataBlock readBlock( } @Override - default List> readBlocks( + default List> readChunks( final String pathName, final DatasetAttributes datasetAttributes, final List blockPositions) throws N5Exception { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), convertedDatasetAttributes); - return convertedDatasetAttributes. getDatasetAccess().readBlocks(posKva, blockPositions); + return convertedDatasetAttributes. getDatasetAccess().readChunks(posKva, blockPositions); } @Override - default DataBlock readShard( + default DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { @@ -132,7 +132,7 @@ default DataBlock readShard( try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), convertedDatasetAttributes); - return convertedDatasetAttributes. getDatasetAccess().readShard(posKva, gridPosition, shardLevel); + return convertedDatasetAttributes. getDatasetAccess().readBlock(posKva, gridPosition, shardLevel); } catch (N5Exception.N5NoSuchKeyException e) { return null; @@ -172,7 +172,7 @@ default String absoluteAttributesPath(final String normalPath) { } @Override - default boolean shardExists( + default boolean blockExists( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index f5bacc94f..42386d368 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -257,7 +257,7 @@ default void writeRegion( } @Override - default void writeBlocks( + default void writeChunks( final String datasetPath, final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { @@ -265,15 +265,15 @@ default void writeBlocks( DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes); - convertedDatasetAttributes.getDatasetAccess().writeBlocks(posKva, Arrays.asList(dataBlocks)); + convertedDatasetAttributes.getDatasetAccess().writeChunks(posKva, Arrays.asList(dataBlocks)); } catch (final UncheckedIOException e) { throw new N5IOException( - "Failed to write blocks into dataset " + datasetPath, e); + "Failed to write chunks into dataset " + datasetPath, e); } } @Override - default void writeBlock( + default void writeChunk( final String path, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception { @@ -281,28 +281,28 @@ default void writeBlock( DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes); - convertedDatasetAttributes. getDatasetAccess().writeBlock(posKva, dataBlock); + convertedDatasetAttributes. getDatasetAccess().writeChunk(posKva, dataBlock); } catch (final UncheckedIOException e) { throw new N5IOException( - "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, + "Failed to write chunk " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, e); } } @Override - default void writeShard( + default void writeBlock( final String path, final DatasetAttributes datasetAttributes, - final DataBlock shard) throws N5Exception { + final DataBlock dataBlock) throws N5Exception { final DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); final int shardLevel = convertedDatasetAttributes.getNestedBlockGrid().numLevels() - 1; try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes); - convertedDatasetAttributes. getDatasetAccess().writeShard(posKva, shard, shardLevel); + convertedDatasetAttributes. getDatasetAccess().writeBlock(posKva, dataBlock, shardLevel); } catch (final UncheckedIOException e) { throw new N5IOException( - "Failed to write block " + Arrays.toString(shard.getGridPosition()) + " into dataset " + path, + "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, e); } } @@ -326,16 +326,26 @@ default boolean deleteBlock( final long... gridPosition) throws N5Exception { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); - return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition); + return posKva.remove(gridPosition); + } + + @Override + default boolean deleteChunk( + final String path, + final DatasetAttributes datasetAttributes, + final long... gridPosition) throws N5Exception { + + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); + return datasetAttributes.getDatasetAccess().deleteChunk(posKva, gridPosition); } @Override - default boolean deleteBlocks( + default boolean deleteChunks( final String path, final DatasetAttributes datasetAttributes, final List gridPositions) throws N5Exception { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); - return datasetAttributes.getDatasetAccess().deleteBlocks(posKva, gridPositions); + return datasetAttributes.getDatasetAccess().deleteChunks(posKva, gridPositions); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index aebbfa67d..c5103e140 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -298,7 +298,7 @@ default DatasetAttributes getConvertedDatasetAttributes(final DatasetAttributes /** - * Reads a {@link DataBlock}. + * Reads a chunk as a {@link DataBlock}. * * @param * the DataBlock data type @@ -312,13 +312,13 @@ default DatasetAttributes getConvertedDatasetAttributes(final DatasetAttributes * @throws N5Exception * the exception */ - DataBlock readBlock( + DataBlock readChunk( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception; /** - * Reads multiple {@link DataBlock}s. + * Reads multiple chunks as {@link DataBlock}s. *

* Implementations may optimize / batch read operations when possible, e.g. * in the case that the datasets are sharded. @@ -335,7 +335,7 @@ DataBlock readBlock( * @throws N5Exception * the exception */ - default List> readBlocks( + default List> readChunks( final String pathName, final DatasetAttributes datasetAttributes, final List gridPositions) throws N5Exception { @@ -343,16 +343,16 @@ default List> readBlocks( DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); final ArrayList> blocks = new ArrayList<>(); for( final long[] p : gridPositions ) - blocks.add(readBlock(pathName, convertedDatasetAttributes, p)); + blocks.add(readChunk(pathName, convertedDatasetAttributes, p)); return blocks; } /** - * Reads a shard as a {@link DataBlock}. + * Reads a block, returning a {@link DataBlock}. Will be a chunk or shard (if the dataset is sharded). *

- * A "shard" is the largest level of the datasets {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid} - * This method's behavior is identical to readBlock for un-sharded datasets. + * A block is the highest (coarsest) level of the dataset's {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid} + * This method's behavior is identical to {@link #readChunk} for un-sharded datasets. * * @param * the DataBlock data type @@ -361,39 +361,40 @@ default List> readBlocks( * @param datasetAttributes * the dataset attributes * @param gridPosition - * the position in the shard grid + * the position in the block grid * @return the data block * @throws N5Exception * the exception * * @see DatasetAttributes#getNestedBlockGrid() */ - DataBlock readShard( + DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception; /** - * Checks if a shard (or block for non-sharded datasets) exists at the - * given grid position without reading the data. + * Checks if a block exists at the given grid position without reading the data. *

* This method only checks for the presence of the key value for the gridPosition, it does not - * read or validate the contents. + * read or validate the contents. As a result, this method refers to chunks in un-sharded datasets + * or shards in sharded datasets. * * @param pathName * dataset path * @param datasetAttributes * the dataset attributes * @param gridPosition - * the shard grid position (or block position for non-sharded datasets) - * @return true if the shard (or block) file exists + * the block grid position + * @return true if the block file exists * @throws N5Exception * the exception */ - boolean shardExists( + boolean blockExists( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception; + /** * Load a {@link DataBlock} as a {@link Serializable}. The offset is given * in @@ -419,7 +420,7 @@ default T readSerializedBlock( final long... gridPosition) throws N5Exception, ClassNotFoundException { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(attributes); - final DataBlock block = readBlock(dataset, convertedDatasetAttributes, gridPosition); + final DataBlock block = readChunk(dataset, convertedDatasetAttributes, gridPosition); if (block == null) return null; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 22ab44667..13de30ccc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -241,29 +241,29 @@ default DatasetAttributes createDataset( } /** - * Writes a {@link DataBlock}. + * Writes a chunk represented by a {@link DataBlock}. * * @param datasetPath dataset path * @param datasetAttributes the dataset attributes - * @param dataBlock the data block + * @param dataBlock the chunk as a DataBlock * @param the data block data type * @throws N5Exception the exception */ - void writeBlock( + void writeChunk( final String datasetPath, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception; /** - * Write multiple data blocks, useful for request aggregation. + * Write multiple chunks represented by {@link DataBlock}s, useful for aggregation. * * @param datasetPath dataset path * @param datasetAttributes the dataset attributes - * @param dataBlocks the data block + * @param dataBlocks the chunks * @param the data block data type * @throws N5Exception the exception */ - default void writeBlocks( + default void writeChunks( final String datasetPath, final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { @@ -271,15 +271,15 @@ default void writeBlocks( // default method is naive DatasetAttributes convertedAttributes = getConvertedDatasetAttributes(datasetAttributes); for (DataBlock block : dataBlocks) { - writeBlock(datasetPath, convertedAttributes, block); + writeChunk(datasetPath, convertedAttributes, block); } } /** - * Writes a shard stored as a {@link DataBlock}. + * Writes a block stored as a {@link DataBlock}. *

- * A "shard" is the largest level of the datasets {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid}. - * This method's behavior is identical to writeBlock for un-sharded datasets. + * A block is the highest (coarsest) level of the dataset's {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid}. + * This method's behavior is identical to {@link #writeChunk} for un-sharded datasets. * * @param pathName dataset path * @param datasetAttributes the dataset attributes @@ -289,7 +289,7 @@ default void writeBlocks( * * @see DatasetAttributes#getNestedBlockGrid() */ - void writeShard( + void writeBlock( final String pathName, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception; @@ -301,7 +301,7 @@ interface DataBlockSupplier { * * @param gridPos * @param existingDataBlock - * existing data to be merged into the new data block (maybe {@code null}) + * existing data to be merged into the new data block (may be {@code null}) * * @return data block at the given gridPos */ @@ -313,17 +313,37 @@ interface DataBlockSupplier { * @param datasetAttributes the dataset attributes * @param min min pixel coordinate of region to write * @param size size in pixels of region to write - * @param dataBlocks is asked to create blocks within the given region + * @param dataBlocks is asked to create chunks within the given region * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. * @throws N5Exception the exception */ - void writeRegion( - final String datasetPath, - final DatasetAttributes datasetAttributes, - final long[] min, - final long[] size, - final DataBlockSupplier dataBlocks, - final boolean writeFully) throws N5Exception; + default void writeRegion( + String datasetPath, + DatasetAttributes datasetAttributes, + long[] min, + long[] size, + DataBlockSupplier dataBlocks, + boolean writeFully) throws N5Exception { + + final NestedGrid grid = datasetAttributes.getNestedBlockGrid(); + final Region region = new Region(min, size, grid); + for (long[] key : Region.gridPositions(region.minPos().key(), region.maxPos().key())) { + final NestedPosition pos = grid.nestedPosition(key, grid.numLevels() - 1); + final long[] gridPosition = pos.absolute(0); + final DataBlock existingDataBlock = writeFully || region.fullyContains(pos) + ? null + : readChunk(datasetPath, datasetAttributes, gridPosition); + final DataBlock dataBlock = dataBlocks.get(gridPosition, existingDataBlock); + // null blocks may be provided when they contain only the fill value + // and only non-empty blocks should be written, for example + if (dataBlock == null) { + deleteChunk(datasetPath, datasetAttributes, gridPosition); + } else { + writeChunk(datasetPath, datasetAttributes, dataBlock); + } + } + + } /** * @param datasetPath the dataset path @@ -332,18 +352,39 @@ void writeRegion( * @param size size in pixels of region to write * @param dataBlocks is asked to create blocks within the given region * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. - * @param exec used to parallelize over blocks and shards + * @param exec used to parallelize over blocks (chunks and shards) * @throws N5Exception the exception */ - void writeRegion( + default void writeRegion( String datasetPath, DatasetAttributes datasetAttributes, long[] min, long[] size, DataBlockSupplier dataBlocks, boolean writeFully, - ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException; - + ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException { + + final NestedGrid grid = datasetAttributes.getNestedBlockGrid(); + final Region region = new Region(min, size, grid); + for (long[] key : Region.gridPositions(region.minPos().key(), region.maxPos().key())) { + exec.submit(() -> { + final NestedPosition pos = grid.nestedPosition(key, grid.numLevels() - 1); + final long[] gridPosition = pos.absolute(0); + final DataBlock existingDataBlock = writeFully || region.fullyContains(pos) + ? null + : readChunk(datasetPath, datasetAttributes, gridPosition); + final DataBlock dataBlock = dataBlocks.get(gridPosition, existingDataBlock); + // null blocks may be provided when they contain only the fill value + // and only non-empty blocks should be written, for example + if (dataBlock == null) { + deleteChunk(datasetPath, datasetAttributes, gridPosition); + } else { + writeChunk(datasetPath, datasetAttributes, dataBlock); + } + }); + } + } + /** * Deletes the block at {@code gridPosition}. * @@ -376,20 +417,51 @@ boolean deleteBlock( long... gridPosition) throws N5Exception; /** - * Deletes the blocks at the given {@code gridPositions}. + * Deletes the chunk at {@code gridPosition}. + * + * @param datasetPath dataset path + * @param gridPosition position of chunk to be deleted + * @throws N5Exception if the chunk exists but could not be deleted + * + * @return {@code true} if the chunk at {@code gridPosition} existed and was deleted. + */ + default boolean deleteChunk( + final String datasetPath, + final long... gridPosition) throws N5Exception { + final DatasetAttributes datasetAttributes = getDatasetAttributes(datasetPath); + return deleteChunk(datasetPath, datasetAttributes, gridPosition); + } + + /** + * Deletes the chunk at {@code gridPosition}. + * + * @param datasetPath the dataset path + * @param datasetAttributes the dataset attributes + * @param gridPosition position of chunk to be deleted + * @throws N5Exception if the chunk exists but could not be deleted + * + * @return {@code true} if the chunk at {@code gridPosition} existed and was deleted. + */ + boolean deleteChunk( + String datasetPath, + DatasetAttributes datasetAttributes, + long... gridPosition) throws N5Exception; + + /** + * Deletes the chunks at the given {@code gridPositions}. * * @param datasetPath dataset path * @param gridPositions a list of grid positions - * @return {@code true} if any of the specified blocks existed and was deleted - * @throws N5Exception if any of the block exists but could not be deleted + * @return {@code true} if any of the specified chunks existed and was deleted + * @throws N5Exception if any of the chunks did exist but could not be deleted */ - default boolean deleteBlocks( + default boolean deleteChunks( String datasetPath, DatasetAttributes datasetAttributes, List gridPositions) throws N5Exception { boolean deleted = false; for (long[] pos : gridPositions) { - deleted |= deleteBlock(datasetPath, datasetAttributes, pos); + deleted |= deleteChunk(datasetPath, datasetAttributes, pos); } return deleted; } @@ -419,6 +491,6 @@ default void writeSerializedBlock( } final byte[] bytes = byteOutputStream.toByteArray(); final DataBlock dataBlock = new ByteArrayDataBlock(null, gridPosition, bytes); - writeBlock(datasetPath, datasetAttributes, dataBlock); + writeChunk(datasetPath, datasetAttributes, dataBlock); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index b09df6782..f6d158261 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -38,7 +38,7 @@ import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; /** - * Wrap an instantiated DataBlock/shard codec hierarchy to implement (single and + * Wrap an instantiated DataBlock/block codec hierarchy to implement (single and * batch) DataBlock read/write methods. * * @param @@ -63,13 +63,13 @@ public interface DatasetAccess { * @throws N5IOException * if any error occurs while reading or decoding the block */ - DataBlock readBlock(PositionValueAccess pva, long[] gridPosition) throws N5IOException; + DataBlock readChunk(PositionValueAccess pva, long[] gridPosition) throws N5IOException; /** * Read the {@code DataBlock}s at the given {@code gridPositions}. *

* The returned {@code List>} is in the same order as the - * requested {@code gridPositions}. That is, the {@code DataBlock} at indwex + * requested {@code gridPositions}. That is, the {@code DataBlock} at index * {@code i} has grid coordinates {@code gridPositions.get(i)}. *

* If a requested block doesn't exist, then the corresponding element in the @@ -80,21 +80,36 @@ public interface DatasetAccess { * @param pva * dataset storage * @param gridPositions - * list of grid position of the DataBlocks to read + * list of grid positions of the DataBlocks to read * @return list of DataBlocks * * @throws N5IOException * if any error occurs while reading or decoding blocks */ - List> readBlocks(PositionValueAccess pva, List gridPositions) throws N5IOException; + List> readChunks(PositionValueAccess pva, List gridPositions) throws N5IOException; - void writeBlock(PositionValueAccess pva, DataBlock dataBlock) throws N5IOException; + /** + * Writes a chunk to the given storage position. + * + */ + void writeChunk(PositionValueAccess pva, DataBlock dataBlock) throws N5IOException; - void writeBlocks(PositionValueAccess pva, List> blocks) throws N5IOException; + /** + * Writes multiple chunks to the given storage positions. + */ + void writeChunks(PositionValueAccess pva, List> blocks) throws N5IOException; - boolean deleteBlock(PositionValueAccess pva, long[] gridPosition) throws N5IOException; + /** + * Deletes the chunk at {@code gridPosition}. @return true if it existed and + * was deleted. + */ + boolean deleteChunk(PositionValueAccess pva, long[] gridPosition) throws N5IOException; - boolean deleteBlocks(PositionValueAccess pva, List positions) throws N5IOException; + /** + * Deletes the chunks at the given {@code positions}. @return true if any + * existed and were deleted. + */ + boolean deleteChunks(PositionValueAccess pva, List positions) throws N5IOException; /** * @@ -106,7 +121,7 @@ public interface DatasetAccess { * @param blocks * is asked to create blocks within the given region * @param writeFully - * if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * if false, merge existing data in blocks/chunks that overlap the region boundary. if true, override everything. * * @throws N5IOException */ @@ -128,9 +143,9 @@ void writeRegion( * @param blocks * is asked to create blocks within the given region. must be thread-safe. * @param writeFully - * if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * if false, merge existing data in blocks/chunks that overlap the region boundary. if true, override everything. * @param exec - * used to parallelize over blocks and shards + * used to parallelize over chunks and blocks * * @throws N5Exception * @throws InterruptedException @@ -146,7 +161,7 @@ void writeRegion( ) throws N5Exception, InterruptedException, ExecutionException; /** - * Read a full shard at {@code shardGridPosition} at the given nesting + * Read a block at {@code shardGridPosition} at the given nesting * {@code level}. The shard data is rearranged and assembled into a (large) * {@code DataBlock}. *

@@ -163,27 +178,27 @@ void writeRegion( * @return * @throws N5IOException */ - DataBlock readShard(PositionValueAccess pva, long[] shardGridPosition, int level) throws N5IOException; + DataBlock readBlock(PositionValueAccess pva, long[] shardGridPosition, int level) throws N5IOException; /** - * Write a full shard the given nesting {@code level}. The shard data is + * Write a full block at the given nesting {@code level}. The block data is * given as a (large) {@code DataBlock} that will be sliced, rearranged, and * written as (level-0) DataBlocks. *

* {@code dataBlock.getGridPosition()} is the grid position with respect to * {@code level}. For example, if {@code level==1}, then this refers to the - * position on the shard grid. + * position on the block grid. * * @param pva * dataset storage * @param dataBlock - * shard/block to write + * block to write * @param level - * grid level of the shard/block to write. + * grid level of the block to write. * * @throws N5IOException */ - void writeShard(PositionValueAccess pva, DataBlock dataBlock, int level) throws N5IOException; + void writeBlock(PositionValueAccess pva, DataBlock dataBlock, int level) throws N5IOException; NestedGrid getGrid(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index fc1db3609..3c652a353 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -71,20 +71,17 @@ public NestedGrid getGrid() { return grid; } - // - // -- readBlock ----------------------------------------------------------- - @Override - public DataBlock readBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { + public DataBlock readChunk(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { final NestedPosition position = grid.nestedPosition(gridPosition); try (final VolatileReadData readData = pva.get(position.key())) { - return readBlockRecursive(readData, position, grid.numLevels() - 1); + return readChunkRecursive(readData, position, grid.numLevels() - 1); } catch (N5NoSuchKeyException ignored) { return null; } } - private DataBlock readBlockRecursive( + private DataBlock readChunkRecursive( final ReadData readData, final NestedPosition position, final int level) { @@ -98,19 +95,16 @@ private DataBlock readBlockRecursive( @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[level]; final RawShard shard = codec.decode(readData, position.absolute(level)).getData(); - return readBlockRecursive(shard.getElementData(position.relative(level - 1)), position, level - 1); + return readChunkRecursive(shard.getElementData(position.relative(level - 1)), position, level - 1); } } - // - // -- readBlocks ---------------------------------------------------------- - @Override - public List> readBlocks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { + public List> readChunks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { // for non-sharded datasets, just read the blocks individually if (grid.numLevels() == 1) { - return gridPositions.stream().map(pos -> readBlock(pva, pos)).collect(Collectors.toList()); + return gridPositions.stream().map(pos -> readChunk(pva, pos)).collect(Collectors.toList()); } // Create a list of DataBlockRequests and sort it such that requests @@ -122,7 +116,7 @@ public List> readBlocks(final PositionValueAccess pva, final List subRequests : split) { final long[] key = subRequests.relativeGridPosition(); try (final VolatileReadData readData = pva.get(key)) { - readBlocksRecursive(readData, subRequests); + readChunksRecursive(readData, subRequests); } catch (N5NoSuchKeyException ignored) { // the key didn't exist (as we found out when lazy-reading the index). // we don't have to do anything: all subRequest blocks remain null. @@ -139,7 +133,7 @@ public List> readBlocks(final PositionValueAccess pva, final List requests ) { @@ -164,23 +158,20 @@ private void readBlocksRecursive( for (final DataBlockRequest request : requests) { final long[] elementPos = request.position.relative(0); final ReadData elementData = shard.getElementData(elementPos); - request.block = readBlockRecursive(elementData, request.position, 0); + request.block = readChunkRecursive(elementData, request.position, 0); } } else { // level > 1 final List> split = requests.split(); for (final DataBlockRequests subRequests : split) { final long[] subShardPosition = subRequests.relativeGridPosition(); final ReadData elementData = shard.getElementData(subShardPosition); - readBlocksRecursive(elementData, subRequests); + readChunksRecursive(elementData, subRequests); } } } - // - // -- writeBlock ---------------------------------------------------------- - @Override - public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException { + public void writeChunk(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException { if (grid.numLevels() == 1) { @SuppressWarnings("unchecked") @@ -224,11 +215,8 @@ private ReadData writeBlockRecursive( } } - // - // -- writeBlocks --------------------------------------------------------- - @Override - public void writeBlocks(final PositionValueAccess pva, final List> dataBlocks) throws N5IOException { + public void writeChunks(final PositionValueAccess pva, final List> dataBlocks) throws N5IOException { if (grid.numLevels() == 1) { @SuppressWarnings("unchecked") @@ -245,7 +233,7 @@ public void writeBlocks(final PositionValueAccess pva, final List> final long[] shardKey = subRequests.relativeGridPosition(); final ReadData modifiedData; try (final VolatileReadData existingData = writeFully ? null : pva.get(shardKey)) { - modifiedData = writeBlocksRecursive(existingData, subRequests); + modifiedData = writeChunksRecursive(existingData, subRequests); // Here, we are about to write the shard data, but with the new blocks modified. // Need to make sure that the read operations happen now before pva.set acquires a write lock modifiedData.materialize(); @@ -261,7 +249,7 @@ public void writeBlocks(final PositionValueAccess pva, final List> * @param existingReadData encoded existing shard data (to decode and partially override) * @param requests for blocks within the shard to be written */ - private ReadData writeBlocksRecursive( + private ReadData writeChunksRecursive( final ReadData existingReadData, // may be null final DataBlockRequests requests ) { @@ -289,7 +277,7 @@ private ReadData writeBlocksRecursive( final boolean nestedWriteFully = writeFully || subRequests.coversShard(); final long[] elementPos = subRequests.relativeGridPosition(); final ReadData existingElementData = nestedWriteFully ? null : shard.getElementData(elementPos); - final ReadData modifiedElementData = writeBlocksRecursive(existingElementData, subRequests); + final ReadData modifiedElementData = writeChunksRecursive(existingElementData, subRequests); shard.setElementData(modifiedElementData, elementPos); } } @@ -297,9 +285,6 @@ private ReadData writeBlocksRecursive( return codec.encode(new RawShardDataBlock(gridPos, shard)); } - // - // -- writeRegion --------------------------------------------------------- - @Override public void writeRegion( final PositionValueAccess pva, @@ -416,7 +401,7 @@ private ReadData writeRegionRecursive( // -- deleteBlock --------------------------------------------------------- @Override - public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { + public boolean deleteChunk(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { if (grid.numLevels() == 1) { // for non-sharded dataset, don't bother getting the value, just remove the key. return pva.remove(gridPosition); @@ -425,7 +410,7 @@ public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosit final long[] key = position.key(); final ReadData modifiedData; try (final VolatileReadData existingData = pva.get(key)) { - modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); + modifiedData = deleteChunkRecursive(existingData, position, grid.numLevels() - 1); if (modifiedData == existingData) { // nothing changed, the blocks we wanted to delete didn't exist anyway return false; @@ -448,7 +433,7 @@ public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosit } } - private ReadData deleteBlockRecursive( + private ReadData deleteChunkRecursive( final ReadData existingReadData, final NestedPosition position, final int level) throws N5NoSuchKeyException { @@ -466,7 +451,7 @@ private ReadData deleteBlockRecursive( // This shard remains unchanged. return existingReadData; } else { - final ReadData modifiedElementData = deleteBlockRecursive(existingElementData, position, level - 1); + final ReadData modifiedElementData = deleteChunkRecursive(existingElementData, position, level - 1); if (modifiedElementData == existingElementData) { // The nested shard was not modified. // This shard remains unchanged. @@ -490,7 +475,7 @@ private ReadData deleteBlockRecursive( // -- deleteBlocks -------------------------------------------------------- @Override - public boolean deleteBlocks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { + public boolean deleteChunks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { // for non-sharded datasets, just delete the blocks individually if (grid.numLevels() == 1) { @@ -652,14 +637,14 @@ private ReadData deleteBlocksRecursive( // easily revisit and change this heuristic. @Override - public DataBlock readShard( + public DataBlock readBlock( final PositionValueAccess pva, final long[] shardGridPosition, final int level ) throws N5IOException { if (level == 0) { - return readBlock(pva, shardGridPosition); + return readChunk(pva, shardGridPosition); } final long[] shardPixelPos = grid.pixelPosition(shardGridPosition, level); @@ -671,10 +656,10 @@ public DataBlock readShard( for (int d = 0; d < n; ++d) { shardSizeInPixels[d] = Math.min(defaultShardSize[d], (int) (datasetSize[d] - shardPixelPos[d])); } - return readShardInternal(pva, shardGridPosition, shardSizeInPixels, level); + return readBlockInternal(pva, shardGridPosition, shardSizeInPixels, level); } - private DataBlock readShardInternal( + private DataBlock readBlockInternal( final PositionValueAccess pva, final long[] shardGridPosition, final int[] shardSizeInPixels, // expected size of this shard in pixels @@ -705,7 +690,7 @@ private DataBlock readShardInternal( // read all blocks in (gridMin, gridMax) and filter out missing blocks final List blockPositions = Region.gridPositions(gridMin, gridMax); - final List> blocks = readBlocks(pva, blockPositions) + final List> blocks = readChunks(pva, blockPositions) .stream().filter(Objects::nonNull).collect(Collectors.toList()); if (blocks.isEmpty()) { return null; @@ -735,14 +720,14 @@ private DataBlock readShardInternal( // -- writeShard ---------------------------------------------------------- @Override - public void writeShard( + public void writeBlock( final PositionValueAccess pva, final DataBlock dataBlock, final int level ) throws N5IOException { if (level == 0) { - writeBlock(pva, dataBlock); + writeChunk(pva, dataBlock); return; } @@ -792,7 +777,7 @@ public void writeShard( blocks.add(block); } - writeBlocks(pva, blocks); + writeChunks(pva, blocks); } // diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index dc12d3943..bc3510ddd 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -74,6 +74,9 @@ /** * Abstract base class for testing N5 functionality. * Subclasses are expected to provide a specific N5 implementation to be tested by defining the {@link #createN5Writer()} method. + *

+ * This class does not create sharded datasets. Its tests generally call read/writeBlock which are equivalent to read/writeChunk + * for the cases being tested here. The test {@link #testReadChunkVsBlock} checks that the equivalence between these methods holds. * * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> * @author Igor Pisarev <pisarevi@janelia.hhmi.org> @@ -162,7 +165,7 @@ protected N5Writer createN5Writer(final String location) throws IOException, URI return createN5Writer(location, new GsonBuilder()); } - /* Tests that overide this should enusre that the `N5Writer` created will remove its container on close() */ + /* Tests that override this should ensure that the `N5Writer` created will remove its container on close() */ protected abstract N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException, URISyntaxException; protected N5Reader createN5Reader(final String location) throws IOException, URISyntaxException { @@ -243,6 +246,7 @@ public void testCreateDataset() { } assertArrayEquals(dimensions, info.getDimensions()); assertArrayEquals(blockSize, info.getBlockSize()); + assertArrayEquals("blockSize == chunkSize when not sharded", blockSize, info.getChunkSize()); assertEquals(DataType.UINT64, info.getDataType()); } @@ -254,8 +258,8 @@ public void testBlocksLargerThanDimensions() { final int[] largeBlockSize = new int[]{5, 7, 10}; try (final N5Writer n5 = createTempN5Writer()) { - n5.createDataset(datasetName, smallDimensions, largeBlockSize, DataType.UINT8, new RawCompression()); - final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + final DatasetAttributes attributes = n5.createDataset( + datasetName, smallDimensions, largeBlockSize, DataType.UINT8, new RawCompression()); // Create a block that is larger than the dataset dimensions final int numElements = largeBlockSize[0] * largeBlockSize[1] * largeBlockSize[2]; @@ -330,8 +334,6 @@ public void testUnalignedBlocksTruncatedAtEnd() { } } - - @Test public void testWriteReadByteBlock() { @@ -475,32 +477,35 @@ public void testWriteReadDoubleBlock() { } @Test - public void testWriteReadShard() { - - // test that writeShard behaves the same as writeBlock - // for unsharded datasets + public void testReadChunkVsBlock() { + // test that readBlock behaves the same as readChunk for unsharded datasets for (final Compression compression : getCompressions()) { try (final N5Writer n5 = createTempN5Writer()) { + final short[] shortData1 = new short[shortBlock.length]; + for( int i = 0; i < shortBlock.length; i++) + shortData1[i] = (short)(2 * shortBlock[i] + 3); + n5.createDataset(datasetName, dimensions, blockSize, DataType.INT16, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); - final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock); - - n5.writeShard(datasetName, attributes, dataBlock); + final ShortArrayDataBlock dataBlock0 = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock); + final ShortArrayDataBlock dataBlock1 = new ShortArrayDataBlock(blockSize, new long[]{1, 0, 0}, shortData1); - // read with readShard - final DataBlock loadedShard = n5.readShard(datasetName, attributes, 0, 0, 0); - assertArrayEquals(shortBlock, (short[])loadedShard.getData()); + n5.writeChunk(datasetName, attributes, dataBlock0); + n5.writeBlock(datasetName, attributes, dataBlock1); // read with readBlock - final DataBlock loadedDataBlock = n5.readShard(datasetName, attributes, 0, 0, 0); - assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData()); + assertArrayEquals(shortBlock, (short[])n5.readBlock(datasetName, attributes, 0, 0, 0).getData()); + assertArrayEquals(shortData1, (short[])n5.readBlock(datasetName, attributes, 1, 0, 0).getData()); + + // read with readChunk + assertArrayEquals(shortBlock, (short[])n5.readChunk(datasetName, attributes, 0, 0, 0).getData()); + assertArrayEquals(shortData1, (short[])n5.readChunk(datasetName, attributes, 1, 0, 0).getData()); } } } - @Test public void testMode1WriteReadByteBlock() { @@ -1302,15 +1307,22 @@ public void testDelete() { n5.writeBlock(datasetName, attributes, dataBlock); // block should exist at position1 but not at position2 - final DataBlock readBlock = n5.readBlock(datasetName, attributes, position1); + final DataBlock readBlock = n5.readChunk(datasetName, attributes, position1); assertNotNull(readBlock); assertTrue(readBlock instanceof ByteArrayDataBlock); assertArrayEquals(byteBlock, ((ByteArrayDataBlock)readBlock).getData()); - assertTrue("deleting existing block should return true", n5.deleteBlock(datasetName, position1)); + assertTrue("deleting existing chunk should return true", n5.deleteChunk(datasetName, position1)); + assertFalse("deleting non-existing chunk should return false", n5.deleteChunk(datasetName, position1)); + assertFalse("deleting non-existing chunk should return false", n5.deleteChunk(datasetName, position2)); + + // for an unsharded dataset, deleteChunk and deleteBlock behave identically assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position1)); assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position2)); + n5.writeChunk(datasetName, attributes, dataBlock); + assertTrue("deleting existing block should return true", n5.deleteBlock(datasetName, position1)); + // no block should exist anymore assertNull(n5.readBlock(datasetName, attributes, position1)); assertNull(n5.readBlock(datasetName, attributes, position2)); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java index 8725baf34..3b763eda4 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java @@ -54,41 +54,45 @@ public void testValidateBlockShardSizesValid() { // Test case 1: shard size equals block size long[] dimensions = new long[]{100, 200, 300}; int[] shardSize = new int[]{64, 64, 64}; - int[] blockSize = new int[]{64, 64, 64}; + int[] chunkSize = new int[]{64, 64, 64}; DataType dataType = DataType.UINT8; // This should not throw any exception - DatasetAttributes attrs = shardDatasetAttributes(dimensions, shardSize, blockSize, dataType); - assertEquals(blockSize, attrs.getBlockSize()); + DatasetAttributes attrs = shardDatasetAttributes(dimensions, shardSize, chunkSize, dataType); + assertEquals(chunkSize, attrs.getChunkSize()); + assertEquals(shardSize, attrs.getBlockSize()); NestedGrid grid = attrs.getNestedBlockGrid(); - assertEquals(blockSize, grid.getBlockSize(0)); + assertEquals(chunkSize, grid.getBlockSize(0)); assertEquals(shardSize, grid.getBlockSize(1)); // Test case 2: shard size is a multiple of block size shardSize = new int[]{128}; - blockSize = new int[]{64}; - attrs = shardDatasetAttributes(new long[]{128}, shardSize, blockSize, dataType); - assertEquals(blockSize, attrs.getBlockSize()); + chunkSize = new int[]{64}; + attrs = shardDatasetAttributes(new long[]{128}, shardSize, chunkSize, dataType); + assertEquals(chunkSize, attrs.getChunkSize()); + assertEquals(shardSize, attrs.getBlockSize()); grid = attrs.getNestedBlockGrid(); - assertEquals(blockSize, grid.getBlockSize(0)); + assertEquals(chunkSize, grid.getBlockSize(0)); assertEquals(shardSize, grid.getBlockSize(1)); // Test case 3: different multiples per dimension shardSize = new int[]{128, 256, 32, 2}; - blockSize = new int[]{32, 64, 32, 1}; - attrs = shardDatasetAttributes(new long[]{128, 128, 128, 128}, shardSize, blockSize, dataType ); - assertEquals(blockSize, attrs.getBlockSize()); + chunkSize = new int[]{32, 64, 32, 1}; + attrs = shardDatasetAttributes(new long[]{128, 128, 128, 128}, shardSize, chunkSize, dataType ); + assertEquals(chunkSize, attrs.getChunkSize()); + assertEquals(shardSize, attrs.getBlockSize()); grid = attrs.getNestedBlockGrid(); - assertEquals(blockSize, grid.getBlockSize(0)); + assertEquals(chunkSize, grid.getBlockSize(0)); assertEquals(shardSize, grid.getBlockSize(1)); // Test case 4: large multiples shardSize = new int[]{1024, 2048, 512}; - blockSize = new int[]{32, 64, 16}; - attrs = shardDatasetAttributes(dimensions, shardSize, blockSize, dataType); - assertEquals(blockSize, attrs.getBlockSize()); + chunkSize = new int[]{32, 64, 16}; + attrs = shardDatasetAttributes(dimensions, shardSize, chunkSize, dataType); + assertEquals(chunkSize, attrs.getChunkSize()); + assertEquals(shardSize, attrs.getBlockSize()); grid = attrs.getNestedBlockGrid(); - assertEquals(blockSize, grid.getBlockSize(0)); + assertEquals(chunkSize, grid.getBlockSize(0)); assertEquals(shardSize, grid.getBlockSize(1)); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java index 546a4072f..198aeae71 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java @@ -141,7 +141,7 @@ public void testDocExample() { n5.createDataset(compressedDatasetName, new long[]{1, 2, 3}, new int[]{1, 2, 3}, DataType.UINT16, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(compressedDatasetName); final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(new int[]{1, 2, 3}, new long[]{0, 0, 0}, dataBlockData); - n5.writeBlock(compressedDatasetName, attributes, dataBlock); + n5.writeChunk(compressedDatasetName, attributes, dataBlock); } catch (final N5Exception e) { fail(e.getMessage()); } @@ -165,7 +165,7 @@ public void benchmarkWritingSpeed() { for (int y = 0; y < nBlocks; ++y) for (int x = 0; x < nBlocks; ++x) { final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(new int[]{64, 64, 64}, new long[]{x, y, z}, data); - n5.writeBlock(compressedDatasetName, attributes, dataBlock); + n5.writeChunk(compressedDatasetName, attributes, dataBlock); } } catch (final N5Exception e) { fail(e.getMessage()); @@ -255,7 +255,7 @@ public void benchmarkParallelWritingSpeed() { exec.submit( () -> { final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(new int[]{64, 64, 64}, new long[]{fx, fy, fz}, data); - n5.writeBlock(compressedDatasetName, attributes, dataBlock); + n5.writeChunk(compressedDatasetName, attributes, dataBlock); return true; })); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java index ecf7643a7..0b959fb5c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java @@ -114,7 +114,7 @@ public void setup() { n5.createDataset(datasetName, dimensions, blockSize, DataType.FLOAT64, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final DoubleArrayDataBlock dataBlock = new DoubleArrayDataBlock(blockSize, new long[] {0, 0, 0}, doubleBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); } n5 = new N5FSReader(basePath, new GsonBuilder()); @@ -129,7 +129,7 @@ public void setup() { @BenchmarkMode( Mode.AverageTime ) @OutputTimeUnit( TimeUnit.MILLISECONDS ) public void bench() { - n5.readBlock(datasetName, attributes, 0, 0, 0); + n5.readChunk(datasetName, attributes, 0, 0, 0); } public static void main( final String... args ) throws RunnerException, IOException diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java index bb579c813..e5a705422 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java @@ -76,7 +76,7 @@ public void backwardReadHelper(final String base, final String dsetPath) throws byte value = 0; long[] p = new long[2]; - DataBlock b00 = n5.readBlock(dsetPath, attrs, p); + DataBlock b00 = n5.readChunk(dsetPath, attrs, p); assertNotNull(b00); assertArrayEquals(new int[]{5,4}, b00.getSize()); assertArrayEquals(expectedData(20, value), b00.getData()); @@ -84,7 +84,7 @@ public void backwardReadHelper(final String base, final String dsetPath) throws p[0] = 1; p[1] = 0; value++; - DataBlock b10 = n5.readBlock(dsetPath, attrs, p); + DataBlock b10 = n5.readChunk(dsetPath, attrs, p); assertNotNull(b10); assertArrayEquals(new int[]{2,4}, b10.getSize()); assertArrayEquals(expectedData(8, value), b10.getData()); @@ -92,7 +92,7 @@ public void backwardReadHelper(final String base, final String dsetPath) throws p[0] = 0; p[1] = 1; value++; - DataBlock b01 = n5.readBlock(dsetPath, attrs, p); + DataBlock b01 = n5.readChunk(dsetPath, attrs, p); assertNotNull(b01); assertArrayEquals(new int[]{5,1}, b01.getSize()); assertArrayEquals(expectedData(5, value), b01.getData()); @@ -100,7 +100,7 @@ public void backwardReadHelper(final String base, final String dsetPath) throws p[0] = 1; p[1] = 1; value++; - DataBlock b11 = n5.readBlock(dsetPath, attrs, p); + DataBlock b11 = n5.readChunk(dsetPath, attrs, p); assertNotNull(b11); assertArrayEquals(new int[]{2,1}, b11.getSize()); assertArrayEquals(expectedData(2, value), b11.getData()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java index 032534b1e..c40b67f26 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java @@ -62,28 +62,28 @@ public static N5FSWriter createSampleData(String baseDir, String dataset, Compre byte val = 0; long[] pos = new long[]{0, 0}; - n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + n5.writeChunk(dsetPath, attrs, createDataBlock(blkSize, pos, val)); pos[0] = 1; pos[1] = 0; blkSize[0] = 2; blkSize[1] = 4; val++; - n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + n5.writeChunk(dsetPath, attrs, createDataBlock(blkSize, pos, val)); pos[0] = 0; pos[1] = 1; blkSize[0] = 5; blkSize[1] = 1; val++; - n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + n5.writeChunk(dsetPath, attrs, createDataBlock(blkSize, pos, val)); pos[0] = 1; pos[1] = 1; blkSize[0] = 2; blkSize[1] = 1; val++; - n5.writeBlock(dsetPath, attrs, createDataBlock( blkSize, pos, val )); + n5.writeChunk(dsetPath, attrs, createDataBlock( blkSize, pos, val )); return n5; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java index ffa35156f..9961906fe 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java @@ -138,7 +138,7 @@ public void setup() { blocks.add(blk); // write data into the read group - n5.writeBlock(readGroup, dsetAttrs, blk); + n5.writeChunk(readGroup, dsetAttrs, blk); } } catch (final IOException e) { @@ -151,7 +151,7 @@ public void setup() { public void writeBenchmark() throws IOException { blocks.forEach(blk -> { - n5.writeBlock(writeGroup, dsetAttrs, blk); + n5.writeChunk(writeGroup, dsetAttrs, blk); }); } @@ -161,7 +161,7 @@ public void readBenchmark(Blackhole hole) throws IOException { final long[] p = new long[numDimensions]; for (int i = 0; i < numBlocks; i++) { p[0] = i; - hole.consume(n5.readBlock(readGroup, dsetAttrs, p)); + hole.consume(n5.readChunk(readGroup, dsetAttrs, p)); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java index 1e37b496b..c0d40b10f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java @@ -159,8 +159,8 @@ public void testEmptyBlock() throws Exception { // Test encode/decode final ByteArrayDataBlock emptyBlock = new ByteArrayDataBlock(blockSize, gridPosition, new byte[0]); - access.writeBlock(store, emptyBlock); - final DataBlock decoded = access.readBlock(store, gridPosition); + access.writeChunk(store, emptyBlock); + final DataBlock decoded = access.readChunk(store, gridPosition); assertEquals("Empty block should have 0 elements", 0, decoded.getNumElements()); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index fe13f0d68..661d556cf 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -116,9 +116,9 @@ public HttpRead return reader.getDatasetAttributes(pathName); } - @Override public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { + @Override public DataBlock readChunk(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { - return reader.readBlock(pathName, getConvertedDatasetAttributes(datasetAttributes), gridPosition); + return reader.readChunk(pathName, getConvertedDatasetAttributes(datasetAttributes), gridPosition); } @Override public T readSerializedBlock(String dataset, DatasetAttributes attributes, long... gridPosition) throws N5Exception, ClassNotFoundException { @@ -272,22 +272,22 @@ public HttpRead return convertedDatasetAttributes; } - @Override public void writeBlock(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { - writer.writeBlock(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); + @Override public void writeChunk(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { + writer.writeChunk(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); } - @Override public void writeShard(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { - writer.writeShard(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); + @Override public void writeBlock(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { + writer.writeBlock(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); } - @Override public boolean deleteBlock(String datasetPath, long... gridPosition) throws N5Exception { + @Override public boolean deleteChunk(String datasetPath, long... gridPosition) throws N5Exception { - return writer.deleteBlock(datasetPath, gridPosition); + return writer.deleteChunk(datasetPath, gridPosition); } - @Override public boolean deleteBlocks(String datasetPath, DatasetAttributes datasetAttributes, List gridPositions) throws N5Exception { + @Override public boolean deleteChunks(String datasetPath, DatasetAttributes datasetAttributes, List gridPositions) throws N5Exception { - return writer.deleteBlocks(datasetPath, datasetAttributes, gridPositions); + return writer.deleteChunks(datasetPath, datasetAttributes, gridPositions); } @Override public void writeSerializedBlock(Serializable object, String datasetPath, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { @@ -315,8 +315,8 @@ public HttpRead writer.writeAttributes(normalGroupPath, attributes); } - @Override public void writeBlocks(String datasetPath, DatasetAttributes datasetAttributes, DataBlock... dataBlocks) throws N5Exception { + @Override public void writeChunks(String datasetPath, DatasetAttributes datasetAttributes, DataBlock... dataBlocks) throws N5Exception { - writer.writeBlocks(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlocks); + writer.writeChunks(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlocks); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java index 1d3becba7..9d54874c0 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java @@ -103,20 +103,20 @@ public void testWriteReadIndividual() { final PositionValueAccess store = new TestPositionValueAccess(); // write some blocks, filled with constant values - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 0, 0}, 1)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 0, 0}, 2)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 1, 0}, 3)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 1, 0}, 4)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {3, 2, 1}, 5)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {8, 4, 1}, 6)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {0, 0, 0}, 1)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {1, 0, 0}, 2)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {0, 1, 0}, 3)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {1, 1, 0}, 4)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {3, 2, 1}, 5)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {8, 4, 1}, 6)); // verify that the written blocks can be read back with the correct values - checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), true, 1); - checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); - checkBlock(datasetAccess.readBlock(store, new long[] {0, 1, 0}), true, 3); - checkBlock(datasetAccess.readBlock(store, new long[] {1, 1, 0}), true, 4); - checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5); - checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6); + checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), true, 1); + checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); + checkBlock(datasetAccess.readChunk(store, new long[] {0, 1, 0}), true, 3); + checkBlock(datasetAccess.readChunk(store, new long[] {1, 1, 0}), true, 4); + checkBlock(datasetAccess.readChunk(store, new long[] {3, 2, 1}), true, 5); + checkBlock(datasetAccess.readChunk(store, new long[] {8, 4, 1}), true, 6); } @Test @@ -132,13 +132,13 @@ public void testWriteReadBulk() { for (int i = 0; i < writeGridPositions.size(); i++) { writeBlocks.add(createDataBlock(dataBlockSize, writeGridPositions.get(i), 1 + i)); } - datasetAccess.writeBlocks(store, writeBlocks); + datasetAccess.writeChunks(store, writeBlocks); // verify that the written blocks can be read back with the correct values final List readGridPositions = Arrays.asList(new long[][] { {1, 0, 0}, {0, 0, 0}, {0, 1, 0}, {2, 4, 2}, {3, 2, 1}, {8, 4, 1} }); - final List> readBlocks = datasetAccess.readBlocks(store, readGridPositions); + final List> readBlocks = datasetAccess.readChunks(store, readGridPositions); checkBlock(readBlocks.get(0), true, 2); checkBlock(readBlocks.get(1), true, 1); checkBlock(readBlocks.get(2), true, 3); @@ -160,20 +160,20 @@ public void testDeleteBlock() { for (int i = 0; i < writeGridPositions.size(); i++) { writeBlocks.add(createDataBlock(dataBlockSize, writeGridPositions.get(i), 1 + i)); } - datasetAccess.writeBlocks(store, writeBlocks); + datasetAccess.writeChunks(store, writeBlocks); // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) - datasetAccess.deleteBlock(store, new long[] {0, 0, 0}); - checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); - checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); + datasetAccess.deleteChunk(store, new long[] {0, 0, 0}); + checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), false, 1); + checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); // if a shard becomes empty the corresponding key should be deleted assertTrue(keyExists(store, new long[] {1, 0, 0})); - datasetAccess.deleteBlock(store, new long[] {8, 4, 1}); + datasetAccess.deleteChunk(store, new long[] {8, 4, 1}); assertFalse(keyExists(store, new long[] {1, 0, 0})); // deleting a non-existent block should not fail - datasetAccess.deleteBlock(store, new long[] {0, 0, 8}); + datasetAccess.deleteChunk(store, new long[] {0, 0, 8}); } private boolean keyExists(final PositionValueAccess store, final long[] key) { @@ -200,20 +200,20 @@ public void testDeleteBlocks() { for (int i = 0; i < writeGridPositions.size(); i++) { writeBlocks.add(createDataBlock(dataBlockSize, writeGridPositions.get(i), 1 + i)); } - datasetAccess.writeBlocks(store, writeBlocks); + datasetAccess.writeChunks(store, writeBlocks); // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) - datasetAccess.deleteBlocks(store, Arrays.asList(new long[][] {{0, 0, 0}, {4, 2, 2}, {3, 2, 1}})); - checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); - checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); + datasetAccess.deleteChunks(store, Arrays.asList(new long[][] {{0, 0, 0}, {4, 2, 2}, {3, 2, 1}})); + checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), false, 1); + checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); // if a shard becomes empty the corresponding key should be deleted assertTrue(keyExists(store, new long[] {1, 0, 0})); - datasetAccess.deleteBlocks(store, Arrays.asList(new long[][] {{8, 4, 1}})); + datasetAccess.deleteChunks(store, Arrays.asList(new long[][] {{8, 4, 1}})); assertFalse(keyExists(store, new long[] {1, 0, 0})); // deleting a non-existent block should not fail - datasetAccess.deleteBlocks(store, Arrays.asList(new long[] {0, 0, 8})); + datasetAccess.deleteChunks(store, Arrays.asList(new long[] {0, 0, 8})); } private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index d881df45f..f4032bd7a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -58,6 +58,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -177,7 +178,7 @@ public void writeReadBlocksTest() { data[i] = (byte)((100) + (10) + i); } - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ @@ -214,7 +215,7 @@ public void writeReadBlocksTest() { final long[][] blockIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; for (long[] blockIndex : blockIndices) { - final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); + final DataBlock block = writer.readChunk(dataset, datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); } @@ -223,7 +224,7 @@ public void writeReadBlocksTest() { data2[i] = (byte)(10 + i); } - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ @@ -255,16 +256,16 @@ public void writeReadBlocksTest() { final long[][] oldBlockIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; for (long[] blockIndex : oldBlockIndices) { - final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); + final DataBlock block = writer.readChunk(dataset, datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); } final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; final List newBlockIndexList = Arrays.asList(newBlockIndices); - final List> readBlocks = writer.readBlocks(dataset, datasetAttributes, newBlockIndexList); + final List> readBlocks = writer.readChunks(dataset, datasetAttributes, newBlockIndexList); for (int i = 0; i < newBlockIndices.length; i++) { final long[] blockIndex = newBlockIndices[i]; - final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); + final DataBlock block = writer.readChunk(dataset, datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])block.getData()); final DataBlock blockFromReadBlocks = readBlocks.get(i); Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])blockFromReadBlocks.getData()); @@ -324,14 +325,14 @@ public void writeShardDataSizeTest() { * and every entry should be null. */ final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; - final List> readBlocks = writer.readBlocks(dataset, datasetAttributes, Arrays.asList(newBlockIndices)); + final List> readBlocks = writer.readChunks(dataset, datasetAttributes, Arrays.asList(newBlockIndices)); assertEquals(newBlockIndices.length, readBlocks.size()); assertTrue("readBlocks for empty shard: all blocks null", readBlocks.stream().allMatch(blk -> blk == null)); /* * Now write blocks */ - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ @@ -386,7 +387,7 @@ public void writeReadBlockTest() { writer.remove(dataset); writer.createDataset(dataset, datasetAttributes); - final int[] blockSize = datasetAttributes.getBlockSize(); + final int[] chunkSize = datasetAttributes.getChunkSize(); final DataType dataType = datasetAttributes.getDataType(); final int numElements = 2 * 2; @@ -395,20 +396,20 @@ public void writeReadBlockTest() { for (int idx1 = 1; idx1 >= 0; idx1--) { for (int idx2 = 1; idx2 >= 0; idx2--) { final long[] gridPosition = {idx1, idx2}; - final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(blockSize, gridPosition, numElements); + final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(chunkSize, gridPosition, numElements); byte[] data = dataBlock.getData(); for (int i = 0; i < data.length; i++) { data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); } - writer.writeBlock(dataset, datasetAttributes, dataBlock); + writer.writeChunk(dataset, datasetAttributes, dataBlock); - final DataBlock block = writer.readBlock(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); + final DataBlock block = writer.readChunk(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); for (Map.Entry entry : writtenBlocks.entrySet()) { final long[] otherGridPosition = entry.getKey(); final byte[] otherData = entry.getValue(); - final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); + final DataBlock otherBlock = writer.readChunk(dataset, datasetAttributes, otherGridPosition); Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); } @@ -425,18 +426,17 @@ public void writeReadShardTest() { final int[] shardSize = new int[] {4,4}; final int shardN = 16; - final int[] blockSize = new int[] {2,2}; - final int blockN = 4; + final int[] chunkSize = new int[] {2,2}; final String dataset = "writeReadShard"; - DatasetAttributes attrs = getTestAttributes(DataType.INT32, new long[]{8, 8}, shardSize, blockSize); + DatasetAttributes attrs = getTestAttributes(DataType.INT32, new long[]{8, 8}, shardSize, chunkSize); final int[] shardData = range(shardN); IntArrayDataBlock shard = new IntArrayDataBlock(shardSize, new long[]{0, 0}, shardData); - n5.writeShard(dataset, attrs, shard); - DataBlock readShard = n5.readShard(dataset, attrs, 0, 0); - assertArrayEquals(shardData, readShard.getData()); + n5.writeBlock(dataset, attrs, shard); + DataBlock readBlock = n5.readBlock(dataset, attrs, 0, 0); + assertArrayEquals(shardData, readBlock.getData()); /** @@ -451,12 +451,12 @@ public void writeReadShardTest() { * 12 13 | 14 15 */ - assertArrayEquals(new int[]{0, 1, 4, 5}, (int[])n5.readBlock(dataset, attrs, 0, 0).getData()); - assertArrayEquals(new int[]{2, 3, 6, 7}, (int[])n5.readBlock(dataset, attrs, 1, 0).getData()); - assertArrayEquals(new int[]{8, 9, 12, 13}, (int[])n5.readBlock(dataset, attrs, 0, 1).getData()); - assertArrayEquals(new int[]{10, 11, 14, 15}, (int[])n5.readBlock(dataset, attrs, 1, 1).getData()); + assertArrayEquals(new int[]{0, 1, 4, 5}, (int[])n5.readChunk(dataset, attrs, 0, 0).getData()); + assertArrayEquals(new int[]{2, 3, 6, 7}, (int[])n5.readChunk(dataset, attrs, 1, 0).getData()); + assertArrayEquals(new int[]{8, 9, 12, 13}, (int[])n5.readChunk(dataset, attrs, 0, 1).getData()); + assertArrayEquals(new int[]{10, 11, 14, 15}, (int[])n5.readChunk(dataset, attrs, 1, 1).getData()); - n5.deleteBlock(dataset, attrs, new long[]{1, 1}); + n5.deleteChunk(dataset, attrs, new long[]{1, 1}); /** * After deleting block (1,1) @@ -467,15 +467,29 @@ public void writeReadShardTest() { * 8 9 | 0 0 * 12 13 | 0 0 */ - final DataBlock partlyEmptyShard = n5.readShard(dataset, attrs, 0, 0); + final DataBlock partlyEmptyShard = n5.readBlock(dataset, attrs, 0, 0); assertArrayEquals(new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 12, 13, 0, 0}, partlyEmptyShard.getData()); // Delete the rest of the blocks - n5.deleteBlocks(dataset, attrs, + n5.deleteChunks(dataset, attrs, Stream.of( new long[] {0,0}, new long[] {1,0}, new long[] {0,1}).collect(Collectors.toList())); - assertNull(n5.readShard(dataset, attrs, 0, 0)); + assertNull(n5.readBlock(dataset, attrs, 0, 0)); + + + // write the shard again + n5.writeBlock(dataset, attrs, shard); + + // delete the block + // ensure it returns true because the block exists + assertTrue(n5.deleteBlock(dataset, attrs, shard.getGridPosition())); + + // ensure it returns false when the block does not exist + assertFalse(n5.deleteBlock(dataset, attrs, shard.getGridPosition())); + + // readBlock must return null for the deleted block + assertNull(n5.readBlock(dataset, attrs, shard.getGridPosition())); } } @@ -505,7 +519,7 @@ public void numReadsTest() { data[i] = (byte)((100) + (10) + i); } - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ @@ -523,7 +537,7 @@ public void numReadsTest() { ); writer.resetNumMaterializeCalls(); - writer.readBlocks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0})); + writer.readChunks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0})); System.out.println(writer.getNumMaterializeCalls()); ArrayList ptList = new ArrayList<>(); @@ -533,7 +547,7 @@ public void numReadsTest() { ptList.add(new long[] {1, 1}); writer.resetNumMaterializeCalls(); - writer.readBlocks(dataset, datasetAttributes, ptList); + writer.readChunks(dataset, datasetAttributes, ptList); System.out.println(writer.getNumMaterializeCalls()); System.out.println(""); } @@ -562,7 +576,7 @@ public void shardExistsTest() { } /* write blocks to shards (0,0), (1,0), and (2,2) */ - writer.writeBlocks( + writer.writeChunks( dataset, attrs, new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), /* shard (0, 0) */ @@ -574,7 +588,7 @@ public void shardExistsTest() { Predicate assertShardExistsTracking = (gridPosition) -> { trackingWriter.resetAllTracking(); - final boolean exists = writer.shardExists(dataset, attrs, gridPosition); + final boolean exists = writer.blockExists(dataset, attrs, gridPosition); assertEquals("isFileCheck incremented", 1, trackingWriter.getNumIsFileCalls()); assertEquals("No Bytes Read", 0, trackingWriter.getTotalBytesRead()); return exists; @@ -629,7 +643,7 @@ public void testPartialReadAggregationBehavior() { ptList.add(new long[] {1,1}); /* write blocks to shard (0,0) */ - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, new ByteArrayDataBlock(blockSize, ptList.get(0), data), @@ -639,14 +653,14 @@ public void testPartialReadAggregationBehavior() { ); writer.resetNumMaterializeCalls(); - writer.readBlocks(dataset, datasetAttributes, ptList); + writer.readChunks(dataset, datasetAttributes, ptList); // TODO change this if and when we implement aggregation of read calls // one for the index, one for each of the four blocks assertEquals(5, writer.getNumMaterializeCalls()); writer.resetNumMaterializeCalls(); - writer.readShard(dataset, datasetAttributes, new long[] {0,0}); + writer.readBlock(dataset, datasetAttributes, new long[] {0,0}); // one for the index, one for each of the four blocks assertEquals(5, writer.getNumMaterializeCalls()); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java index 5c900a4a6..02737334f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java @@ -100,7 +100,7 @@ public static void main(String[] args) { System.out.println("shard.getSize() = " + Arrays.toString(shard.getSize())); System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); System.out.println(); - datasetAccess.writeShard(store, shard, 1); + datasetAccess.writeBlock(store, shard, 1); System.out.println(); System.out.println(); } @@ -110,7 +110,7 @@ public static void main(String[] args) { System.out.println("shard.getSize() = " + Arrays.toString(shard.getSize())); System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); System.out.println(); - datasetAccess.writeShard(store, shard, 1); + datasetAccess.writeBlock(store, shard, 1); System.out.println(); System.out.println(); } @@ -201,7 +201,7 @@ public void testWriteNullBlockRemovesShard() throws Exception { final long[] blockGridPosition = {3}; final DataBlock block = createDataBlock(datablockSize, blockGridPosition, 100); - datasetAccess.writeBlock(store, block); + datasetAccess.writeChunk(store, block); // Verify the shard exists assertTrue("Shard should exist after writing block", store.exists(shardKey)); @@ -241,7 +241,7 @@ public void testWriteNullBlockRemovesShard() throws Exception { assertFalse("Shard should not exist at the start of the test", store.exists(shardKey2)); // write blocks - datasetAccess.writeBlocks(store, Streams.of(block1, block2, block3).collect(Collectors.toList())); + datasetAccess.writeChunks(store, Streams.of(block1, block2, block3).collect(Collectors.toList())); // Verify the shard exists assertTrue("Shard should exist after writing blocks", store.exists(shardKey1)); @@ -260,16 +260,16 @@ public void testWriteNullBlockRemovesShard() throws Exception { assertTrue("Shard should still exist because was not affected", store.exists(shardKey2)); // Verify we can still read block [2] - final DataBlock readBlock = datasetAccess.readBlock(store, new long[]{2}); + final DataBlock readBlock = datasetAccess.readChunk(store, new long[]{2}); assertTrue("Block [2] should still be readable", readBlock != null); assertTrue("Block [2] data should match", Arrays.equals(block1.getData(), readBlock.getData())); // Verify block [3] is gone - final DataBlock readBlock2 = datasetAccess.readBlock(store, new long[]{3}); + final DataBlock readBlock2 = datasetAccess.readChunk(store, new long[]{3}); assertNull("Block [3] should be null after removal", readBlock2); // Verify block [4] exists - final DataBlock readBlock3 = datasetAccess.readBlock(store, new long[]{4}); + final DataBlock readBlock3 = datasetAccess.readChunk(store, new long[]{4}); assertTrue("Block [4] should still be readable", readBlock3 != null); assertTrue("Block [4] data should match", Arrays.equals(block3.getData(), readBlock3.getData())); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java index 86c0308a2..452bfce23 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java @@ -90,7 +90,7 @@ public static void main(String[] args) { System.out.println("shard.getGridPosition() = " + Arrays.toString(shard.getGridPosition())); System.out.println("shard.getSize() = " + Arrays.toString(shard.getSize())); System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); - datasetAccess.writeShard(store, shard, 1); + datasetAccess.writeBlock(store, shard, 1); // we should get these DataBlock values @@ -103,15 +103,15 @@ public static void main(String[] args) { // 31, 32, 33, | 34, 35, 36 System.out.println("{4, 2}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {4, 2}).getData())); + datasetAccess.readChunk(store, new long[] {4, 2}).getData())); System.out.println("{5, 2}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {5, 2}).getData())); + datasetAccess.readChunk(store, new long[] {5, 2}).getData())); System.out.println("{4, 3}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {4, 3}).getData())); + datasetAccess.readChunk(store, new long[] {4, 3}).getData())); System.out.println("{5, 3}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {5, 3}).getData())); + datasetAccess.readChunk(store, new long[] {5, 3}).getData())); - final DataBlock readShard = datasetAccess.readShard(store, new long[] {2, 1}, 1); + final DataBlock readShard = datasetAccess.readBlock(store, new long[] {2, 1}, 1); System.out.println("readShard.getData() = " + Arrays.toString(readShard.getData())); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java index 87737668f..73f239d50 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java @@ -90,7 +90,7 @@ public static void main(String[] args) { System.out.println("shard.getGridPosition() = " + Arrays.toString(shard.getGridPosition())); System.out.println("shard.getSize() = " + Arrays.toString(shard.getSize())); System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); - datasetAccess.writeShard(store, shard, 1); + datasetAccess.writeBlock(store, shard, 1); // we should get these DataBlock values @@ -103,13 +103,13 @@ public static void main(String[] args) { // 31, 32, 33, | 34, 35, 36 System.out.println("{10, 4}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {10, 4}).getData())); + datasetAccess.readChunk(store, new long[] {10, 4}).getData())); System.out.println("{11, 4}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {11, 4}).getData())); + datasetAccess.readChunk(store, new long[] {11, 4}).getData())); System.out.println("{10, 5}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {10, 5}).getData())); + datasetAccess.readChunk(store, new long[] {10, 5}).getData())); System.out.println("{11, 5}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {11, 5}).getData())); + datasetAccess.readChunk(store, new long[] {11, 5}).getData())); // 1, 2, 3, | 4 @@ -119,7 +119,7 @@ public static void main(String[] args) { // 19, 20, 21, | 22 // 25, 26, 27, | 28 - final DataBlock readShard = datasetAccess.readShard(store, new long[] {5, 2}, 1); + final DataBlock readShard = datasetAccess.readBlock(store, new long[] {5, 2}, 1); System.out.println("readShard.getData() = " + Arrays.toString(readShard.getData())); }