diff --git a/pom.xml b/pom.xml index 80e778aa4..8cabbe6c4 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.janelia.saalfeldlab n5 - 4.0.0-alpha-7-SNAPSHOT + 4.0.0-alpha-8-SNAPSHOT N5 Not HDF5 diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index d083449d7..a938e65a3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -298,10 +298,9 @@ default boolean remove(final String path) throws N5Exception { @Override default boolean deleteBlock( final String path, + final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { - final String normalPath = N5URI.normalizeGroupPath(path); - final DatasetAttributes datasetAttributes = getDatasetAttributes(normalPath); final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition); } @@ -309,10 +308,9 @@ default boolean deleteBlock( @Override default boolean deleteBlocks( final String path, + final DatasetAttributes datasetAttributes, final List gridPositions) throws N5Exception { - final String normalPath = N5URI.normalizeGroupPath(path); - final DatasetAttributes datasetAttributes = getDatasetAttributes(normalPath); final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); return datasetAttributes.getDatasetAccess().deleteBlocks(posKva, gridPositions); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 04aab9c14..79c5da110 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -39,6 +39,9 @@ import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; +import org.janelia.saalfeldlab.n5.shard.Region; /** * A simple structured container API for hierarchies of chunked @@ -298,13 +301,33 @@ interface DataBlockSupplier { * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. * @throws N5Exception the exception */ - void writeRegion( + default void writeRegion( String datasetPath, DatasetAttributes datasetAttributes, long[] min, long[] size, DataBlockSupplier dataBlocks, - boolean writeFully) throws N5Exception; + boolean writeFully) throws N5Exception { + + final NestedGrid grid = datasetAttributes.getNestedBlockGrid(); + final Region region = new Region(min, size, grid); + for (long[] key : Region.gridPositions(region.minPos().key(), region.maxPos().key())) { + final NestedPosition pos = grid.nestedPosition(key, grid.numLevels() - 1); + final long[] gridPosition = pos.absolute(0); + final DataBlock existingDataBlock = writeFully || region.fullyContains(pos) + ? null + : readBlock(datasetPath, datasetAttributes, gridPosition); + final DataBlock dataBlock = dataBlocks.get(gridPosition, existingDataBlock); + // null blocks may be provided when they contain only the fill value + // and only non-empty blocks should be written, for example + if (dataBlock == null) { + deleteBlock(datasetPath, datasetAttributes, gridPosition); + } else { + writeBlock(datasetPath, datasetAttributes, dataBlock); + } + } + + } /** * @param datasetPath the dataset path @@ -316,14 +339,35 @@ void writeRegion( * @param exec used to parallelize over blocks and shards * @throws N5Exception the exception */ - void writeRegion( + default void writeRegion( String datasetPath, DatasetAttributes datasetAttributes, long[] min, long[] size, DataBlockSupplier dataBlocks, boolean writeFully, - ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException; + ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException { + + final NestedGrid grid = datasetAttributes.getNestedBlockGrid(); + final Region region = new Region(min, size, grid); + for (long[] key : Region.gridPositions(region.minPos().key(), region.maxPos().key())) { + exec.submit(() -> { + final NestedPosition pos = grid.nestedPosition(key, grid.numLevels() - 1); + final long[] gridPosition = pos.absolute(0); + final DataBlock existingDataBlock = writeFully || region.fullyContains(pos) + ? null + : readBlock(datasetPath, datasetAttributes, gridPosition); + final DataBlock dataBlock = dataBlocks.get(gridPosition, existingDataBlock); + // null blocks may be provided when they contain only the fill value + // and only non-empty blocks should be written, for example + if (dataBlock == null) { + deleteBlock(datasetPath, datasetAttributes, gridPosition); + } else { + writeBlock(datasetPath, datasetAttributes, dataBlock); + } + }); + } + } /** * Deletes the block at {@code gridPosition}. @@ -334,9 +378,27 @@ void writeRegion( * * @return {@code true} if the block at {@code gridPosition} existed and was deleted. */ - boolean deleteBlock( + default boolean deleteBlock( final String datasetPath, - final long... gridPosition) throws N5Exception; + final long... gridPosition) throws N5Exception { + final DatasetAttributes datasetAttributes = getDatasetAttributes(datasetPath); + return deleteBlock(datasetPath, datasetAttributes, gridPosition); + } + + /** + * Deletes the block at {@code gridPosition}. + * + * @param datasetPath the dataset path + * @param datasetAttributes the dataset attributes + * @param gridPosition position of block to be deleted + * @throws N5Exception if the block exists but could not be deleted + * + * @return {@code true} if the block at {@code gridPosition} existed and was deleted. + */ + boolean deleteBlock( + String datasetPath, + DatasetAttributes datasetAttributes, + long... gridPosition) throws N5Exception; /** * Deletes the blocks at the given {@code gridPositions}. @@ -347,11 +409,12 @@ boolean deleteBlock( * @throws N5Exception if any of the block exists but could not be deleted */ default boolean deleteBlocks( - final String datasetPath, - final List gridPositions) throws N5Exception { + String datasetPath, + DatasetAttributes datasetAttributes, + List gridPositions) throws N5Exception { boolean deleted = false; for (long[] pos : gridPositions) { - deleted |= deleteBlock(datasetPath, pos); + deleted |= deleteBlock(datasetPath, datasetAttributes, pos); } return deleted; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java index 5c611186f..38a267ba1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java @@ -39,7 +39,7 @@ * Provides methods to find which blocks and shards are contained in the * region, iterate sub-NestedPositions, etc. */ -class Region { +public class Region { /** * The dimensions of the full dataset. @@ -72,7 +72,7 @@ class Region { */ private final Nesting.NestedPosition maxPos; - Region(final long[] min, final long[] size, final NestedGrid grid) { + public Region(final long[] min, final long[] size, final NestedGrid grid) { this.min = min; this.size = size; this.grid = grid; @@ -93,14 +93,14 @@ class Region { /** * Get the {@code NestedPosition} of the minimum DataBlock touched by the region. */ - Nesting.NestedPosition minPos() { + public Nesting.NestedPosition minPos() { return minPos; } /** * Get the {@code NestedPosition} of the maximum DataBlock touched by the region. */ - Nesting.NestedPosition maxPos() { + public Nesting.NestedPosition maxPos() { return maxPos; } @@ -116,7 +116,7 @@ Nesting.NestedPosition maxPos() { * * @return true, if the given position is fully contained in this region */ - boolean fullyContains(final Nesting.NestedPosition position) { + public boolean fullyContains(final Nesting.NestedPosition position) { final long[] pmin = position.pixelPosition(); for (int d = 0; d < pmin.length; d++) { @@ -165,7 +165,7 @@ List containedNestedPositions(final Nesting.NestedPositi } // TODO: Revise to accept Consumer for handling each position - static List gridPositions(final long[] min, final long[] max) { + public static List gridPositions(final long[] min, final long[] max) { final int n = min.length; final long[] pos = min.clone(); int numElements = 1; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index 4e3e5d66d..b5555313a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -281,9 +281,9 @@ public HttpRead return writer.deleteBlock(datasetPath, gridPosition); } - @Override public boolean deleteBlocks(String datasetPath, List gridPositions) throws N5Exception { + @Override public boolean deleteBlocks(String datasetPath, DatasetAttributes datasetAttributes, List gridPositions) throws N5Exception { - return writer.deleteBlocks(datasetPath, gridPositions); + return writer.deleteBlocks(datasetPath, datasetAttributes, gridPositions); } @Override public void writeSerializedBlock(Serializable object, String datasetPath, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception {