Merge pull request #198 from saalfeldlab/default-write-region

Add default implementation of N5writer.writeRegion

Author: cmhulbert

pom.xml

@@ -11,7 +11,7 @@
 
 	<groupId>org.janelia.saalfeldlab</groupId>
 	<artifactId>n5</artifactId>
-	<version>4.0.0-alpha-7-SNAPSHOT</version>
+	<version>4.0.0-alpha-8-SNAPSHOT</version>
 
 	<name>N5</name>
 	<description>Not HDF5</description>

src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java

@@ -298,21 +298,19 @@ default boolean remove(final String path) throws N5Exception {
 	@Override
 	default boolean deleteBlock(
 			final String path,
+			final DatasetAttributes datasetAttributes,
 			final long... gridPosition) throws N5Exception {
 
-		final String normalPath = N5URI.normalizeGroupPath(path);
-		final DatasetAttributes datasetAttributes = getDatasetAttributes(normalPath);
 		final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);
 		return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition);
 	}
 
 	@Override
 	default boolean deleteBlocks(
 			final String path,
+			final DatasetAttributes datasetAttributes,
 			final List<long[]> gridPositions) throws N5Exception {
 
-		final String normalPath = N5URI.normalizeGroupPath(path);
-		final DatasetAttributes datasetAttributes = getDatasetAttributes(normalPath);
 		final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);
 		return datasetAttributes.getDatasetAccess().deleteBlocks(posKva, gridPositions);
 	}

src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java

@@ -39,6 +39,9 @@
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
+import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;
+import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition;
+import org.janelia.saalfeldlab.n5.shard.Region;
 
 /**
  * A simple structured container API for hierarchies of chunked
@@ -298,13 +301,33 @@ interface DataBlockSupplier<T> {
 	 * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything.
 	 * @throws N5Exception the exception
 	 */
-	<T> void writeRegion(
+	default <T> void writeRegion(
 			String datasetPath,
 			DatasetAttributes datasetAttributes,
 			long[] min,
 			long[] size,
 			DataBlockSupplier<T> dataBlocks,
-			boolean writeFully) throws N5Exception;
+			boolean writeFully) throws N5Exception {
+
+		final NestedGrid grid = datasetAttributes.getNestedBlockGrid();
+		final Region region = new Region(min, size, grid);
+		for (long[] key : Region.gridPositions(region.minPos().key(), region.maxPos().key())) {
+			final NestedPosition pos = grid.nestedPosition(key, grid.numLevels() - 1);
+			final long[] gridPosition = pos.absolute(0);
+			final DataBlock<T> existingDataBlock = writeFully || region.fullyContains(pos)
+					? null
+					: readBlock(datasetPath, datasetAttributes, gridPosition);
+			final DataBlock<T> dataBlock = dataBlocks.get(gridPosition, existingDataBlock);
+			// null blocks may be provided when they contain only the fill value
+			// and only non-empty blocks should be written, for example
+			if (dataBlock == null) {
+				deleteBlock(datasetPath, datasetAttributes, gridPosition);
+			} else {
+				writeBlock(datasetPath, datasetAttributes, dataBlock);
+			}
+		}
+
+	}
 
 	/**
 	 * @param datasetPath the dataset path
@@ -316,14 +339,35 @@ <T> void writeRegion(
 	 * @param exec used to parallelize over blocks and shards
 	 * @throws N5Exception the exception
 	 */
-	<T> void writeRegion(
+	default <T> void writeRegion(
 			String datasetPath,
 			DatasetAttributes datasetAttributes,
 			long[] min,
 			long[] size,
 			DataBlockSupplier<T> dataBlocks,
 			boolean writeFully,
-			ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException;
+			ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException {
+
+		final NestedGrid grid = datasetAttributes.getNestedBlockGrid();
+		final Region region = new Region(min, size, grid);
+		for (long[] key : Region.gridPositions(region.minPos().key(), region.maxPos().key())) {
+			exec.submit(() -> {
+				final NestedPosition pos = grid.nestedPosition(key, grid.numLevels() - 1);
+				final long[] gridPosition = pos.absolute(0);
+				final DataBlock<T> existingDataBlock = writeFully || region.fullyContains(pos)
+						? null
+						: readBlock(datasetPath, datasetAttributes, gridPosition);
+				final DataBlock<T> dataBlock = dataBlocks.get(gridPosition, existingDataBlock);
+				// null blocks may be provided when they contain only the fill value
+				// and only non-empty blocks should be written, for example
+				if (dataBlock == null) {
+					deleteBlock(datasetPath, datasetAttributes, gridPosition);
+				} else {
+					writeBlock(datasetPath, datasetAttributes, dataBlock);
+				}
+			});
+		}
+	}
 
 	/**
 	 * Deletes the block at {@code gridPosition}.
@@ -334,9 +378,27 @@ <T> void writeRegion(
 	 *
 	 * @return {@code true} if the block at {@code gridPosition} existed and was deleted.
 	 */
-	boolean deleteBlock(
+	default boolean deleteBlock(
 			final String datasetPath,
-			final long... gridPosition) throws N5Exception;
+			final long... gridPosition) throws N5Exception {
+		final DatasetAttributes datasetAttributes = getDatasetAttributes(datasetPath);
+		return deleteBlock(datasetPath, datasetAttributes, gridPosition);
+	}
+
+	/**
+	 * Deletes the block at {@code gridPosition}.
+	 *
+	 * @param datasetPath the dataset path
+	 * @param datasetAttributes the dataset attributes
+	 * @param gridPosition position of block to be deleted
+	 * @throws N5Exception if the block exists but could not be deleted
+	 *
+	 * @return {@code true} if the block at {@code gridPosition} existed and was deleted.
+	 */
+	boolean deleteBlock(
+			String datasetPath,
+			DatasetAttributes datasetAttributes,
+			long... gridPosition) throws N5Exception;
 
 	/**
 	 * Deletes the blocks at the given {@code gridPositions}.
@@ -347,11 +409,12 @@ boolean deleteBlock(
 	 * @throws N5Exception if any of the block exists but could not be deleted
 	 */
 	default boolean deleteBlocks(
-			final String datasetPath,
-			final List<long[]> gridPositions) throws N5Exception {
+			String datasetPath,
+			DatasetAttributes datasetAttributes,
+			List<long[]> gridPositions) throws N5Exception {
 		boolean deleted = false;
 		for (long[] pos : gridPositions) {
-			deleted |= deleteBlock(datasetPath, pos);
+			deleted |= deleteBlock(datasetPath, datasetAttributes, pos);
 		}
 		return deleted;
 	}

src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java

@@ -39,7 +39,7 @@
  * Provides methods to find which blocks and shards are contained in the
  * region, iterate sub-NestedPositions, etc.
  */
-class Region {
+public class Region {
 
 	/**
 	 * The dimensions of the full dataset.
@@ -72,7 +72,7 @@ class Region {
 	 */
 	private final Nesting.NestedPosition maxPos;
 
-	Region(final long[] min, final long[] size, final NestedGrid grid) {
+	public Region(final long[] min, final long[] size, final NestedGrid grid) {
 		this.min = min;
 		this.size = size;
 		this.grid = grid;
@@ -93,14 +93,14 @@ class Region {
 	/**
 	 * Get the {@code NestedPosition} of the minimum DataBlock touched by the region.
 	 */
-	Nesting.NestedPosition minPos() {
+	public Nesting.NestedPosition minPos() {
 		return minPos;
 	}
 
 	/**
 	 * Get the {@code NestedPosition} of the maximum DataBlock touched by the region.
 	 */
-	Nesting.NestedPosition maxPos() {
+	public Nesting.NestedPosition maxPos() {
 		return maxPos;
 	}
 
@@ -116,7 +116,7 @@ Nesting.NestedPosition maxPos() {
 	 *
 	 * @return true, if the given position is fully contained in this region
 	 */
-	boolean fullyContains(final Nesting.NestedPosition position) {
+	public boolean fullyContains(final Nesting.NestedPosition position) {
 
 		final long[] pmin = position.pixelPosition();
 		for (int d = 0; d < pmin.length; d++) {
@@ -165,7 +165,7 @@ List<Nesting.NestedPosition> containedNestedPositions(final Nesting.NestedPositi
 	}
 
 	// TODO: Revise to accept Consumer<long[]> for handling each position
-	static List<long[]> gridPositions(final long[] min, final long[] max) {
+	public static List<long[]> gridPositions(final long[] min, final long[] max) {
 		final int n = min.length;
 		final long[] pos = min.clone();
 		int numElements = 1;

src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java

@@ -281,9 +281,9 @@ public <W extends GsonKeyValueN5Writer, R extends GsonKeyValueN5Reader> HttpRead
 		return writer.deleteBlock(datasetPath, gridPosition);
 	}
 
-	@Override public boolean deleteBlocks(String datasetPath, List<long[]> gridPositions) throws N5Exception {
+	@Override public boolean deleteBlocks(String datasetPath, DatasetAttributes datasetAttributes, List<long[]> gridPositions) throws N5Exception {
 
-		return writer.deleteBlocks(datasetPath, gridPositions);
+		return writer.deleteBlocks(datasetPath, datasetAttributes, gridPositions);
 	}
 
 	@Override public void writeSerializedBlock(Serializable object, String datasetPath, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception {