Add bulk DatasetAccess.deleteBlocks() and use in N5Writer

Author: tpietzsch

src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java

@@ -301,4 +301,15 @@ default boolean deleteBlock(
 		final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);
 		return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition);
 	}
+
+	@Override
+	default boolean deleteBlocks(
+			final String path,
+			final List<long[]> gridPositions) throws N5Exception {
+
+		final String normalPath = N5URI.normalizeGroupPath(path);
+		final DatasetAttributes datasetAttributes = getDatasetAttributes(normalPath);
+		final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);
+		return datasetAttributes.getDatasetAccess().deleteBlocks(posKva, gridPositions);
+	}
 }

src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java

@@ -28,19 +28,14 @@
  */
 package org.janelia.saalfeldlab.n5.shard;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
-import java.util.TreeMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import org.janelia.saalfeldlab.n5.DataBlock;
 import org.janelia.saalfeldlab.n5.N5Exception;
 import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
 import org.janelia.saalfeldlab.n5.N5Writer.DataBlockSupplier;
 import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;
-import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition;
 
 /**
  * Wrap an instantiated DataBlock/shard codec hierarchy to implement (single and
@@ -99,8 +94,7 @@ public interface DatasetAccess<T> {
 
 	boolean deleteBlock(PositionValueAccess pva, long[] gridPosition) throws N5IOException;
 
-//	TODO:
-//	  boolean deleteBlocks(PositionValueAccess pva, List<long[]> positions) throws N5IOException;
+	boolean deleteBlocks(PositionValueAccess pva, List<long[]> positions) throws N5IOException;
 
 	/**
 	 *

src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java

@@ -387,7 +387,7 @@ public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosit
 				throw new N5Exception("The shard at " + Arrays.toString(key) + " could not be deleted.", e);
 			}
 		} else {
-			final ReadData existingData = pva.get(key);
+			final ReadData existingData = pva.get(key); // TODO: use getExistingReadData() instead !?
 			final ReadData modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1);
 			if (existingData != null && modifiedData == null) {
 				return pva.remove(key);
@@ -438,6 +438,115 @@ private ReadData deleteBlockRecursive(
 		}
 	}
 
+	//
+	// -- deleteBlocks --------------------------------------------------------
+
+	@Override
+	public boolean deleteBlocks(final PositionValueAccess pva, final List<long[]> gridPositions) throws N5IOException {
+
+		boolean deleted = false;
+
+		// for non-sharded datasets, just delete the blocks individually
+		if (grid.numLevels() == 1) {
+			for (long[] pos : gridPositions) {
+				deleted |= deleteBlock(pva, pos);
+			}
+			return deleted;
+		}
+
+		// Create a list of DataBlockRequests and sort it such that requests
+		// from the same (nested) shard are grouped contiguously.
+		// Despite the name, createReadRequests() works for delete requests as well ...
+		final DataBlockRequests<T> requests = createReadRequests(gridPositions);
+		requests.removeDuplicates();
+
+		final List<DataBlockRequests<T>> split = requests.split();
+		for (final DataBlockRequests<T> subRequests : split) {
+			final boolean writeFully = subRequests.coversShard();
+			final long[] key = subRequests.relativeGridPosition();
+			final ReadData existingData = writeFully ? null : getExistingReadData(pva, key);
+			final ReadData modifiedData = deleteBlocksRecursive(existingData, subRequests);
+			if (existingData != null && modifiedData == null) {
+				deleted |= pva.remove(key);
+			} else if (modifiedData != existingData) {
+				pva.put(key, modifiedData);
+				deleted = true;
+			}
+		}
+
+		return deleted;
+	}
+
+	/**
+	 * Bulk Delete operation on a shard.
+	 *
+	 * @param existingReadData encoded existing shard data (to decode and partially override)
+	 * @param requests for blocks within the shard to be deleted
+	 */
+	private ReadData deleteBlocksRecursive(
+			final ReadData existingReadData, // may be null
+			final DataBlockRequests<T> requests
+	) {
+		assert !requests.requests.isEmpty();
+		assert requests.level > 0;
+
+		if (existingReadData == null) {
+			return null;
+		}
+
+		final int level = requests.level();
+		@SuppressWarnings("unchecked")
+		final BlockCodec<RawShard> codec = (BlockCodec<RawShard>) codecs[level];
+		final long[] gridPos = requests.gridPosition();
+		final RawShard shard = codec.decode(existingReadData, gridPos).getData();
+
+		boolean modified = false;
+		boolean shardElementSetToNull = false;
+		if ( level == 1 ) {
+			// Base case, delete the blocks
+			for (final DataBlockRequest<T> request : requests) {
+				final long[] elementPos = request.position.relative(0);
+				if (shard.getElementData(elementPos) != null) {
+					shard.setElementData(null, elementPos);
+					modified = true;
+					shardElementSetToNull = true;
+				}
+			}
+		} else { // level > 1
+			final List<DataBlockRequests<T>> split = requests.split();
+			for (final DataBlockRequests<T> subRequests : split) {
+				final boolean writeFully = subRequests.coversShard();
+				final long[] elementPos = subRequests.relativeGridPosition();
+				final ReadData existingElementData = writeFully ? null : shard.getElementData(elementPos);
+				final ReadData modifiedElementData = deleteBlocksRecursive(existingElementData, subRequests);
+				if (modifiedElementData != existingElementData) {
+					shard.setElementData(modifiedElementData, elementPos);
+					modified = true;
+					shardElementSetToNull |= (modifiedElementData == null);
+				}
+			}
+		}
+
+		if (!modified) {
+			// No nested shard or DataBlock was modified.
+			// This shard remains unchanged.
+			return existingReadData;
+		}
+
+		if (shardElementSetToNull) {
+			// At least one DataBlock or nested shard was removed.
+			// Check whether this shard becomes empty.
+			if (shard.index().allElementsNull()) {
+				// This shard is empty and should be removed.
+				return null;
+			}
+		}
+
+		return codec.encode(new RawShardDataBlock(gridPos, shard));
+	}
+
+
+
 	//
 	// -- readShard -----------------------------------------------------------
 

src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java

@@ -275,6 +275,11 @@ public <W extends GsonKeyValueN5Writer, R extends GsonKeyValueN5Reader> HttpRead
 		return writer.deleteBlock(datasetPath, gridPosition);
 	}
 
+	@Override public boolean deleteBlocks(String datasetPath, List<long[]> gridPositions) throws N5Exception {
+
+		return writer.deleteBlocks(datasetPath, gridPositions);
+	}
+
 	@Override public void writeSerializedBlock(Serializable object, String datasetPath, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception {
 
 		writer.writeSerializedBlock(object, datasetPath, datasetAttributes, gridPosition);