diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 02ace27e3..be6a47d9c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -301,4 +301,15 @@ default boolean deleteBlock( final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition); } + + @Override + default boolean deleteBlocks( + final String path, + final List gridPositions) throws N5Exception { + + final String normalPath = N5URI.normalizeGroupPath(path); + final DatasetAttributes datasetAttributes = getDatasetAttributes(normalPath); + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); + return datasetAttributes.getDatasetAccess().deleteBlocks(posKva, gridPositions); + } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index 826162e6d..b09df6782 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -28,11 +28,7 @@ */ package org.janelia.saalfeldlab.n5.shard; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.List; -import java.util.TreeMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import org.janelia.saalfeldlab.n5.DataBlock; @@ -40,7 +36,6 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5Writer.DataBlockSupplier; import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; -import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; /** * Wrap an instantiated DataBlock/shard codec hierarchy to implement (single and @@ -99,8 +94,7 @@ public interface DatasetAccess { boolean deleteBlock(PositionValueAccess pva, long[] gridPosition) throws N5IOException; -// TODO: -// boolean deleteBlocks(PositionValueAccess pva, List positions) throws N5IOException; + boolean deleteBlocks(PositionValueAccess pva, List positions) throws N5IOException; /** * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 8efea03f5..a2bd4f4e6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -387,7 +387,7 @@ public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosit throw new N5Exception("The shard at " + Arrays.toString(key) + " could not be deleted.", e); } } else { - final ReadData existingData = pva.get(key); + final ReadData existingData = pva.get(key); // TODO: use getExistingReadData() instead !? final ReadData modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); if (existingData != null && modifiedData == null) { return pva.remove(key); @@ -428,7 +428,7 @@ private ReadData deleteBlockRecursive( if (modifiedElementData == null) { // The DataBlock or nested shard was removed. // Check whether this shard becomes empty. - if (shard.index().allElementsNull()) { + if (shard.isEmpty()) { // This shard is empty and should be removed. return null; } @@ -438,6 +438,115 @@ private ReadData deleteBlockRecursive( } } + // + // -- deleteBlocks -------------------------------------------------------- + + @Override + public boolean deleteBlocks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { + + boolean deleted = false; + + // for non-sharded datasets, just delete the blocks individually + if (grid.numLevels() == 1) { + for (long[] pos : gridPositions) { + deleted |= deleteBlock(pva, pos); + } + return deleted; + } + + // Create a list of DataBlockRequests and sort it such that requests + // from the same (nested) shard are grouped contiguously. + // Despite the name, createReadRequests() works for delete requests as well ... + final DataBlockRequests requests = createReadRequests(gridPositions); + requests.removeDuplicates(); + + final List> split = requests.split(); + for (final DataBlockRequests subRequests : split) { + final boolean writeFully = subRequests.coversShard(); + final long[] key = subRequests.relativeGridPosition(); + final ReadData existingData = writeFully ? null : getExistingReadData(pva, key); + final ReadData modifiedData = deleteBlocksRecursive(existingData, subRequests); + if (existingData != null && modifiedData == null) { + deleted |= pva.remove(key); + } else if (modifiedData != existingData) { + pva.put(key, modifiedData); + deleted = true; + } + } + + return deleted; + } + + /** + * Bulk Delete operation on a shard. + * + * @param existingReadData encoded existing shard data (to decode and partially override) + * @param requests for blocks within the shard to be deleted + */ + private ReadData deleteBlocksRecursive( + final ReadData existingReadData, // may be null + final DataBlockRequests requests + ) { + assert !requests.requests.isEmpty(); + assert requests.level > 0; + + if (existingReadData == null) { + return null; + } + + final int level = requests.level(); + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[level]; + final long[] gridPos = requests.gridPosition(); + final RawShard shard = codec.decode(existingReadData, gridPos).getData(); + + boolean modified = false; + boolean shardElementSetToNull = false; + if ( level == 1 ) { + // Base case, delete the blocks + for (final DataBlockRequest request : requests) { + final long[] elementPos = request.position.relative(0); + if (shard.getElementData(elementPos) != null) { + shard.setElementData(null, elementPos); + modified = true; + shardElementSetToNull = true; + } + } + } else { // level > 1 + final List> split = requests.split(); + for (final DataBlockRequests subRequests : split) { + final boolean writeFully = subRequests.coversShard(); + final long[] elementPos = subRequests.relativeGridPosition(); + final ReadData existingElementData = writeFully ? null : shard.getElementData(elementPos); + final ReadData modifiedElementData = deleteBlocksRecursive(existingElementData, subRequests); + if (modifiedElementData != existingElementData) { + shard.setElementData(modifiedElementData, elementPos); + modified = true; + shardElementSetToNull |= (modifiedElementData == null); + } + } + } + + if (!modified) { + // No nested shard or DataBlock was modified. + // This shard remains unchanged. + return existingReadData; + } + + if (shardElementSetToNull) { + // At least one DataBlock or nested shard was removed. + // Check whether this shard becomes empty. + if (shard.index().allElementsNull()) { + // This shard is empty and should be removed. + return null; + } + } + + return codec.encode(new RawShardDataBlock(gridPos, shard)); + } + + + // // -- readShard ----------------------------------------------------------- diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java index ddaa6c761..1e37b496b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java @@ -49,8 +49,8 @@ import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; import org.janelia.saalfeldlab.n5.shard.DatasetAccess; +import org.janelia.saalfeldlab.n5.shard.DatasetAccessTest; import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; -import org.janelia.saalfeldlab.n5.shard.RawShardTest; import org.janelia.saalfeldlab.n5.shard.TestPositionValueAccess; import org.junit.Test; @@ -146,7 +146,7 @@ public void testEmptyBlock() throws Exception { final int[] blockSize = {0, 0}; final long[] gridPosition = {0, 0}; final N5BlockCodecInfo blockCodecInfo = new N5BlockCodecInfo(); - final RawShardTest.TestDatasetAttributes attributes = new RawShardTest.TestDatasetAttributes( + final DatasetAccessTest.TestDatasetAttributes attributes = new DatasetAccessTest.TestDatasetAttributes( new long[]{64, 64}, new int[]{8, 8}, DataType.UINT8, diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index 28139f4da..78ba66ec3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -275,6 +275,11 @@ public HttpRead return writer.deleteBlock(datasetPath, gridPosition); } + @Override public boolean deleteBlocks(String datasetPath, List gridPositions) throws N5Exception { + + return writer.deleteBlocks(datasetPath, gridPositions); + } + @Override public void writeSerializedBlock(Serializable object, String datasetPath, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { writer.writeSerializedBlock(object, datasetPath, datasetAttributes, gridPosition); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java similarity index 54% rename from src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java rename to src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java index 6d8b51338..382728854 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java @@ -28,7 +28,9 @@ */ package org.janelia.saalfeldlab.n5.shard; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; @@ -39,22 +41,32 @@ import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; -public class RawShardTest { +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; - public static void main(String[] args) { +public class DatasetAccessTest { - int[] datablockSize = {3, 3, 3}; - int[] level1ShardSize = {6, 6, 6}; - int[] level2ShardSize = {24, 24, 24}; + // DataBlocks are 3x3x3 + // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) + // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) + private final int[] dataBlockSize = {3, 3, 3}; + private final int[] level1ShardSize = {6, 6, 6}; + private final int[] level2ShardSize = {24, 24, 24}; + private final long[] datasetDimensions = {240, 240, 240}; + + private DatasetAccess datasetAccess; + + @Before + public void setup() { - // DataBlocks are 3x3x3 - // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) - // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + dataBlockSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -69,26 +81,24 @@ public static void main(String[] args) { new DataCodecInfo[] {new RawCompression()}, IndexLocation.START ); - - TestDatasetAttributes attributes = new TestDatasetAttributes( - new long[] {}, + final TestDatasetAttributes attributes = new TestDatasetAttributes( + datasetDimensions, level2ShardSize, DataType.INT8, c2, - new RawCompression()); + new RawCompression() + ); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + datasetAccess = attributes.datasetAccess(); + } - final PositionValueAccess store = new TestPositionValueAccess(); + @Test + public void tesWriteReadIndividual() { - // --------------------------------------------------------------- - // Some "tests" - // TODO: Turn into unit tests - // --------------------------------------------------------------- + final PositionValueAccess store = new TestPositionValueAccess(); // write some blocks, filled with constant values - final int[] dataBlockSize = c1.getInnerBlockSize(); datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 0, 0}, 1)); datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 0, 0}, 2)); datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 1, 0}, 3)); @@ -103,6 +113,50 @@ public static void main(String[] args) { checkBlock(datasetAccess.readBlock(store, new long[] {1, 1, 0}), true, 4); checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5); checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6); + } + + @Test + public void tesWriteReadBulk() { + + final PositionValueAccess store = new TestPositionValueAccess(); + + // write some blocks, filled with constant values + final List writeGridPositions = Arrays.asList(new long[][] { + {0, 0, 0}, {1, 0, 0}, {0, 1, 0}, {1, 1, 0}, {3, 2, 1}, {8, 4, 1} + }); + final List> writeBlocks = new ArrayList<>(); + for (int i = 0; i < writeGridPositions.size(); i++) { + writeBlocks.add(createDataBlock(dataBlockSize, writeGridPositions.get(i), 1 + i)); + } + datasetAccess.writeBlocks(store, writeBlocks); + + // verify that the written blocks can be read back with the correct values + final List readGridPositions = Arrays.asList(new long[][] { + {1, 0, 0}, {0, 0, 0}, {0, 1, 0}, {2, 4, 2}, {3, 2, 1}, {8, 4, 1} + }); + final List> readBlocks = datasetAccess.readBlocks(store, readGridPositions); + checkBlock(readBlocks.get(0), true, 2); + checkBlock(readBlocks.get(1), true, 1); + checkBlock(readBlocks.get(2), true, 3); + checkBlock(readBlocks.get(3), false, 4); + checkBlock(readBlocks.get(4), true, 5); + checkBlock(readBlocks.get(5), true, 6); + } + + @Test + public void tesDeleteBlock() { + + final PositionValueAccess store = new TestPositionValueAccess(); + + // write some blocks, filled with constant values + final List writeGridPositions = Arrays.asList(new long[][] { + {0, 0, 0}, {1, 0, 0}, {0, 1, 0}, {1, 1, 0}, {3, 2, 1}, {8, 4, 1} + }); + final List> writeBlocks = new ArrayList<>(); + for (int i = 0; i < writeGridPositions.size(); i++) { + writeBlocks.add(createDataBlock(dataBlockSize, writeGridPositions.get(i), 1 + i)); + } + datasetAccess.writeBlocks(store, writeBlocks); // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) datasetAccess.deleteBlock(store, new long[] {0, 0, 0}); @@ -110,40 +164,57 @@ public static void main(String[] args) { checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); // if a shard becomes empty the corresponding key should be deleted - if ( store.get(new long[] {1, 0, 0}) == null ) { - throw new IllegalStateException("expected non-null readData"); - } + assertNotNull(store.get(new long[] {1, 0, 0})); datasetAccess.deleteBlock(store, new long[] {8, 4, 1}); - if ( store.get(new long[] {1, 0, 0}) != null ) { - throw new IllegalStateException("expected null readData"); - } + assertNull(store.get(new long[] {1, 0, 0})); // deleting a non-existent block should not fail datasetAccess.deleteBlock(store, new long[] {0, 0, 8}); + } + + @Test + public void tesDeleteBlocks() { + + final PositionValueAccess store = new TestPositionValueAccess(); + + // write some blocks, filled with constant values + final List writeGridPositions = Arrays.asList(new long[][] { + {0, 0, 0}, {1, 0, 0}, {0, 1, 0}, {1, 1, 0}, {3, 2, 1}, {8, 4, 1} + }); + final List> writeBlocks = new ArrayList<>(); + for (int i = 0; i < writeGridPositions.size(); i++) { + writeBlocks.add(createDataBlock(dataBlockSize, writeGridPositions.get(i), 1 + i)); + } + datasetAccess.writeBlocks(store, writeBlocks); - System.out.println("all good"); + // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) + datasetAccess.deleteBlocks(store, Arrays.asList(new long[][] {{0, 0, 0}, {4, 2, 2}, {3, 2, 1}})); + checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); + checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); + + // if a shard becomes empty the corresponding key should be deleted + assertNotNull(store.get(new long[] {1, 0, 0})); + datasetAccess.deleteBlocks(store, Arrays.asList(new long[][] {{8, 4, 1}})); + assertNull(store.get(new long[] {1, 0, 0})); + + // deleting a non-existent block should not fail + datasetAccess.deleteBlocks(store, Arrays.asList(new long[] {0, 0, 8})); } private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { - if (dataBlock == null) { - if (expectedNonNull) { - throw new IllegalStateException("expected non-null dataBlock"); + if (expectedNonNull) { + assertNotNull("expected non-null dataBlock", dataBlock); + for (byte b : dataBlock.getData()) { + Assert.assertTrue("expected all values to be " + expectedFillValue, b == (byte) expectedFillValue); } } else { - if (!expectedNonNull) { - throw new IllegalStateException("expected null dataBlock"); - } - final byte[] bytes = dataBlock.getData(); - for (byte b : bytes) { - if (b != (byte) expectedFillValue) { - throw new IllegalStateException("expected all values to be " + expectedFillValue); - } - } + assertNull("expected null dataBlock", dataBlock); } } private static DataBlock createDataBlock(int[] size, long[] gridPosition, int fillValue) { + final byte[] bytes = new byte[DataBlock.getNumElements(size)]; Arrays.fill(bytes, (byte) fillValue); return new ByteArrayDataBlock(size, gridPosition, bytes);