diff --git a/LICENSE.md b/LICENSE.md index 46d2d7fac..0e55f3f4b 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ ## BSD 2-Clause License -Copyright (c) 2017-2025, Stephan Saalfeld +Copyright (c) 2017-2026, Stephan Saalfeld All rights reserved. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 8ede4da3a..926449104 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -148,4 +148,27 @@ default String absoluteAttributesPath(final String normalPath) { return getKeyValueAccess().compose(getURI(), normalPath, getAttributesKey()); } + + @Override + default boolean shardExists( + final String pathName, + final DatasetAttributes datasetAttributes, + final long... gridPosition) throws N5Exception { + + final String normalPath = N5URI.normalizeGroupPath(pathName); + final String blockPath = getKeyValueAccess().compose(getURI(), normalPath, + datasetAttributes.relativeBlockPath(gridPosition)); + return getKeyValueAccess().isFile(blockPath); + } + + @Override + default List> readBlocksExists( + final String pathName, + final DatasetAttributes datasetAttributes, + final List gridPositions) throws N5Exception { + + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), + N5URI.normalizeGroupPath(pathName), datasetAttributes); + return datasetAttributes.getDatasetAccess().readBlocksExists(posKva, gridPositions); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index c378dda68..6a571b044 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -331,6 +331,52 @@ default List> readBlocks( return blocks; } + /** + * Checks if a shard (or block for non-sharded datasets) exists at the + * given grid position without reading the data. + *

+ * This method only checks for the presence of the key value for the gridPosition, it does not + * read or validate the contents. + * + * @param pathName + * dataset path + * @param datasetAttributes + * the dataset attributes + * @param gridPosition + * the shard grid position (or block position for non-sharded datasets) + * @return true if the shard (or block) file exists + * @throws N5Exception + * the exception + */ + boolean shardExists( + final String pathName, + final DatasetAttributes datasetAttributes, + final long... gridPosition) throws N5Exception; + + /** + * Reads and returns only the blocks that exist at the given grid positions. + *

+ * For non-sharded datasets, this checks if each block file exists and reads those that do. + * For sharded datasets, this reads the shard index first to determine which blocks exist, + * then reads only those blocks. + * + * @param + * the DataBlock data type + * @param pathName + * dataset path + * @param datasetAttributes + * the dataset attributes + * @param gridPositions + * a list of grid positions to check + * @return list of blocks that exist (excludes non-existent blocks) + * @throws N5Exception + * the exception + */ + List> readBlocksExists( + final String pathName, + final DatasetAttributes datasetAttributes, + final List gridPositions) throws N5Exception; + /** * Load a {@link DataBlock} as a {@link Serializable}. The offset is given * in diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodec.java index dee1ba8ba..4c0ec2707 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodec.java @@ -6,7 +6,7 @@ /** * A Codec that transforms the contents of a {@link DataBlock}. *

- * This class is N5's analogue to Zarr's array -> array codec. + * This class is N5's analogue to Zarr's array-to-array codec. */ public interface DatasetCodec { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposeCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposeCodecInfo.java index fac0eb257..100e60f9a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposeCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposeCodecInfo.java @@ -20,7 +20,7 @@ * result = [9, 7, 8] // permuted block size * *

- * See the specification of Zarr's Transpose codec. + * See the specification of Zarr's Transpose codec. */ @NameConfig.Name(value = TransposeCodecInfo.TYPE) public class TransposeCodecInfo implements DatasetCodecInfo { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index a320acf1e..4d0b1e5b0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -60,6 +60,23 @@ public interface DatasetAccess { List> readBlocks(PositionValueAccess pva, List positions) throws N5IOException; + /** + * Attempt to read all blocks at the given {@code positions}, and return a list of the blocks that exist. + *

+ * For non-sharded datasets, this checks if each block value in the KV store exists, and read those that do. + * For sharded datasets, this reads the shard index first to determine which blocks exist, + * then reads only those blocks. + * + * @param pva + * the position value access + * @param positions + * the list of block grid positions to check + * @return list of blocks that exist (excludes non-existent blocks) + * @throws N5IOException + * if an error occurs + */ + List> readBlocksExists(PositionValueAccess pva, List positions) throws N5IOException; + void writeBlocks(PositionValueAccess pva, List> blocks) throws N5IOException; // TODO: diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 942507dd7..707536346 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -72,7 +72,7 @@ public NestedGrid getGrid() { @Override public DataBlock readBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { final NestedPosition position = grid.nestedPosition(gridPosition); - return (DataBlock)decodeWithDatasetCodecs(readBlockRecursive(pva.get(position.key()), position, grid.numLevels() - 1)); + return readBlockRecursive(pva.get(position.key()), position, grid.numLevels() - 1); } private DataBlock readBlockRecursive( @@ -83,8 +83,9 @@ private DataBlock readBlockRecursive( return null; } else if (level == 0) { @SuppressWarnings("unchecked") - final BlockCodec codec = (BlockCodec) codecs[0]; - return codec.decode(readData, position.absolute(0)); + final BlockCodec codec = codecs[0]; + DataBlock rawBlock = codec.decode(readData, position.absolute(0)); + return (DataBlock)decodeWithDatasetCodecs(rawBlock); } else { @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[level]; @@ -126,6 +127,122 @@ public List> readBlocks(PositionValueAccess pva, List posit return blocks; } + /** + * For a Non-Sharded dataset, reads all existing data blocks from the given positions. + * The method checks if a block exists at a given position before attempting to read it. + * + * @param pva a {@code PositionValueAccess} instance to access data based on positions + * @param positions a list of long arrays representing the grid positions of the blocks to be read + * @return a list of {@code DataBlock} objects for all valid and existing blocks at the specified positions + */ + private List> readBlocksExistsNonSharded(PositionValueAccess pva, List positions) { + final ArrayList> blocks = new ArrayList<>(); + for (long[] pos : positions) { + final NestedPosition position = grid.nestedPosition(pos); + if (pva.exists(position.key())) { + final DataBlock block = readBlock(pva, pos); + if (block != null) + blocks.add(block); + } + } + return blocks; + } + + /** + * For a Sharded Dataset, reads all existing data blocks from the given positions. + * The method groups blocks by shard, checks shard existence, reads shard data, + * and determines which blocks within the shard exist before returning them. + * If the KVA supports partial reads, this should take advantage of reading + * the index only to check existence and only read blocks that exist. + * + * @param pva a {@code PositionValueAccess} instance to access data based on shard keys + * @param positions a list of long arrays representing the grid positions of the blocks to be read + * @return a list of {@code DataBlock} objects for all valid and existing blocks at the specified positions + */ + private List> readBlocksExistsSharded(PositionValueAccess pva, List positions) { + /* group blocks by shard, read shard index to determine which blocks exist */ + final List blockPositions = positions.stream().map(grid::nestedPosition).collect(Collectors.toList()); + final int outermostLevel = grid.numLevels() - 1; + final Collection> blocksPerOutermostShard = groupInnerPositions(grid, blockPositions, outermostLevel); + + final ArrayList> blocks = new ArrayList<>(); + for (List blocksInSingleShard : blocksPerOutermostShard) { + if (blocksInSingleShard.isEmpty()) + continue; + + final NestedPosition firstBlock = blocksInSingleShard.get(0); + final long[] shardKey = firstBlock.key(); + + /* check if the shard key value exists */ + boolean shardExists = pva.exists(shardKey); + if (!shardExists) + continue; + + + final ReadData shardData = pva.get(shardKey); + final List> shardBlocks = readBlocksExistsShardedRecursive(shardData, blocksInSingleShard, outermostLevel); + blocks.addAll(shardBlocks); + } + return blocks; + } + + /** + * Read only the blocks that exist within a shard by checking the shard index first. + */ + private List> readBlocksExistsShardedRecursive( + final ReadData readData, + final List positions, + final int level) { + + if (readData == null || level == 0 || positions.isEmpty()) + return Collections.emptyList(); + + final NestedPosition firstBlock = positions.get(0); + final long[] shardPosition = firstBlock.absolute(level); + + final BlockCodec codec = (BlockCodec) codecs[level]; + final RawShard shard = codec.decode(readData, shardPosition).getData(); + + final ArrayList> blocks = new ArrayList<>(); + if (level == 1) { + /* base case: check the index and read blocks that exist */ + for (NestedPosition blockPosition : positions) { + final long[] elementPos = blockPosition.relative(0); + if (shard.index().get(elementPos) != null) { + final ReadData elementData = shard.getElementData(elementPos); + final DataBlock block = readBlockRecursive(elementData, blockPosition, 0); + if (block != null) + blocks.add(block); + } + } + } else { + /* nested shards: group by next level and recurse */ + final Collection> nextLevelShards = groupInnerPositions(grid, positions, level - 1); + for (List innerPositions : nextLevelShards) { + if (innerPositions.isEmpty()) + continue; + final NestedPosition firstInner = innerPositions.get(0); + final long[] innerShardPos = firstInner.relative(level - 1); + if (shard.index().get(innerShardPos) != null) { + final ReadData innerShardData = shard.getElementData(innerShardPos); + blocks.addAll(readBlocksExistsShardedRecursive(innerShardData, innerPositions, level - 1)); + } + } + } + + return blocks; + } + + @Override + public List> readBlocksExists(PositionValueAccess pva, List positions) throws N5IOException { + + boolean unsharded = grid.numLevels() == 1; + if (unsharded) + return readBlocksExistsNonSharded(pva, positions); + else + return readBlocksExistsSharded(pva, positions); + } + /** * Bulk Read operation on a shard. `positions` MUST all be in the same shard. * That is, for each `position` in `positions`, `position.absolute(level)` must be the same. @@ -182,12 +299,11 @@ private List> readShardRecursive( public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlockArg) throws N5IOException { @SuppressWarnings("unchecked") - final DataBlock dataBlock = (DataBlock)encodeWithDatasetCodecs(dataBlockArg); - final NestedPosition position = grid.nestedPosition(dataBlock.getGridPosition()); + final NestedPosition position = grid.nestedPosition(dataBlockArg.getGridPosition()); final long[] key = position.key(); final ReadData existingData = getExistingReadData(pva, key); - final ReadData modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1); + final ReadData modifiedData = writeBlockRecursive(existingData, dataBlockArg, position, grid.numLevels() - 1); pva.put(key, modifiedData); } @@ -199,7 +315,8 @@ private ReadData writeBlockRecursive( if (level == 0) { @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[0]; - return codec.encode(dataBlock); + DataBlock datasetEncodedDataBlock = (DataBlock)encodeWithDatasetCodecs(dataBlock); + return codec.encode(datasetEncodedDataBlock); } else { @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[level]; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java index d1ab7f0a2..382eaa909 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java @@ -59,6 +59,8 @@ public interface PositionValueAccess { void put(long[] key, ReadData data) throws N5Exception.N5IOException; + boolean exists(long[] key) throws N5Exception.N5IOException; + boolean remove(long[] key) throws N5Exception.N5IOException; public static PositionValueAccess fromKva( @@ -105,6 +107,11 @@ public ReadData get(long[] key) throws N5IOException { return kva.createReadData(absolutePath(key)); } + @Override + public boolean exists(long[] key) throws N5IOException { + return kva.isFile(absolutePath(key)); + } + @Override public void put(long[] key, ReadData data) throws N5IOException { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 58bcf04e5..e48fc918c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -42,11 +42,11 @@ import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; -import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.*; +import org.janelia.saalfeldlab.n5.codec.transpose.TransposeCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; +import org.janelia.saalfeldlab.n5.codec.transpose.TransposeCodecInfo; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -76,6 +76,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; @RunWith(Parameterized.class) public class ShardTest { @@ -488,34 +489,409 @@ public void numReadsTest() { System.out.println(""); } + @Test + public void shardExistsTest() { + + final N5Writer writer = tempN5Factory.createTempN5Writer(); + + final DatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + new int[]{2, 2} + ); + + final String dataset = "shardExists"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); + + final int[] blockSize = datasetAttributes.getBlockSize(); + final int numElements = blockSize[0] * blockSize[1]; + + final byte[] data = new byte[numElements]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)(i); + } + + /* write blocks to shards (0,0), (1,0), and (2,2) */ + writer.writeBlocks( + dataset, + datasetAttributes, + new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), /* shard (0, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), /* shard (1, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) /* shard (2, 2) */ + ); + + /* shards that should exist */ + Assert.assertTrue("Shard (0,0) should exist", writer.shardExists(dataset, datasetAttributes, 0, 0)); + Assert.assertTrue("Shard (1,0) should exist", writer.shardExists(dataset, datasetAttributes, 1, 0)); + Assert.assertTrue("Shard (2,2) should exist", writer.shardExists(dataset, datasetAttributes, 2, 2)); + + /* shards that should NOT exist */ + Assert.assertFalse("Shard (0,1) should not exist", writer.shardExists(dataset, datasetAttributes, 0, 1)); + Assert.assertFalse("Shard (1,1) should not exist", writer.shardExists(dataset, datasetAttributes, 1, 1)); + Assert.assertFalse("Shard (2,0) should not exist", writer.shardExists(dataset, datasetAttributes, 2, 0)); + Assert.assertFalse("Shard (0,2) should not exist", writer.shardExists(dataset, datasetAttributes, 0, 2)); + } + + @Test + public void readBlocksExistsTest() { + + final TrackingN5Writer writer = (TrackingN5Writer)tempN5Factory.createTempN5Writer(); + + final DatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + new int[]{2, 2} + ); + + final String dataset = "readBlocksExists"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); + + final int[] blockSize = datasetAttributes.getBlockSize(); + final int numElements = blockSize[0] * blockSize[1]; + + final byte[] data = new byte[numElements]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)(i); + } + + /* write only some blocks: (0,0), (1,1) in shard (0,0) and (4,0) in shard (1,0) */ + final long[][] blockPositions = new long[][]{ + {0, 0}, + {1, 1}, + {4, 0} + }; + writer.writeBlocks( + dataset, + datasetAttributes, + new ByteArrayDataBlock(blockSize, blockPositions[0], data), + new ByteArrayDataBlock(blockSize, blockPositions[1], data), + new ByteArrayDataBlock(blockSize, blockPositions[2], data) + ); + + /* request a mix of existing and non-existing blocks */ + final List requestedPositions = Arrays.asList( + new long[]{0, 0}, /* exists in shard (0,0) */ + new long[]{0, 1}, /* does NOT exist in shard (0,0) */ + new long[]{1, 1}, /* exists in shard (0,0) */ + new long[]{4, 0}, /* exists in shard (1,0) */ + new long[]{5, 0}, /* does NOT exist in shard (1,0) */ + new long[]{10, 10} /* shard (2,2) does NOT exist at all */ + ); + + final List> existingBlocks = writer.readBlocksExists(dataset, datasetAttributes, requestedPositions); + + /* should only return the 3 existing blocks */ + Assert.assertEquals("Should return exactly 3 existing blocks", 3, existingBlocks.size()); + + /* sort returned blocks by grid position for comparison */ + existingBlocks.sort((a, b) -> { + final long[] posA = a.getGridPosition(); + final long[] posB = b.getGridPosition(); + for (int i = 0; i < posA.length; i++) { + int cmp = Long.compare(posA[i], posB[i]); + if (cmp != 0) + return cmp; + } + return 0; + }); + + + + /* verify grid positions and data of returned blocks */ + for (int i = 0; i < existingBlocks.size(); i++) { + final DataBlock block = existingBlocks.get(i); + Assert.assertArrayEquals("Block grid position should match", blockPositions[i], block.getGridPosition()); + Assert.assertArrayEquals("Block data should match", data, (byte[])block.getData()); + } + } + + @Test + public void readBlocksExistsNoReadBeforeExistenceCheckTest() { + + final TrackingN5Writer writer = (TrackingN5Writer)tempN5Factory.createTempN5Writer(); + + final DatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + new int[]{2, 2} + ); + + final String dataset = "readBlocksExistsNoReadBeforeCheck"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); + + final int[] blockSize = datasetAttributes.getBlockSize(); + final int numElements = blockSize[0] * blockSize[1]; + + final byte[] data = new byte[numElements]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)(i); + } + + writer.writeBlock(dataset, datasetAttributes, new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data)); + + writer.resetNumMaterializeCalls(); + writer.resetNumIsFileCalls(); + + final List nonExistentShardPositions = Arrays.asList( + new long[]{8, 8}, + new long[]{9, 9}, + new long[]{10, 10} + ); + + final List> blocksFromNonExistentShard = writer.readBlocksExists( + dataset, datasetAttributes, nonExistentShardPositions); + + Assert.assertTrue("Should return no blocks from non-existent shard", blocksFromNonExistentShard.isEmpty()); + Assert.assertTrue("Should have checked existence via isFile", writer.getNumIsFileCalls() > 0); + Assert.assertEquals("Should not materialize data for non-existent shard", 0, writer.getNumMaterializeCalls()); + } + + @Test + public void readBlocksExistsOnlyReadsExistingBlocksInShardTest() { + + final TrackingN5Writer writer = (TrackingN5Writer)tempN5Factory.createTempN5Writer(); + + /* + * shard size 8x8 with block size 2x2 means 16 block slots per shard + * N5 block header is 12 bytes, block data is 4 bytes = 16 bytes per block + * index is 16 blocks * 16 bytes (2 longs per block) = 256 bytes + */ + final int n5HeaderSizeBytes = 12; + + DefaultShardCodecInfo blockCodec = new DefaultShardCodecInfo( + new int[]{2, 2}, + new N5BlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + IndexLocation.END); + + final DatasetAttributes datasetAttributes = new DatasetAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + DataType.UINT8, + blockCodec); + + final String dataset = "readBlocksExistsOnlyReadsExisting"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); + + final int[] blockSize = datasetAttributes.getBlockSize(); + final int numElements = blockSize[0] * blockSize[1]; + final int blockDataSize = numElements + n5HeaderSizeBytes; + final int indexSize = 16 * 16; /* 16 blocks * 2 longs * 8 bytes */ + + final byte[] data = new byte[numElements]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)(i); + } + + /* write only 2 blocks to the shard (0,0) out of 16 possible slots */ + writer.writeBlocks( + dataset, + datasetAttributes, + new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data) + ); + + final KeyValueAccess kva = writer.getKeyValueAccess(); + final String shardPath = kva.compose(writer.getURI(), dataset, "0", "0"); + final long fullShardSize = kva.size(shardPath); + + /* the shard should contain: index (256 bytes) + 2 blocks (16 bytes each) = 288 bytes */ + final long expectedShardSize = indexSize + 2 * blockDataSize; + Assert.assertEquals("Shard size should match expected", expectedShardSize, fullShardSize); + + writer.resetAllTracking(); + + /* + * request 4 blocks: 2 exist, 2 don't exist (but are in the same shard) + * we should only read: index + 2 existing blocks, NOT the non-existent block data + */ + final List requestedPositions = Arrays.asList( + new long[]{0, 0}, /* exists */ + new long[]{0, 1}, /* does NOT exist */ + new long[]{1, 0}, /* does NOT exist */ + new long[]{1, 1} /* exists */ + ); + + final List> existingBlocks = writer.readBlocksExists( + dataset, datasetAttributes, requestedPositions); + + Assert.assertEquals("Should return exactly 2 existing blocks", 2, existingBlocks.size()); + + /* + * the total bytes read should be: index + 2 blocks + * this verifies we didn't try to read non-existent blocks + */ + final long expectedBytesRead = indexSize + 2 * blockDataSize; + final long actualBytesRead = writer.getTotalBytesRead(); + + Assert.assertEquals( + "Should only read index + existing blocks, not non-existent block data", + expectedBytesRead, + actualBytesRead); + } + + /** + * Tests that dataset codecs (like transpose) are correctly applied when + * reading blocks from a sharded dataset using bulk read operations. + */ + @Test + public void readBlocksAppliesDatasetCodecsTest() { + + final N5Writer writer = tempN5Factory.createTempN5Writer(); + + /* 2x3 block with transpose codec [1, 0] will swap to 3x2 in storage */ + final int[] blockSize = new int[]{2, 3}; + final int[] shardSize = new int[]{4, 6}; + final long[] dimensions = new long[]{4, 6}; + + + final DefaultShardCodecInfo shardCodec = new DefaultShardCodecInfo( + blockSize, + new N5BlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + IndexLocation.END); + + AtomicInteger transposeEncodeCount = new AtomicInteger(0); + AtomicInteger transposeDecodeCount = new AtomicInteger(0); + + final TransposeCodecInfo trackingTransposeCodec = new TransposeCodecInfo(new int[]{1, 0}) { + @Override + public DatasetCodec create(DatasetAttributes datasetAttributes) { + + return new TransposeCodec(datasetAttributes.getDataType(), getOrder()) { + @Override + public DataBlock encode(DataBlock dataBlock) { + transposeEncodeCount.incrementAndGet(); + return super.encode(dataBlock); + } + + @Override + public DataBlock decode(DataBlock dataBlock) { + transposeDecodeCount.incrementAndGet(); + return super.decode(dataBlock); + } + }; + } + }; + + final DatasetAttributes datasetAttributes = new DatasetAttributes( + dimensions, + shardSize, + DataType.UINT8, + shardCodec, + new DatasetCodecInfo[]{trackingTransposeCodec}); + + final String dataset = "readBlocksDatasetCodecs"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); + + /* + * Create asymmetric data so we can detect if transpose was applied. + * For a 2x3 block, we'll use row-major data: + * [0, 1] + * [2, 3] + * [4, 5] + * If transpose is NOT applied on read, data would be in storage order + * (3x2 transposed layout). + */ + final byte[] originalData = new byte[]{0, 1, 2, 3, 4, 5}; + final ByteArrayDataBlock block = new ByteArrayDataBlock(blockSize, new long[]{0, 0}, originalData); + + writer.writeBlocks(dataset, datasetAttributes, block); + Assert.assertEquals("Transpose Encode called once", 1, transposeEncodeCount.get()); + Assert.assertEquals("Transpose Decode not called", 0, transposeDecodeCount.get()); + + /* read back using bulk read (readBlocks) which uses the sharded path */ + final List positions = Collections.singletonList(new long[]{0, 0}); + final List> readBlocks = writer.readBlocks(dataset, datasetAttributes, positions); + + Assert.assertEquals("Should read exactly 1 block", 1, readBlocks.size()); + + final DataBlock readBlock = readBlocks.get(0); + final byte[] readData = (byte[])readBlock.getData(); + + + Assert.assertEquals("Transpose Encode called once still", 1, transposeEncodeCount.get()); + Assert.assertEquals("Transpose Decode called once now", 1, transposeDecodeCount.get()); + + /* + * If decodeWithDatasetCodecs was applied, the data should match original. + * If NOT applied, data would still be in transposed storage order. + */ + Assert.assertArrayEquals( + "Block data should match original after dataset codec decode", + originalData, + readData); + } + /** - * An N5Writer that tracks the number of materialize calls performed by + * An N5Writer that tracks the number of materialize calls performed by * its underlying key value access. */ public static class TrackingN5Writer extends N5KeyValueWriter { + + final TrackingFileSystemKeyValueAccess tkva; public TrackingN5Writer(String basePath) { - super( new TrackingFileSystemKeyValueAccess(FileSystems.getDefault()), - basePath, new GsonBuilder(), false); + super( new TrackingFileSystemKeyValueAccess(FileSystems.getDefault()), basePath, new GsonBuilder(), false); + tkva = (TrackingFileSystemKeyValueAccess)getKeyValueAccess(); } public void resetNumMaterializeCalls() { - ((TrackingFileSystemKeyValueAccess)super.getKeyValueAccess()).numMaterializeCalls = 0; + tkva.numMaterializeCalls = 0; } public int getNumMaterializeCalls() { - return ((TrackingFileSystemKeyValueAccess)super.getKeyValueAccess()).numMaterializeCalls; + return tkva.numMaterializeCalls; + } + + public void resetNumIsFileCalls() { + tkva.numIsFileCalls = 0; + } + + public int getNumIsFileCalls() { + return tkva.numIsFileCalls; + } + + public void resetTotalBytesRead() { + tkva.totalBytesRead = 0; + } + + public long getTotalBytesRead() { + return tkva.totalBytesRead; + } + + public void resetAllTracking() { + tkva.numMaterializeCalls = 0; + tkva.numIsFileCalls = 0; + tkva.totalBytesRead = 0; } } private static class TrackingFileSystemKeyValueAccess extends FileSystemKeyValueAccess { private int numMaterializeCalls = 0; + private int numIsFileCalls = 0; + private long totalBytesRead = 0; protected TrackingFileSystemKeyValueAccess(FileSystem fileSystem) { super(fileSystem); } + @Override + public boolean isFile(String normalPath) { + numIsFileCalls++; + return super.isFile(normalPath); + } + @Override public ReadData createReadData(final String normalPath) { return new KeyValueAccessReadData(new TrackingFileLazyRead(normalPath)); @@ -551,6 +927,7 @@ public ReadData materialize(final long offset, final long length) { throw new IndexOutOfBoundsException(); final int sz = (int)(length < 0 ? channelSize : length); + totalBytesRead += sz; final byte[] data = new byte[sz]; final ByteBuffer buf = ByteBuffer.wrap(data); channel.read(buf); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java index 3590ebc92..639f03c40 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java @@ -34,7 +34,6 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; public class TestPositionValueAccess implements PositionValueAccess { @@ -52,6 +51,11 @@ public void put(final long[] key, final ReadData data) { map.put(new Key(key), bytes); } + @Override + public boolean exists(long[] key) throws N5IOException { + return map.containsKey(new Key(key)); + } + @Override public boolean remove(final long[] key) throws N5IOException { return map.remove(new Key(key)) != null;