Implement shardExists and readBlocksExist

LICENSE.md

@@ -1,6 +1,6 @@
 ## BSD 2-Clause License
 
-Copyright (c) 2017-2025, Stephan Saalfeld
+Copyright (c) 2017-2026, Stephan Saalfeld
 
 All rights reserved.
 

src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java

@@ -148,4 +148,27 @@ default String absoluteAttributesPath(final String normalPath) {
 
 		return getKeyValueAccess().compose(getURI(), normalPath, getAttributesKey());
 	}
+
+	@Override
+	default boolean shardExists(
+			final String pathName,
+			final DatasetAttributes datasetAttributes,
+			final long... gridPosition) throws N5Exception {
+
+		final String normalPath = N5URI.normalizeGroupPath(pathName);
+		final String blockPath = getKeyValueAccess().compose(getURI(), normalPath,
+				datasetAttributes.relativeBlockPath(gridPosition));
+		return getKeyValueAccess().isFile(blockPath);
+	}
+
+	@Override
+	default <T> List<DataBlock<T>> readBlocksExists(
+			final String pathName,
+			final DatasetAttributes datasetAttributes,
+			final List<long[]> gridPositions) throws N5Exception {
+
+		final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(),
+				N5URI.normalizeGroupPath(pathName), datasetAttributes);
+		return datasetAttributes.<T>getDatasetAccess().readBlocksExists(posKva, gridPositions);
+	}
 }

src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java

@@ -331,6 +331,52 @@ default <T> List<DataBlock<T>> readBlocks(
 		return blocks;
 	}
 
+	/**
+	 * Checks if a shard (or block for non-sharded datasets) exists at the
+	 * given grid position without reading the data.
+	 * <p>
+	 * This method only checks for the presence of the key value for the gridPosition, it does not
+	 * read or validate the contents.
+	 *
+	 * @param pathName
+	 *            dataset path
+	 * @param datasetAttributes
+	 *            the dataset attributes
+	 * @param gridPosition
+	 *            the shard grid position (or block position for non-sharded datasets)
+	 * @return true if the shard (or block) file exists
+	 * @throws N5Exception
+	 *             the exception
+	 */
+	boolean shardExists(
+			final String pathName,
+			final DatasetAttributes datasetAttributes,
+			final long... gridPosition) throws N5Exception;
+
+	/**
+	 * Reads and returns only the blocks that exist at the given grid positions.
+	 * <p>
+	 * For non-sharded datasets, this checks if each block file exists and reads those that do.
+	 * For sharded datasets, this reads the shard index first to determine which blocks exist,
+	 * then reads only those blocks.
+	 *
+	 * @param <T>
+	 *            the DataBlock data type
+	 * @param pathName
+	 *            dataset path
+	 * @param datasetAttributes
+	 *            the dataset attributes
+	 * @param gridPositions
+	 *            a list of grid positions to check
+	 * @return list of blocks that exist (excludes non-existent blocks)
+	 * @throws N5Exception
+	 *             the exception
+	 */
+	<T> List<DataBlock<T>> readBlocksExists(
+			final String pathName,
+			final DatasetAttributes datasetAttributes,
+			final List<long[]> gridPositions) throws N5Exception;
+
 	/**
 	 * Load a {@link DataBlock} as a {@link Serializable}. The offset is given
 	 * in

src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodec.java

@@ -6,7 +6,7 @@
 /**
  * A Codec that transforms the contents of a {@link DataBlock}.
  * <p>
- * This class is N5's analogue to Zarr's array -> array codec.
+ * This class is N5's analogue to Zarr's array-to-array codec.
  */
 public interface DatasetCodec<T> {
 

src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposeCodecInfo.java

@@ -20,7 +20,7 @@
  * 		result = [9, 7, 8] 		// permuted block size
  *
  * <p>
- * See the specification of <a href="https://zarr-specs.readthedocs.io/en/latest/v3/codecs/transpose/index.html#transpose-codec">Zarr's Transpose codec<a>.
+ * See the specification of <a href="https://zarr-specs.readthedocs.io/en/latest/v3/codecs/transpose/index.html#transpose-codec">Zarr's Transpose codec</a>.
  */
 @NameConfig.Name(value = TransposeCodecInfo.TYPE)
 public class TransposeCodecInfo implements DatasetCodecInfo {

src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java

@@ -60,6 +60,23 @@ public interface DatasetAccess<T> {
 
 	List<DataBlock<T>> readBlocks(PositionValueAccess pva, List<long[]> positions) throws N5IOException;
 
+	/**
+	 * Attempt to read all blocks at the given {@code positions}, and return a list of the blocks that exist.
+	 * <p>
+	 * For non-sharded datasets, this checks if each block value in the KV store exists, and read those that do.
+	 * For sharded datasets, this reads the shard index first to determine which blocks exist,
+	 * then reads only those blocks.
+	 *
+	 * @param pva
+	 * 		the position value access
+	 * @param positions
+	 * 		the list of block grid positions to check
+	 * @return list of blocks that exist (excludes non-existent blocks)
+	 * @throws N5IOException
+	 * 		if an error occurs
+	 */
+	List<DataBlock<T>> readBlocksExists(PositionValueAccess pva, List<long[]> positions) throws N5IOException;
+
 	void writeBlocks(PositionValueAccess pva, List<DataBlock<T>> blocks) throws N5IOException;
 
 //	TODO:

src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java

@@ -72,7 +72,7 @@ public NestedGrid getGrid() {
 	@Override
 	public DataBlock<T> readBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException {
 		final NestedPosition position = grid.nestedPosition(gridPosition);
-		return (DataBlock<T>)decodeWithDatasetCodecs(readBlockRecursive(pva.get(position.key()), position, grid.numLevels() - 1));
+		return readBlockRecursive(pva.get(position.key()), position, grid.numLevels() - 1);
 	}
 
 	private DataBlock<T> readBlockRecursive(
@@ -83,8 +83,9 @@ private DataBlock<T> readBlockRecursive(
 			return null;
 		} else if (level == 0) {
 			@SuppressWarnings("unchecked")
-			final BlockCodec<T> codec = (BlockCodec<T>) codecs[0];
-			return codec.decode(readData, position.absolute(0));
+			final BlockCodec<?> codec = codecs[0];
+			DataBlock<?> rawBlock = codec.decode(readData, position.absolute(0));
+			return (DataBlock<T>)decodeWithDatasetCodecs(rawBlock);
 		} else {
 			@SuppressWarnings("unchecked")
 			final BlockCodec<RawShard> codec = (BlockCodec<RawShard>) codecs[level];
@@ -126,6 +127,122 @@ public List<DataBlock<T>> readBlocks(PositionValueAccess pva, List<long[]> posit
 		return blocks;
 	}
 
+	/**
+	 * For a Non-Sharded dataset, reads all existing data blocks from the given positions.
+	 * The method checks if a block exists at a given position before attempting to read it.
+	 *
+	 * @param pva a {@code PositionValueAccess} instance to access data based on positions
+	 * @param positions a list of long arrays representing the grid positions of the blocks to be read
+	 * @return a list of {@code DataBlock<T>} objects for all valid and existing blocks at the specified positions
+	 */
+	private List<DataBlock<T>> readBlocksExistsNonSharded(PositionValueAccess pva, List<long[]> positions) {
+		final ArrayList<DataBlock<T>> blocks = new ArrayList<>();
+		for (long[] pos : positions) {
+			final NestedPosition position = grid.nestedPosition(pos);
+			if (pva.exists(position.key())) {
+				final DataBlock<T> block = readBlock(pva, pos);
+				if (block != null)
+					blocks.add(block);
+			}
+		}
+		return blocks;
+	}
+
+	/**
+	 * For a Sharded Dataset, reads all existing data blocks from the given positions.
+	 * The method groups blocks by shard, checks shard existence, reads shard data,
+	 * and determines which blocks within the shard exist before returning them.
+	 * If the KVA supports partial reads, this should take advantage of reading
+	 * the index only to check existence and only read blocks that exist.
+	 *
+	 * @param pva a {@code PositionValueAccess} instance to access data based on shard keys
+	 * @param positions a list of long arrays representing the grid positions of the blocks to be read
+	 * @return a list of {@code DataBlock<T>} objects for all valid and existing blocks at the specified positions
+	 */
+	private List<DataBlock<T>> readBlocksExistsSharded(PositionValueAccess pva, List<long[]> positions) {
+		/* group blocks by shard, read shard index to determine which blocks exist */
+		final List<NestedPosition> blockPositions = positions.stream().map(grid::nestedPosition).collect(Collectors.toList());
+		final int outermostLevel = grid.numLevels() - 1;
+		final Collection<List<NestedPosition>> blocksPerOutermostShard = groupInnerPositions(grid, blockPositions, outermostLevel);
+
+		final ArrayList<DataBlock<T>> blocks = new ArrayList<>();
+		for (List<NestedPosition> blocksInSingleShard : blocksPerOutermostShard) {
+			if (blocksInSingleShard.isEmpty())
+				continue;
+
+			final NestedPosition firstBlock = blocksInSingleShard.get(0);
+			final long[] shardKey = firstBlock.key();
+
+			/* check if the shard key value exists */
+			boolean shardExists = pva.exists(shardKey);
+			if (!shardExists)
+				continue;
+
+
+			final ReadData shardData = pva.get(shardKey);
+			final List<DataBlock<T>> shardBlocks = readBlocksExistsShardedRecursive(shardData, blocksInSingleShard, outermostLevel);
+			blocks.addAll(shardBlocks);
+		}
+		return blocks;
+	}
+
+	/**
+	 * Read only the blocks that exist within a shard by checking the shard index first.
+	 */
+	private List<DataBlock<T>> readBlocksExistsShardedRecursive(
+			final ReadData readData,
+			final List<NestedPosition> positions,
+			final int level) {
+
+		if (readData == null || level == 0 || positions.isEmpty())
+			return Collections.emptyList();
+
+		final NestedPosition firstBlock = positions.get(0);
+		final long[] shardPosition = firstBlock.absolute(level);
+
+		final BlockCodec<RawShard> codec = (BlockCodec<RawShard>) codecs[level];
+		final RawShard shard = codec.decode(readData, shardPosition).getData();
+
+		final ArrayList<DataBlock<T>> blocks = new ArrayList<>();
+		if (level == 1) {
+			/* base case: check the index and read blocks that exist */
+			for (NestedPosition blockPosition : positions) {
+				final long[] elementPos = blockPosition.relative(0);
+				if (shard.index().get(elementPos) != null) {
+					final ReadData elementData = shard.getElementData(elementPos);
+					final DataBlock<T> block = readBlockRecursive(elementData, blockPosition, 0);
+					if (block != null)
+						blocks.add(block);
+				}
+			}
+		} else {
+			/* nested shards: group by next level and recurse */
+			final Collection<List<NestedPosition>> nextLevelShards = groupInnerPositions(grid, positions, level - 1);
+			for (List<NestedPosition> innerPositions : nextLevelShards) {
+				if (innerPositions.isEmpty())
+					continue;
+				final NestedPosition firstInner = innerPositions.get(0);
+				final long[] innerShardPos = firstInner.relative(level - 1);
+				if (shard.index().get(innerShardPos) != null) {
+					final ReadData innerShardData = shard.getElementData(innerShardPos);
+					blocks.addAll(readBlocksExistsShardedRecursive(innerShardData, innerPositions, level - 1));
+				}
+			}
+		}
+
+		return blocks;
+	}
+
+	@Override
+	public List<DataBlock<T>> readBlocksExists(PositionValueAccess pva, List<long[]> positions) throws N5IOException {
+
+		boolean unsharded = grid.numLevels() == 1;
+		if (unsharded)
+			return readBlocksExistsNonSharded(pva, positions);
+		else
+			return readBlocksExistsSharded(pva, positions);
+	}
+
 	/**
 	 * Bulk Read operation on a shard. `positions` MUST all be in the same shard.
 	 * That is, for each `position` in `positions`, `position.absolute(level)` must be the same.
@@ -182,12 +299,11 @@ private List<DataBlock<T>> readShardRecursive(
 	public void writeBlock(final PositionValueAccess pva, final DataBlock<T> dataBlockArg) throws N5IOException {
 
 		@SuppressWarnings("unchecked")
-		final DataBlock<T> dataBlock = (DataBlock<T>)encodeWithDatasetCodecs(dataBlockArg);
-		final NestedPosition position = grid.nestedPosition(dataBlock.getGridPosition());
+		final NestedPosition position = grid.nestedPosition(dataBlockArg.getGridPosition());
 		final long[] key = position.key();
 
 		final ReadData existingData = getExistingReadData(pva, key);
-		final ReadData modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1);
+		final ReadData modifiedData = writeBlockRecursive(existingData, dataBlockArg, position, grid.numLevels() - 1);
 		pva.put(key, modifiedData);
 	}
 
@@ -199,7 +315,8 @@ private ReadData writeBlockRecursive(
 		if (level == 0) {
 			@SuppressWarnings("unchecked")
 			final BlockCodec<T> codec = (BlockCodec<T>) codecs[0];
-			return codec.encode(dataBlock);
+			DataBlock<T> datasetEncodedDataBlock = (DataBlock<T>)encodeWithDatasetCodecs(dataBlock);
+			return codec.encode(datasetEncodedDataBlock);
 		} else {
 			@SuppressWarnings("unchecked")
 			final BlockCodec<RawShard> codec = (BlockCodec<RawShard>) codecs[level];

src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java

@@ -59,6 +59,8 @@ public interface PositionValueAccess {
 
 	void put(long[] key, ReadData data) throws N5Exception.N5IOException;
 
+	boolean exists(long[] key) throws N5Exception.N5IOException;
+
 	boolean remove(long[] key) throws N5Exception.N5IOException;
 
 	public static PositionValueAccess fromKva(
@@ -105,6 +107,11 @@ public ReadData get(long[] key) throws N5IOException {
 			return kva.createReadData(absolutePath(key));
 		}
 
+		@Override
+		public boolean exists(long[] key) throws N5IOException {
+			return kva.isFile(absolutePath(key));
+		}
+
 		@Override
 		public void put(long[] key, ReadData data) throws N5IOException {