feat: add readShard writeShard methods

* add a few tests

Author: bogovicj

src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java

@@ -119,6 +119,24 @@ default <T> List<DataBlock<T>> readBlocks(
 		return convertedDatasetAttributes.<T> getDatasetAccess().readBlocks(posKva, blockPositions);
 	}
 
+	@Override
+	default <T> DataBlock<T> readShard(
+			final String pathName,
+			final DatasetAttributes datasetAttributes,
+			final long... gridPosition) throws N5Exception {
+
+		final DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);
+		final int shardLevel = convertedDatasetAttributes.getNestedBlockGrid().numLevels() - 1;
+		try {
+			final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName),
+					convertedDatasetAttributes);
+			return convertedDatasetAttributes.<T> getDatasetAccess().readShard(posKva, gridPosition, shardLevel);
+
+		} catch (N5Exception.N5NoSuchKeyException e) {
+			return null;
+		}
+	}
+
 	@Override
 	default String[] list(final String pathName) throws N5Exception {
 

src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java

@@ -280,7 +280,24 @@ default <T> void writeBlock(
 					"Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path,
 					e);
 		}
+	}
+
+	@Override
+	default <T> void writeShard(
+			final String path,
+			final DatasetAttributes datasetAttributes,
+			final DataBlock<T> shard) throws N5Exception {
 
+		final DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes);
+		final int shardLevel = convertedDatasetAttributes.getNestedBlockGrid().numLevels() - 1;
+		try {
+			final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes);
+			convertedDatasetAttributes.<T> getDatasetAccess().writeShard(posKva, shard, shardLevel);
+		} catch (final UncheckedIOException e) {
+			throw new N5IOException(
+					"Failed to write block " + Arrays.toString(shard.getGridPosition()) + " into dataset " + path,
+					e);
+		}
 	}
 
 	@Override

src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java

@@ -348,6 +348,31 @@ default <T> List<DataBlock<T>> readBlocks(
 		return blocks;
 	}
 
+	/**
+	 * Reads a shard as a {@link DataBlock}.
+	 * <p>
+	 * A "shard" is the largest level of the datasets {@link NestedGrid}
+	 * This method's behavior is identical to readBlock for un-sharded datasets.
+	 *
+	 * @param <T>
+	 *            the DataBlock data type
+	 * @param pathName
+	 *            dataset path
+	 * @param datasetAttributes
+	 *            the dataset attributes
+	 * @param gridPosition
+	 *            the position in the shard grid
+	 * @return the data block
+	 * @throws N5Exception
+	 *             the exception
+	 *
+	 * @see DatasetAttributes#getNestedBlockGrid()
+	 */
+	<T> DataBlock<T> readShard(
+			final String pathName,
+			final DatasetAttributes datasetAttributes,
+			final long... gridPosition) throws N5Exception;
+
 	/**
 	 * Checks if a shard (or block for non-sharded datasets) exists at the
 	 * given grid position without reading the data.

src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java

@@ -278,6 +278,25 @@ default <T> void writeBlocks(
 		}
 	}
 
+	/**
+	 * Writes a shard stored as a {@link DataBlock}.
+	 * <p>
+	 * A "shard" is the largest level of the datasets {@link NestedGrid}.
+	 * This method's behavior is identical to writeBlock for un-sharded datasets.
+	 *
+	 * @param datasetPath dataset path
+	 * @param datasetAttributes the dataset attributes
+	 * @param dataBlock the data block
+	 * @param <T> the data block data type
+	 * @throws N5Exception the exception
+	 *
+	 * @see DatasetAttributes#getNestedBlockGrid()
+	 */
+	<T> void writeShard(
+			final String pathName,
+			final DatasetAttributes datasetAttributes,
+			final DataBlock<T> dataBlock) throws N5Exception;
+
 	@FunctionalInterface
 	interface DataBlockSupplier<T> {
 

src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java

@@ -474,6 +474,33 @@ public void testWriteReadDoubleBlock() {
 		}
 	}
 
+	@Test
+	public void testWriteReadShard() {
+
+		// test that writeShard behaves the same as writeBlock
+		// for unsharded datasets
+
+		for (final Compression compression : getCompressions()) {
+			try (final N5Writer n5 = createTempN5Writer()) {
+
+				n5.createDataset(datasetName, dimensions, blockSize, DataType.INT16, compression);
+				final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName);
+				final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock);
+
+				n5.writeShard(datasetName, attributes, dataBlock);
+
+				// read with readShard
+				final DataBlock<?> loadedShard = n5.readShard(datasetName, attributes, 0, 0, 0);
+				assertArrayEquals(shortBlock, (short[])loadedShard.getData());
+
+				// read with readBlock
+				final DataBlock<?> loadedDataBlock = n5.readShard(datasetName, attributes, 0, 0, 0);
+				assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData());
+			}
+		}
+	}
+
+
 	@Test
 	public void testMode1WriteReadByteBlock() {
 

src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java

@@ -276,6 +276,10 @@ public <W extends GsonKeyValueN5Writer, R extends GsonKeyValueN5Reader> HttpRead
 		writer.writeBlock(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock);
 	}
 
+	@Override public <T> void writeShard(String datasetPath, DatasetAttributes datasetAttributes, DataBlock<T> dataBlock) throws N5Exception {
+		writer.writeShard(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock);
+	}
+
 	@Override public boolean deleteBlock(String datasetPath, long... gridPosition) throws N5Exception {
 
 		return writer.deleteBlock(datasetPath, gridPosition);

src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java

@@ -56,7 +56,9 @@
 import java.nio.file.NoSuchFileException;
 import java.util.*;
 import java.util.function.Function;
+import java.util.stream.IntStream;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -128,6 +130,11 @@ public void removeTempWriters() {
 
 	private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] blockSize) {
 
+		return getTestAttributes(DataType.UINT8, dimensions, shardSize, blockSize);
+	}
+
+	private DatasetAttributes getTestAttributes(DataType dataType, long[] dimensions, int[] shardSize, int[] blockSize) {
+
 		DefaultShardCodecInfo blockCodec = new DefaultShardCodecInfo(
 				blockSize,
 				new N5BlockCodecInfo(),
@@ -139,7 +146,7 @@ private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize,
 		return new DatasetAttributes(
 				dimensions,
 				shardSize,
-				DataType.UINT8,
+				dataType,
 				blockCodec);
 	}
 
@@ -412,6 +419,51 @@ public void writeReadBlockTest() {
 		}
 	}
 
+	@Test
+	public void writeReadShardTest() {
+
+		try ( final N5Writer n5 = tempN5Factory.createTempN5Writer() ) {
+
+			final int[] shardSize = new int[] {4,4};
+			final int shardN = 16;
+
+			final int[] blockSize = new int[] {2,2};
+			final int blockN = 4;
+
+			final String dataset = "writeReadShard";
+			DatasetAttributes attrs = getTestAttributes(DataType.INT32, new long[]{8, 8}, shardSize, blockSize);
+
+			final int[] shardData = range(shardN);
+			IntArrayDataBlock shard = new IntArrayDataBlock(shardSize, new long[]{0, 0}, shardData);
+
+			n5.writeShard(dataset, attrs, shard);
+			DataBlock<int[]> readShard = n5.readShard(dataset, attrs, 0, 0);
+			assertArrayEquals(shardData, readShard.getData());
+
+
+			/**
+			 * The 4x4 shard at (0,0)
+			 * and the 2x2 blocks it contains
+			 *
+			 *
+			 * 	0   1  |  2   3
+			 *  4   5  |  6	  7
+			 *  ----------------
+			 *  8	9  | 10	 11
+			 * 12  13  | 14  15
+			 */
+
+			assertArrayEquals(new int[]{0, 1, 4, 5}, (int[])n5.readBlock(dataset, attrs, 0, 0).getData());
+			assertArrayEquals(new int[]{2, 3, 6, 7}, (int[])n5.readBlock(dataset, attrs, 1, 0).getData());
+			assertArrayEquals(new int[]{8, 9, 12, 13}, (int[])n5.readBlock(dataset, attrs, 0, 1).getData());
+			assertArrayEquals(new int[]{10, 11, 14, 15}, (int[])n5.readBlock(dataset, attrs, 1, 1).getData());
+		}
+	}
+
+	private int[] range(int N) {
+		return IntStream.range(0, N).toArray();
+	}
+
 	/**
 	 * Checks how many read calls to the backend are performed for a particular readBlocks
 	 * call. At this time (Nov 4 2025), one read for the index, and one read per block are performed.
@@ -484,7 +536,7 @@ public void shardExistsTest() {
 
         final String dataset = "shardExists";
         writer.remove(dataset);
-        writer.createDataset(dataset, datasetAttributes);
+        DatasetAttributes attrs = writer.createDataset(dataset, datasetAttributes);
 
         final int[] blockSize = datasetAttributes.getBlockSize();
         final int numElements = blockSize[0] * blockSize[1];
@@ -497,7 +549,7 @@ public void shardExistsTest() {
         /* write blocks to shards (0,0), (1,0), and (2,2) */
         writer.writeBlocks(
                 dataset,
-                datasetAttributes,
+                attrs,
                 new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data),  /* shard (0, 0) */
                 new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data),  /* shard (1, 0) */
                 new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) /* shard (2, 2) */
@@ -507,7 +559,7 @@ public void shardExistsTest() {
 
         Function<long[], Boolean> assertShardExistsTracking = (gridPosition) -> {
             trackingWriter.resetAllTracking();
-            final Boolean exists = writer.shardExists(dataset, datasetAttributes, gridPosition);
+            final Boolean exists = writer.shardExists(dataset, attrs, gridPosition);
             assertEquals("isFileCheck incremented", 1, trackingWriter.getNumIsFileCalls());
             assertEquals("No Bytes Read", 0, trackingWriter.getTotalBytesRead());
             return exists;