Write full shards, revise DatasetCodec

.github/workflows/build.yml

@@ -4,11 +4,13 @@ on:
   push:
     branches:
       - master
+      - development
     tags:
       - "*-[0-9]+.*"
   pull_request:
     branches:
       - master
+      - development
 
 jobs:
   build:

LICENSE.md

@@ -1,6 +1,6 @@
 ## BSD 2-Clause License
 
-Copyright (c) 2017-2025, Stephan Saalfeld
+Copyright (c) 2017-2026, Stephan Saalfeld
 
 All rights reserved.
 

src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java

@@ -197,7 +197,7 @@ protected DatasetAccess<?> createDatasetAccess() {
 		// The inner-most codec (the DataBlock codec) is at index 0.
 		final int[][] blockSizes = new int[m][];
 
-		// NestedGrid validates block sizes, so instantiate it before creating the blockCodecs  
+		// NestedGrid validates block sizes, so instantiate it before creating the blockCodecs
 		// blockCodecInfo.create below could fail unexpecedly with invalid
 		// blockSizes so validate first
 		blockSizes[m - 1] = outerBlockSize;
@@ -211,27 +211,24 @@ protected DatasetAccess<?> createDatasetAccess() {
 		BlockCodecInfo currentBlockCodecInfo = blockCodecInfo;
 		DataCodecInfo[] currentDataCodecInfos = dataCodecInfos;
 
-		final NestedGrid grid = new NestedGrid(blockSizes);
+		final NestedGrid grid = new NestedGrid(blockSizes, dimensions);
 		final BlockCodec<?>[] blockCodecs = new BlockCodec[m];
 		for (int l = m - 1; l >= 0; --l) {
 			blockCodecs[l] = currentBlockCodecInfo.create(dataType, blockSizes[l], currentDataCodecInfos);
 			if (l > 0) {
-				final ShardCodecInfo info = (ShardCodecInfo)currentBlockCodecInfo;
+				final ShardCodecInfo info = (ShardCodecInfo) currentBlockCodecInfo;
 				currentBlockCodecInfo = info.getInnerBlockCodecInfo();
 				currentDataCodecInfos = info.getInnerDataCodecInfos();
 			}
 		}
 
-		final DatasetCodec[] datasetCodecs;
 		if (datasetCodecInfos != null) {
-			datasetCodecs = new DatasetCodec[datasetCodecInfos.length];
-			for (int i = 0; i < datasetCodecInfos.length; i++)
-				datasetCodecs[i] = datasetCodecInfos[i].create(this);
-
-		} else
-			datasetCodecs = new DatasetCodec[0];
+			for (final DatasetCodecInfo info : datasetCodecInfos) {
+				blockCodecs[0] = DatasetCodec.concatenate(info.create(this), (BlockCodec) blockCodecs[0]);
+			}
+		}
 
-		return new DefaultDatasetAccess<>(grid, blockCodecs, datasetCodecs);
+		return new DefaultDatasetAccess<>(grid, blockCodecs);
 	}
 
 	private static int nestingDepth(BlockCodecInfo info) {

src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java

@@ -148,4 +148,16 @@ default String absoluteAttributesPath(final String normalPath) {
 
 		return getKeyValueAccess().compose(getURI(), normalPath, getAttributesKey());
 	}
+
+	@Override
+	default boolean shardExists(
+			final String pathName,
+			final DatasetAttributes datasetAttributes,
+			final long... gridPosition) throws N5Exception {
+
+		final String normalPath = N5URI.normalizeGroupPath(pathName);
+		final String blockPath = getKeyValueAccess().compose(getURI(), normalPath,
+				datasetAttributes.relativeBlockPath(gridPosition));
+		return getKeyValueAccess().isFile(blockPath);
+	}
 }

src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java

@@ -222,7 +222,7 @@ default <T> void writeRegion(
 			final boolean writeFully) throws N5Exception {
 		try {
 			final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), datasetAttributes);
-			datasetAttributes.<T>getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, datasetAttributes.getDimensions(), writeFully);
+			datasetAttributes.<T>getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, writeFully);
 		} catch (final UncheckedIOException e) {
 			throw new N5IOException(
 					"Failed to write blocks into dataset " + datasetPath, e);
@@ -240,7 +240,7 @@ default <T> void writeRegion(
 			final ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException {
 		try {
 			final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), datasetAttributes);
-			datasetAttributes.<T>getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, datasetAttributes.getDimensions(), writeFully, exec);
+			datasetAttributes.<T>getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, writeFully, exec);
 		} catch (final UncheckedIOException e) {
 			throw new N5IOException(
 					"Failed to write blocks into dataset " + datasetPath, e);

src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java

@@ -331,6 +331,27 @@ default <T> List<DataBlock<T>> readBlocks(
 		return blocks;
 	}
 
+	/**
+	 * Checks if a shard (or block for non-sharded datasets) exists at the
+	 * given grid position without reading the data.
+	 * <p>
+	 * This method only checks for the presence of the key value for the gridPosition, it does not
+	 * read or validate the contents.
+	 *
+	 * @param pathName
+	 *            dataset path
+	 * @param datasetAttributes
+	 *            the dataset attributes
+	 * @param gridPosition
+	 *            the shard grid position (or block position for non-sharded datasets)
+	 * @return true if the shard (or block) file exists
+	 * @throws N5Exception
+	 *             the exception
+	 */
+	boolean shardExists(
+			final String pathName,
+			final DatasetAttributes datasetAttributes,
+			final long... gridPosition) throws N5Exception;
 	/**
 	 * Load a {@link DataBlock} as a {@link Serializable}. The offset is given
 	 * in

src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodec.java

@@ -2,20 +2,46 @@
 
 import org.janelia.saalfeldlab.n5.DataBlock;
 import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
 
 /**
  * A Codec that transforms the contents of a {@link DataBlock}.
  * <p>
- * This class is N5's analogue to Zarr's array -> array codec.
+ * This class is N5's analogue to Zarr's array-to-array codec.
+ *
+ * @param <S> source data type (data contained in decoded blocks)
+ * @param <T> target data type (data contained in encoded blocks)
  */
-public interface DatasetCodec<T> {
+public interface DatasetCodec<S, T> {
 
 	// TODO Name ideas:
 	// "ImageCodec"?
 	// BlockTransformationCodec
 
-	DataBlock<?> encode(DataBlock<T> block) throws N5IOException;
+	DataBlock<T> encode(DataBlock<S> block) throws N5IOException;
 
-	DataBlock<T> decode(DataBlock<?> dataBlock) throws N5IOException;
+	DataBlock<S> decode(DataBlock<T> dataBlock) throws N5IOException;
 
+	/**
+	 * Create a {@code BlockCodec} that, for encoding, first applies {@code
+	 * datasetCodec} and then {@code blockCodec} (and does the same in reverse
+	 * order for decoding).
+	 *
+	 * @return the concatenated BlockCodec
+	 */
+	static <S, T> BlockCodec<S> concatenate(final DatasetCodec<S, T> datasetCodec, final BlockCodec<T> blockCodec) {
+
+		return new BlockCodec<S>() {
+
+			@Override
+			public ReadData encode(final DataBlock<S> dataBlock) throws N5IOException {
+				return blockCodec.encode(datasetCodec.encode(dataBlock));
+			}
+
+			@Override
+			public DataBlock<S> decode(final ReadData readData, final long[] gridPosition) throws N5IOException {
+				return datasetCodec.decode(blockCodec.decode(readData, gridPosition));
+			}
+		};
+	}
 }

src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodecInfo.java

@@ -31,8 +31,8 @@
 import org.janelia.saalfeldlab.n5.DatasetAttributes;
 import org.janelia.saalfeldlab.n5.serialization.NameConfig;
 
-@NameConfig.Prefix("data-codec")
+@NameConfig.Prefix("data-codec") // TODO: is this Prefix correct?
 public interface DatasetCodecInfo extends CodecInfo {
 
-	<T> DatasetCodec<T> create(final DatasetAttributes attributes);
+	DatasetCodec<?, ?> create(final DatasetAttributes attributes);
 }

src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/Transpose.java

@@ -1,6 +1,7 @@
 package org.janelia.saalfeldlab.n5.codec.transpose;
 
 import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.util.MemCopy;
 
 class Transpose<T> {
 
@@ -112,129 +113,4 @@ Transpose<T> newInstance() {
 		return new Transpose<>(memCopy, ssize.length);
 	}
 
-	/**
-	 * Low-level range copying methods between source and target primitve array
-	 * (type {@code T}, e.g., {@code double[]}).
-	 *
-	 * @param <T>
-	 * 		the source/target type. Must be a primitive array type (e.g., {@code double[]})
-	 */
-	interface MemCopy<T> {
-
-		MemCopy.MemCopyByte BYTE = new MemCopy.MemCopyByte();
-		MemCopy.MemCopyShort SHORT = new MemCopy.MemCopyShort();
-		MemCopy.MemCopyInt INT = new MemCopy.MemCopyInt();
-		MemCopy.MemCopyLong LONG = new MemCopy.MemCopyLong();
-		MemCopy.MemCopyFloat FLOAT = new MemCopy.MemCopyFloat();
-		MemCopy.MemCopyDouble DOUBLE = new MemCopy.MemCopyDouble();
-
-		static MemCopy<?> forDataType(final DataType dataType) {
-			switch (dataType) {
-			case UINT8:
-			case INT8:
-				return BYTE;
-			case UINT16:
-			case INT16:
-				return SHORT;
-			case UINT32:
-			case INT32:
-				return INT;
-			case UINT64:
-			case INT64:
-				return LONG;
-			case FLOAT32:
-				return FLOAT;
-			case FLOAT64:
-				return DOUBLE;
-			case STRING:
-			case OBJECT:
-				throw new UnsupportedOperationException("TODO?");
-			default:
-				throw new IllegalArgumentException();
-			}
-		}
-
-		/**
-		 * Copy {@code length} components from the {@code src} array to the {@code
-		 * dest} array. The components at positions {@code srcPos} through {@code
-		 * srcPos+length-1} in the source array are copied into positions {@code
-		 * destPos}, {@code destPos+destStride}, {@code destPos + 2*destStride},
-		 * etc., through {@code destPos+(length-1)*destStride} of the destination
-		 * array.
-		 */
-		void copyStrided(T src, int srcPos, T dest, int destPos, int destStride, int length);
-
-		class MemCopyByte implements MemCopy<byte[]> {
-
-			@Override
-			public void copyStrided(final byte[] src, final int srcPos, final byte[] dest, final int destPos, final int destStride, final int length) {
-				if (destStride == 1)
-					System.arraycopy(src, srcPos, dest, destPos, length);
-				else
-					for (int i = 0; i < length; ++i)
-						dest[destPos + i * destStride] = src[srcPos + i];
-			}
-		}
-
-		class MemCopyShort implements MemCopy<short[]> {
-
-			@Override
-			public void copyStrided(final short[] src, final int srcPos, final short[] dest, final int destPos, final int destStride, final int length) {
-				if (destStride == 1)
-					System.arraycopy(src, srcPos, dest, destPos, length);
-				else
-					for (int i = 0; i < length; ++i)
-						dest[destPos + i * destStride] = src[srcPos + i];
-			}
-		}
-
-		class MemCopyInt implements MemCopy<int[]> {
-
-			@Override
-			public void copyStrided(final int[] src, final int srcPos, final int[] dest, final int destPos, final int destStride, final int length) {
-				if (destStride == 1)
-					System.arraycopy(src, srcPos, dest, destPos, length);
-				else
-					for (int i = 0; i < length; ++i)
-						dest[destPos + i * destStride] = src[srcPos + i];
-			}
-		}
-
-		class MemCopyLong implements MemCopy<long[]> {
-
-			@Override
-			public void copyStrided(final long[] src, final int srcPos, final long[] dest, final int destPos, final int destStride, final int length) {
-				if (destStride == 1)
-					System.arraycopy(src, srcPos, dest, destPos, length);
-				else
-					for (int i = 0; i < length; ++i)
-						dest[destPos + i * destStride] = src[srcPos + i];
-			}
-		}
-
-		class MemCopyFloat implements MemCopy<float[]> {
-
-			@Override
-			public void copyStrided(final float[] src, final int srcPos, final float[] dest, final int destPos, final int destStride, final int length) {
-				if (destStride == 1)
-					System.arraycopy(src, srcPos, dest, destPos, length);
-				else
-					for (int i = 0; i < length; ++i)
-						dest[destPos + i * destStride] = src[srcPos + i];
-			}
-		}
-
-		class MemCopyDouble implements MemCopy<double[]> {
-
-			@Override
-			public void copyStrided(final double[] src, final int srcPos, final double[] dest, final int destPos, final int destStride, final int length) {
-				if (destStride == 1)
-					System.arraycopy(src, srcPos, dest, destPos, length);
-				else
-					for (int i = 0; i < length; ++i)
-						dest[destPos + i * destStride] = src[srcPos + i];
-			}
-		}
-	}
-
 }