Merge pull request #192 from saalfeldlab/mergeDevelopmentRebaseOnMaster

Align development and master histories by rebasing development onto master

Author: cmhulbert

.github/workflows/build.yml

@@ -4,11 +4,13 @@ on:
   push:
     branches:
       - master
+      - development
     tags:
       - "*-[0-9]+.*"
   pull_request:
     branches:
       - master
+      - development
 
 jobs:
   build:

LICENSE.md

@@ -1,6 +1,6 @@
 ## BSD 2-Clause License
 
-Copyright (c) 2017-2025, Stephan Saalfeld
+Copyright (c) 2017-2026, Stephan Saalfeld
 
 All rights reserved.
 

pom.xml

@@ -175,6 +175,10 @@
 			<groupId>commons-io</groupId>
 			<artifactId>commons-io</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>commons-codec</groupId>
+			<artifactId>commons-codec</artifactId>
+		</dependency>
 
 		<!-- Test dependencies -->
 		<dependency>

src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java

@@ -31,6 +31,7 @@
 import com.google.gson.JsonDeserializationContext;
 import com.google.gson.JsonDeserializer;
 import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParseException;
 import com.google.gson.JsonSerializationContext;
@@ -52,6 +53,8 @@
 import java.util.stream.Collectors;
 
 import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DatasetCodec;
+import org.janelia.saalfeldlab.n5.codec.DatasetCodecInfo;
 
 
 /**
@@ -95,23 +98,30 @@ public class DatasetAttributes implements Serializable {
 
 	private final DataType dataType;
 
+	private final JsonElement defaultValue;
+
 	private final BlockCodecInfo blockCodecInfo;
 	private final DataCodecInfo[] dataCodecInfos;
+	private final DatasetCodecInfo[] datasetCodecInfos;
 
 	private transient final DatasetAccess<?> access;
 
 	public DatasetAttributes(
 			final long[] dimensions,
 			final int[] outerBlockSize,
 			final DataType dataType,
+			final JsonElement defaultValue,
 			final BlockCodecInfo blockCodecInfo,
+			final DatasetCodecInfo[] datasetCodecInfos,
 			final DataCodecInfo... dataCodecInfos) {
 
 		this.dimensions = dimensions;
 		this.dataType = dataType;
 		this.outerBlockSize = outerBlockSize;
+		this.defaultValue = defaultValue == null ? JsonNull.INSTANCE : defaultValue;
 
 		this.blockCodecInfo = blockCodecInfo == null ? defaultBlockCodecInfo() : blockCodecInfo;
+		this.datasetCodecInfos = datasetCodecInfos;
 
 		if (dataCodecInfos == null)
 			this.dataCodecInfos = new DataCodecInfo[0];
@@ -124,6 +134,28 @@ public DatasetAttributes(
 		blockSize = access.getGrid().getBlockSize(0);
 	}
 
+	public DatasetAttributes(
+			final long[] dimensions,
+			final int[] outerBlockSize,
+			final DataType dataType,
+			final BlockCodecInfo blockCodecInfo,
+			final DatasetCodecInfo[] datasetCodecInfos,
+			final DataCodecInfo... dataCodecInfos) {
+
+		this(dimensions, outerBlockSize, dataType, JsonNull.INSTANCE,
+				blockCodecInfo, datasetCodecInfos, dataCodecInfos);
+	}
+
+	public DatasetAttributes(
+			final long[] dimensions,
+			final int[] outerBlockSize,
+			final DataType dataType,
+			final BlockCodecInfo blockCodecInfo,
+			final DataCodecInfo... dataCodecInfos) {
+
+		this(dimensions, outerBlockSize, dataType, blockCodecInfo, null, dataCodecInfos);
+	}
+
 	/**
 	 * Constructs a DatasetAttributes instance with specified dimensions, block size, data type,
 	 * and single compressor with default codec.
@@ -165,7 +197,7 @@ protected DatasetAccess<?> createDatasetAccess() {
 		// The inner-most codec (the DataBlock codec) is at index 0.
 		final int[][] blockSizes = new int[m][];
 
-		// NestedGrid validates block sizes, so instantiate it before creating the blockCodecs  
+		// NestedGrid validates block sizes, so instantiate it before creating the blockCodecs
 		// blockCodecInfo.create below could fail unexpecedly with invalid
 		// blockSizes so validate first
 		blockSizes[m - 1] = outerBlockSize;
@@ -179,17 +211,23 @@ protected DatasetAccess<?> createDatasetAccess() {
 		BlockCodecInfo currentBlockCodecInfo = blockCodecInfo;
 		DataCodecInfo[] currentDataCodecInfos = dataCodecInfos;
 
-		final NestedGrid grid = new NestedGrid(blockSizes);
+		final NestedGrid grid = new NestedGrid(blockSizes, dimensions);
 		final BlockCodec<?>[] blockCodecs = new BlockCodec[m];
 		for (int l = m - 1; l >= 0; --l) {
 			blockCodecs[l] = currentBlockCodecInfo.create(dataType, blockSizes[l], currentDataCodecInfos);
 			if (l > 0) {
-				final ShardCodecInfo info = (ShardCodecInfo)currentBlockCodecInfo;
+				final ShardCodecInfo info = (ShardCodecInfo) currentBlockCodecInfo;
 				currentBlockCodecInfo = info.getInnerBlockCodecInfo();
 				currentDataCodecInfos = info.getInnerDataCodecInfos();
 			}
 		}
 
+		if (datasetCodecInfos != null) {
+			for (final DatasetCodecInfo info : datasetCodecInfos) {
+				blockCodecs[0] = DatasetCodec.concatenate(info.create(this), (BlockCodec) blockCodecs[0]);
+			}
+		}
+
 		return new DefaultDatasetAccess<>(grid, blockCodecs);
 	}
 
@@ -202,7 +240,6 @@ private static int nestingDepth(BlockCodecInfo info) {
 		}
 	}
 
-
 	protected BlockCodecInfo defaultBlockCodecInfo() {
 
 		return new N5BlockCodecInfo();
@@ -223,6 +260,11 @@ public int[] getBlockSize() {
 		return blockSize;
 	}
 
+	public JsonElement getDefaultValue() {
+
+		return defaultValue;
+	}
+
 	public boolean isSharded() {
 
 		return blockCodecInfo instanceof ShardCodecInfo;
@@ -284,6 +326,11 @@ public DataCodecInfo[] getDataCodecInfos() {
 		return dataCodecInfos;
 	}
 
+	public DatasetCodecInfo[] getDatasetCodecInfos() {
+
+		return datasetCodecInfos;
+	}
+
 	public String relativeBlockPath(long... position) {
 
 		return Arrays.stream(position).mapToObj(Long::toString).collect(Collectors.joining("/"));

src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java

@@ -148,4 +148,16 @@ default String absoluteAttributesPath(final String normalPath) {
 
 		return getKeyValueAccess().compose(getURI(), normalPath, getAttributesKey());
 	}
+
+	@Override
+	default boolean shardExists(
+			final String pathName,
+			final DatasetAttributes datasetAttributes,
+			final long... gridPosition) throws N5Exception {
+
+		final String normalPath = N5URI.normalizeGroupPath(pathName);
+		final String blockPath = getKeyValueAccess().compose(getURI(), normalPath,
+				datasetAttributes.relativeBlockPath(gridPosition));
+		return getKeyValueAccess().isFile(blockPath);
+	}
 }

src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java

@@ -222,7 +222,7 @@ default <T> void writeRegion(
 			final boolean writeFully) throws N5Exception {
 		try {
 			final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), datasetAttributes);
-			datasetAttributes.<T>getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, datasetAttributes.getDimensions(), writeFully);
+			datasetAttributes.<T>getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, writeFully);
 		} catch (final UncheckedIOException e) {
 			throw new N5IOException(
 					"Failed to write blocks into dataset " + datasetPath, e);
@@ -240,7 +240,7 @@ default <T> void writeRegion(
 			final ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException {
 		try {
 			final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), datasetAttributes);
-			datasetAttributes.<T>getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, datasetAttributes.getDimensions(), writeFully, exec);
+			datasetAttributes.<T>getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, writeFully, exec);
 		} catch (final UncheckedIOException e) {
 			throw new N5IOException(
 					"Failed to write blocks into dataset " + datasetPath, e);
@@ -276,6 +276,7 @@ default <T> void writeBlock(
 					"Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path,
 					e);
 		}
+
 	}
 
 	@Override
@@ -300,4 +301,4 @@ default boolean deleteBlock(
 		final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);
 		return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition);
 	}
-}
+}

src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java

@@ -331,6 +331,27 @@ default <T> List<DataBlock<T>> readBlocks(
 		return blocks;
 	}
 
+	/**
+	 * Checks if a shard (or block for non-sharded datasets) exists at the
+	 * given grid position without reading the data.
+	 * <p>
+	 * This method only checks for the presence of the key value for the gridPosition, it does not
+	 * read or validate the contents.
+	 *
+	 * @param pathName
+	 *            dataset path
+	 * @param datasetAttributes
+	 *            the dataset attributes
+	 * @param gridPosition
+	 *            the shard grid position (or block position for non-sharded datasets)
+	 * @return true if the shard (or block) file exists
+	 * @throws N5Exception
+	 *             the exception
+	 */
+	boolean shardExists(
+			final String pathName,
+			final DatasetAttributes datasetAttributes,
+			final long... gridPosition) throws N5Exception;
 	/**
 	 * Load a {@link DataBlock} as a {@link Serializable}. The offset is given
 	 * in

src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java

@@ -28,6 +28,7 @@
  */
 package org.janelia.saalfeldlab.n5;
 
+
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.ObjectOutputStream;
@@ -196,39 +197,46 @@ default boolean remove() throws N5Exception {
 
 	/**
 	 * Creates a dataset. This does not create any data but the path and
-	 * mandatory attributes only.
+	 * mandatory attributes only. The returned DatasetAttributes should be used
+	 * for future read/write operations on this dataset. It may not be the same
+	 * DatasetAttributes object that was provided, depending on the implementation.
 	 *
 	 * @param datasetPath dataset path
 	 * @param datasetAttributes the dataset attributes
-	 * @throws N5Exception the exception
+	 * @return DatasetAttributes optimal attributes object to be used for read/write operations
+	 * @throws N5Exception
 	 */
-	default void createDataset(
+	default DatasetAttributes createDataset(
 			final String datasetPath,
 			final DatasetAttributes datasetAttributes) throws N5Exception {
 
 		final String normalPath = N5URI.normalizeGroupPath(datasetPath);
 		createGroup(normalPath);
 		setDatasetAttributes(normalPath, datasetAttributes);
+		return datasetAttributes;
 	}
 
 	/**
 	 * Creates a dataset. This does not create any data but the path and
-	 * mandatory attributes only.
+	 * mandatory attributes only. Returns the DatasetAttributes object to be
+	 * used for future read/write operations on this dataset.
 	 *
 	 * @param datasetPath dataset path
 	 * @param dimensions the dataset dimensions
 	 * @param blockSize the block size
 	 * @param dataType the data type
 	 * @param compression the compression
+	 * @return DatasetAttributes optimal attributes object to be used for read/write operations
+	 * @throws N5Exception
 	 */
-	default void createDataset(
+	default DatasetAttributes createDataset(
 			final String datasetPath,
 			final long[] dimensions,
 			final int[] blockSize,
 			final DataType dataType,
 			final Compression compression) throws N5Exception {
 
-		createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression));
+		return createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression));
 	}
 
 	/**

src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java

@@ -1,31 +1,3 @@
-/*-
- * #%L
- * Not HDF5
- * %%
- * Copyright (C) 2017 - 2025 Stephan Saalfeld
- * %%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * 
- * 1. Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- *    this list of conditions and the following disclaimer in the documentation
- *    and/or other materials provided with the distribution.
- * 
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- * #L%
- */
 /**
  * Copyright (c) 2017, Stephan Saalfeld
  * All rights reserved.

src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodec.java

@@ -0,0 +1,47 @@
+package org.janelia.saalfeldlab.n5.codec;
+
+import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+
+/**
+ * A Codec that transforms the contents of a {@link DataBlock}.
+ * <p>
+ * This class is N5's analogue to Zarr's array-to-array codec.
+ *
+ * @param <S> source data type (data contained in decoded blocks)
+ * @param <T> target data type (data contained in encoded blocks)
+ */
+public interface DatasetCodec<S, T> {
+
+	// TODO Name ideas:
+	// "ImageCodec"?
+	// BlockTransformationCodec
+
+	DataBlock<T> encode(DataBlock<S> block) throws N5IOException;
+
+	DataBlock<S> decode(DataBlock<T> dataBlock) throws N5IOException;
+
+	/**
+	 * Create a {@code BlockCodec} that, for encoding, first applies {@code
+	 * datasetCodec} and then {@code blockCodec} (and does the same in reverse
+	 * order for decoding).
+	 *
+	 * @return the concatenated BlockCodec
+	 */
+	static <S, T> BlockCodec<S> concatenate(final DatasetCodec<S, T> datasetCodec, final BlockCodec<T> blockCodec) {
+
+		return new BlockCodec<S>() {
+
+			@Override
+			public ReadData encode(final DataBlock<S> dataBlock) throws N5IOException {
+				return blockCodec.encode(datasetCodec.encode(dataBlock));
+			}
+
+			@Override
+			public DataBlock<S> decode(final ReadData readData, final long[] gridPosition) throws N5IOException {
+				return datasetCodec.decode(blockCodec.decode(readData, gridPosition));
+			}
+		};
+	}
+}