T getAttribute(
final String normalizedAttributePath = N5URI.normalizeAttributePath(key);
JsonElement attributes;
if (cacheMeta()) {
- attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON);
+ attributes = getCache().getAttributes(normalPathName, getAttributesKey());
} else {
attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName);
}
@@ -136,7 +136,7 @@ default boolean exists(final String pathName) {
final String normalPathName = N5URI.normalizeGroupPath(pathName);
if (cacheMeta())
- return getCache().isGroup(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON);
+ return getCache().isGroup(normalPathName, getAttributesKey());
else {
return existsFromContainer(normalPathName, null);
}
@@ -180,7 +180,7 @@ default boolean datasetExists(final String pathName) throws N5IOException {
final String normalPathName = N5URI.normalizeGroupPath(pathName);
if (cacheMeta()) {
- return getCache().isDataset(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON);
+ return getCache().isDataset(normalPathName, getAttributesKey());
}
return isDatasetFromContainer(normalPathName);
}
@@ -212,7 +212,7 @@ default JsonElement getAttributes(final String pathName) throws N5IOException {
/* If cached, return the cache */
if (cacheMeta()) {
- return getCache().getAttributes(groupPath, N5KeyValueReader.ATTRIBUTES_JSON);
+ return getCache().getAttributes(groupPath, getAttributesKey());
} else {
return GsonKeyValueN5Reader.super.getAttributes(groupPath);
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java
index 5e6c85d15..d402eeddf 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java
@@ -28,10 +28,6 @@
*/
package org.janelia.saalfeldlab.n5;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.Arrays;
-
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import com.google.gson.Gson;
@@ -61,19 +57,11 @@ default void createGroup(final String path) throws N5Exception {
// avoid hitting the backend if this path is already a group according to the cache
// else if exists is true (then a dataset is present) so throw an exception to avoid
// overwriting / invalidating existing data
- if (cacheMeta()) {
- if (getCache().isGroup(normalPath, N5KeyValueReader.ATTRIBUTES_JSON))
- return;
- else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) {
- throw new N5Exception("Can't make a group on existing path.");
- }
- }
+ if (groupExists(normalPath))
+ return;
+ else if (datasetExists(normalPath))
+ throw new N5Exception("Can't make a group on existing dataset.");
- // N5Writer.super.createGroup(path);
- /*
- * the lines below duplicate the single line above but would have to call
- * normalizeGroupPath again the below duplicates code, but avoids extra work
- */
getKeyValueAccess().createDirectories(absoluteGroupPath(normalPath));
if (cacheMeta()) {
@@ -88,8 +76,8 @@ else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) {
for (final String child : pathParts) {
final String childPath = parent.isEmpty() ? child : parent + "/" + child;
- getCache().initializeNonemptyCache(childPath, N5KeyValueReader.ATTRIBUTES_JSON);
- getCache().updateCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON);
+ getCache().initializeNonemptyCache(childPath, getAttributesKey());
+ getCache().updateCacheInfo(childPath, getAttributesKey());
// only add if the parent exists and has children cached already
if (parent != null && !child.isEmpty())
@@ -130,7 +118,7 @@ default void writeAndCacheAttributes(
nullRespectingAttributes = getGson().toJsonTree(attributes);
}
/* Update the cache, and write to the writer */
- getCache().updateCacheInfo(normalGroupPath, N5KeyValueReader.ATTRIBUTES_JSON, nullRespectingAttributes);
+ getCache().updateCacheInfo(normalGroupPath, getAttributesKey(), nullRespectingAttributes);
}
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java
index 1cd5e913f..8c059c8a0 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import java.io.Serializable;
@@ -61,8 +36,8 @@
import java.lang.annotation.Target;
import org.janelia.saalfeldlab.n5.codec.DataCodec;
-import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
import org.janelia.saalfeldlab.n5.codec.CodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
import org.scijava.annotations.Indexable;
/**
@@ -74,11 +49,12 @@
* serialization.
*
* See also: an alternative method for serializing general {@link CodecInfo}s is
- * with the {@link NameConfigAdapter}.
+ * with the {@link NameConfigAdapter}. This interface remains for legacy
+ * (de)serialization.
*
* @author Stephan Saalfeld
*/
-public interface Compression extends Serializable, DataCodecInfo, DataCodec {
+public interface Compression extends Serializable, DataCodec, DataCodecInfo {
/**
* Annotation for runtime discovery of compression schemes.
@@ -102,7 +78,6 @@ public interface Compression extends Serializable, DataCodecInfo, DataCodec {
@Target(ElementType.FIELD)
@interface CompressionParameter {}
- @Override
default String getType() {
final CompressionType compressionType = getClass().getAnnotation(CompressionType.class);
@@ -116,4 +91,5 @@ default String getType() {
default DataCodec create() {
return this;
}
+
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java
index 3cd399a48..022f4b78a 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import java.lang.reflect.Constructor;
@@ -124,7 +99,7 @@ public static synchronized void update(final boolean override) {
newInstance.compressionConstructors.put(type, constructor);
newInstance.compressionParameters.put(type, parameters);
- } catch (final ClassNotFoundException | NoSuchMethodException | ClassCastException
+ } catch (final NoClassDefFoundError | ClassNotFoundException | NoSuchMethodException | ClassCastException
| UnsatisfiedLinkError e) {
System.err.println("Compression '" + item.className() + "' could not be registered");
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java
index 506096e08..b2e2cf1d6 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
/**
@@ -75,7 +50,7 @@ public interface DataBlock {
int[] getSize();
/**
- * Returns the position of this data block on the block grid.
+ * Returns the position of this data block on the block grid relative to dataset.
*
* The dimensionality of the grid position is expected to be equal to the
* dimensionality of the dataset. Consistency is not enforced.
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java
index 3ef9a410d..08bebf1e6 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import java.lang.reflect.Type;
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java
index 1127c658a..ad02e518f 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java
@@ -28,14 +28,31 @@
*/
package org.janelia.saalfeldlab.n5;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+import org.janelia.saalfeldlab.n5.codec.BlockCodec;
+import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.CodecInfo;
+import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo;
+import org.janelia.saalfeldlab.n5.shard.DatasetAccess;
+import org.janelia.saalfeldlab.n5.shard.DefaultDatasetAccess;
+import org.janelia.saalfeldlab.n5.shard.ShardCodecInfo;
+import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;
+
import java.io.Serializable;
+import java.lang.reflect.Type;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.stream.Collectors;
-import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo;
-import org.janelia.saalfeldlab.n5.codec.BlockCodec;
import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
-import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo;
+
/**
* Mandatory dataset attributes:
@@ -44,11 +61,10 @@
*
long[] : dimensions
* int[] : blockSize
* {@link DataType} : dataType
- * {@link Compression} : compression
+ * {@link CodecInfo}... : encode/decode routines
*
*
* @author Stephan Saalfeld
- *
*/
public class DatasetAttributes implements Serializable {
@@ -56,46 +72,137 @@ public class DatasetAttributes implements Serializable {
public static final String DIMENSIONS_KEY = "dimensions";
public static final String BLOCK_SIZE_KEY = "blockSize";
+ public static final String SHARD_SIZE_KEY = "shardSize";
public static final String DATA_TYPE_KEY = "dataType";
public static final String COMPRESSION_KEY = "compression";
+ public static final String CODEC_KEY = "codecs";
+
+ public static final String[] N5_DATASET_ATTRIBUTES = new String[]{
+ DIMENSIONS_KEY, BLOCK_SIZE_KEY, DATA_TYPE_KEY, COMPRESSION_KEY, CODEC_KEY
+ };
/* version 0 */
protected static final String compressionTypeKey = "compressionType";
private final long[] dimensions;
+
+ // number of samples per block per dimension
private final int[] blockSize;
+
+ // TODO add a getter?
+ // the shard size
+ private final int[] outerBlockSize;
+
private final DataType dataType;
private final BlockCodecInfo blockCodecInfo;
private final DataCodecInfo[] dataCodecInfos;
- private final BlockCodec blockCodec;
+ private transient final DatasetAccess access;
public DatasetAttributes(
final long[] dimensions,
- final int[] blockSize,
+ final int[] outerBlockSize,
final DataType dataType,
final BlockCodecInfo blockCodecInfo,
final DataCodecInfo... dataCodecInfos) {
this.dimensions = dimensions;
- this.blockSize = blockSize;
this.dataType = dataType;
+ this.outerBlockSize = outerBlockSize;
this.blockCodecInfo = blockCodecInfo == null ? defaultBlockCodecInfo() : blockCodecInfo;
- this.dataCodecInfos = Arrays.stream(dataCodecInfos).filter(it -> !(it instanceof RawCompression)).toArray(DataCodecInfo[]::new);
- blockCodec = this.blockCodecInfo.create(this, this.dataCodecInfos);
+
+ if (dataCodecInfos == null)
+ this.dataCodecInfos = new DataCodecInfo[0];
+ else
+ this.dataCodecInfos = Arrays.stream(dataCodecInfos)
+ .filter(it -> it != null && !(it instanceof RawCompression))
+ .toArray(DataCodecInfo[]::new);
+
+ access = createDatasetAccess();
+ blockSize = access.getGrid().getBlockSize(0);
}
+ /**
+ * Constructs a DatasetAttributes instance with specified dimensions, block size, data type,
+ * and single compressor with default codec.
+ *
+ * @param dimensions the dimensions of the dataset
+ * @param blockSize the size of the blocks in the dataset
+ * @param dataType the data type of the dataset
+ * @param dataCodecInfos the codecs used encode/decode the data
+ */
public DatasetAttributes(
final long[] dimensions,
final int[] blockSize,
final DataType dataType,
- final DataCodecInfo compression) {
+ final DataCodecInfo... dataCodecInfos) {
+
+ this(dimensions, blockSize, dataType, null, dataCodecInfos);
+ }
+
+ /**
+ * Constructs a DatasetAttributes instance with specified dimensions, block size, data type, and default codecs
+ *
+ * @param dimensions the dimensions of the dataset
+ * @param blockSize the size of the blocks in the dataset
+ * @param dataType the data type of the dataset
+ */
+ public DatasetAttributes(
+ final long[] dimensions,
+ final int[] blockSize,
+ final DataType dataType) {
+
+ this(dimensions, blockSize, dataType, new DataCodecInfo[0]);
+ }
+
+ protected DatasetAccess createDatasetAccess() {
+
+ final int m = nestingDepth(blockCodecInfo);
+
+ // There are m codecs: 1 DataBlock codecs, and m-1 shard codecs.
+ // The inner-most codec (the DataBlock codec) is at index 0.
+ final int[][] blockSizes = new int[m][];
+
+ // NestedGrid validates block sizes, so instantiate it before creating the blockCodecs
+ // blockCodecInfo.create below could fail unexpecedly with invalid
+ // blockSizes so validate first
+ blockSizes[m - 1] = outerBlockSize;
+ BlockCodecInfo tmpInfo = blockCodecInfo;
+ for (int l = m - 1; l > 0; --l) {
+ final ShardCodecInfo info = (ShardCodecInfo)tmpInfo;
+ blockSizes[l - 1] = info.getInnerBlockSize();
+ tmpInfo = info.getInnerBlockCodecInfo();
+ }
+
+ BlockCodecInfo currentBlockCodecInfo = blockCodecInfo;
+ DataCodecInfo[] currentDataCodecInfos = dataCodecInfos;
+
+ final NestedGrid grid = new NestedGrid(blockSizes);
+ final BlockCodec[] blockCodecs = new BlockCodec[m];
+ for (int l = m - 1; l >= 0; --l) {
+ blockCodecs[l] = currentBlockCodecInfo.create(dataType, blockSizes[l], currentDataCodecInfos);
+ if (l > 0) {
+ final ShardCodecInfo info = (ShardCodecInfo)currentBlockCodecInfo;
+ currentBlockCodecInfo = info.getInnerBlockCodecInfo();
+ currentDataCodecInfos = info.getInnerDataCodecInfos();
+ }
+ }
- this(dimensions, blockSize, dataType, null, compression);
+ return new DefaultDatasetAccess<>(grid, blockCodecs);
}
+ private static int nestingDepth(BlockCodecInfo info) {
+
+ if (info instanceof ShardCodecInfo) {
+ return 1 + nestingDepth(((ShardCodecInfo)info).getInnerBlockCodecInfo());
+ } else {
+ return 1;
+ }
+ }
+
+
protected BlockCodecInfo defaultBlockCodecInfo() {
return new N5BlockCodecInfo();
@@ -116,6 +223,21 @@ public int[] getBlockSize() {
return blockSize;
}
+ public boolean isSharded() {
+
+ return blockCodecInfo instanceof ShardCodecInfo;
+ }
+
+ /**
+ * Only used for deserialization for N5 backwards compatibility.
+ * {@link Compression} is no longer a special case. Prefer to reference {@link #getDataCodecInfos()}
+ * Will return {@link RawCompression} if no compression is otherwise provided, for legacy compatibility.
+ *
+ * Deprecated in favor of {@link #getDataCodecInfos()}.
+ *
+ * @return compression CodecInfo, if one was present, or else RawCompression
+ */
+ @Deprecated
public Compression getCompression() {
return Arrays.stream(dataCodecInfos)
@@ -131,19 +253,40 @@ public DataType getDataType() {
}
/**
- * Get the {@link BlockCodecInfo} for this dataset.
+ * Get the {@link DatasetAccess} for this dataset.
+ *
+ * @return the {@code DatasetAccess} for this dataset
+ */
+ DatasetAccess getDatasetAccess() {
+
+ return (DatasetAccess)access;
+ }
+
+ /**
+ * Returns the {@code NestedGrid} for this dataset, from which block and
+ * shard sizes are accessible.
*
- * @return the {@code BlockCodecInfo} for this dataset
+ * @return the NestedGrid
*/
+ public NestedGrid getNestedBlockGrid() {
+
+ return getDatasetAccess().getGrid();
+ }
+
+
public BlockCodecInfo getBlockCodecInfo() {
return blockCodecInfo;
}
- @SuppressWarnings("unchecked")
- BlockCodec getBlockCodec() {
+ public DataCodecInfo[] getDataCodecInfos() {
- return (BlockCodec) blockCodec;
+ return dataCodecInfos;
+ }
+
+ public String relativeBlockPath(long... position) {
+
+ return Arrays.stream(position).mapToObj(Long::toString).collect(Collectors.joining("/"));
}
public HashMap asMap() {
@@ -156,37 +299,94 @@ public HashMap asMap() {
return map;
}
- static DatasetAttributes from(
- final long[] dimensions,
- final DataType dataType,
- int[] blockSize,
- Compression compression,
- final String compressionVersion0Name) {
+ private static DatasetAttributesAdapter adapter = null;
+
+ public static DatasetAttributesAdapter getJsonAdapter() {
+
+ if (adapter == null) {
+ adapter = new DatasetAttributesAdapter();
+ }
+ return adapter;
+ }
- if (blockSize == null)
- blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray();
+ public static class DatasetAttributesAdapter implements JsonSerializer, JsonDeserializer {
+
+ @Override public DatasetAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
+
+ if (json == null || !json.isJsonObject())
+ return null;
+ final JsonObject obj = json.getAsJsonObject();
+ final boolean validKeySet = obj.has(DIMENSIONS_KEY)
+ && obj.has(BLOCK_SIZE_KEY)
+ && obj.has(DATA_TYPE_KEY)
+ && (obj.has(CODEC_KEY) || obj.has(COMPRESSION_KEY) || obj.has(compressionTypeKey));
+
+ if (!validKeySet)
+ return null;
+
+ final long[] dimensions = context.deserialize(obj.get(DIMENSIONS_KEY), long[].class);
+ final int[] blockSize = context.deserialize(obj.get(BLOCK_SIZE_KEY), int[].class);
+
+ final DataType dataType = context.deserialize(obj.get(DATA_TYPE_KEY), DataType.class);
+
+ final BlockCodecInfo blockCodecInfo;
+ final DataCodecInfo[] dataCodecs;
+ if (obj.has(CODEC_KEY)) {
+ final CodecInfo[] codecs = context.deserialize(obj.get(CODEC_KEY), CodecInfo[].class);
+ blockCodecInfo = (BlockCodecInfo)codecs[0];
+ dataCodecs = new DataCodecInfo[codecs.length - 1];
+ for (int i = 1; i < codecs.length; i++) {
+ dataCodecs[i - 1] = (DataCodecInfo)codecs[i];
+ }
+ } else if (obj.has(COMPRESSION_KEY)) {
+ final Compression compression = CompressionAdapter.getJsonAdapter().deserialize(obj.get(COMPRESSION_KEY), Compression.class, context);
+ dataCodecs = new DataCodecInfo[]{compression};
+ blockCodecInfo = new N5BlockCodecInfo();
+ } else if (obj.has(compressionTypeKey)) {
+ final Compression compression = getCompressionVersion0(obj.get(compressionTypeKey).getAsString());
+ dataCodecs = new DataCodecInfo[]{compression};
+ blockCodecInfo = new N5BlockCodecInfo();
+ } else {
+ return null;
+ }
+
+ return new DatasetAttributes(dimensions, blockSize, dataType, blockCodecInfo, dataCodecs);
+ }
+
+ //FIXME
+ // this implements multi-codec serialization for N5. We probably don't want this now
+ @Override public JsonElement serialize(DatasetAttributes src, Type typeOfSrc, JsonSerializationContext context) {
+
+ final JsonObject obj = new JsonObject();
+ obj.add(DIMENSIONS_KEY, context.serialize(src.dimensions));
+ obj.add(BLOCK_SIZE_KEY, context.serialize(src.blockSize));
+ obj.add(DATA_TYPE_KEY, context.serialize(src.dataType));
+
+ final DataCodecInfo[] codecs = src.dataCodecInfos;
+ // length > 1 is actually invalid, but this is checked on construction
+ if (codecs.length == 0)
+ obj.add(COMPRESSION_KEY, context.serialize(new RawCompression()));
+ else
+ obj.add(COMPRESSION_KEY, context.serialize(codecs[0]));
+
+ return obj;
+ }
+
+ private static Compression getCompressionVersion0(final String compressionVersion0Name) {
- /* version 0 */
- if (compression == null) {
switch (compressionVersion0Name) {
case "raw":
- compression = new RawCompression();
- break;
+ return new RawCompression();
case "gzip":
- compression = new GzipCompression();
- break;
+ return new GzipCompression();
case "bzip2":
- compression = new Bzip2Compression();
- break;
+ return new Bzip2Compression();
case "lz4":
- compression = new Lz4Compression();
- break;
+ return new Lz4Compression();
case "xz":
- compression = new XzCompression();
- break;
+ return new XzCompression();
}
+ return null;
}
-
- return new DatasetAttributes(dimensions, blockSize, dataType, compression);
}
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java
index 91bdfe84f..6346556e0 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
public class DoubleArrayDataBlock extends AbstractDataBlock {
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java
index d6a9803b8..e6ae8439b 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017--2021, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java
index 52c11b401..6fb46a6bd 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
public class FloatArrayDataBlock extends AbstractDataBlock {
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java
index e30c7f88e..8ede4da3a 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java
@@ -32,9 +32,10 @@
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UncheckedIOException;
+import java.util.List;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
-import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.janelia.saalfeldlab.n5.shard.PositionValueAccess;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
@@ -91,55 +92,35 @@ default JsonElement getAttributes(final String pathName) throws N5Exception {
}
@Override
- default DataBlock readBlock(
+ default DataBlock readBlock(
final String pathName,
final DatasetAttributes datasetAttributes,
final long... gridPosition) throws N5Exception {
- final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), gridPosition);
-
try {
- final ReadData blockData = getKeyValueAccess().createReadData(path);
- return datasetAttributes.getBlockCodec().decode(blockData, gridPosition);
+ final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName),
+ datasetAttributes);
+ return datasetAttributes. getDatasetAccess().readBlock(posKva, gridPosition);
+
} catch (N5Exception.N5NoSuchKeyException e) {
return null;
}
}
@Override
- default String[] list(final String pathName) throws N5Exception {
+ default List> readBlocks(
+ final String pathName,
+ final DatasetAttributes datasetAttributes,
+ final List blockPositions) throws N5Exception {
- return getKeyValueAccess().listDirectories(absoluteGroupPath(pathName));
+ final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), datasetAttributes);
+ return datasetAttributes. getDatasetAccess().readBlocks(posKva, blockPositions);
}
- /**
- * Constructs the path for a data block in a dataset at a given grid
- * position.
- *
- * The returned path is
- *
- *
- * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
- *
- *
- * This is the file into which the data block will be stored.
- *
- * @param normalPath
- * normalized dataset path
- * @param gridPosition to the target data block
- * @return the absolute path to the data block ad gridPosition
- */
- default String absoluteDataBlockPath(
- final String normalPath,
- final long... gridPosition) {
-
- final String[] components = new String[gridPosition.length + 1];
- components[0] = normalPath;
- int i = 0;
- for (final long p : gridPosition)
- components[++i] = Long.toString(p);
+ @Override
+ default String[] list(final String pathName) throws N5Exception {
- return getKeyValueAccess().compose(getURI(), components);
+ return getKeyValueAccess().listDirectories(absoluteGroupPath(pathName));
}
/**
@@ -165,6 +146,6 @@ default String absoluteGroupPath(final String normalGroupPath) {
*/
default String absoluteAttributesPath(final String normalPath) {
- return getKeyValueAccess().compose(getURI(), normalPath, N5KeyValueReader.ATTRIBUTES_JSON);
+ return getKeyValueAccess().compose(getURI(), normalPath, getAttributesKey());
}
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java
index 97b9a745e..d0a9c0e66 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java
@@ -26,35 +26,9 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017--2021, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import java.io.IOException;
-import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.util.Arrays;
import java.util.List;
@@ -62,6 +36,7 @@
import com.google.gson.JsonSyntaxException;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.shard.PositionValueAccess;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
@@ -218,7 +193,7 @@ default T removeAttribute(final String pathName, final String key, final Cla
throw new N5Exception.N5ClassCastException(e);
}
if (obj != null) {
- writeAttributes(normalPath, attributes);
+ setAttributes(normalPath, attributes);
}
return obj;
}
@@ -235,23 +210,36 @@ default boolean removeAttributes(final String pathName, final List attri
return removed;
}
+ @Override
+ default void writeBlocks(
+ final String datasetPath,
+ final DatasetAttributes datasetAttributes,
+ final DataBlock... dataBlocks) throws N5Exception {
+
+ try {
+ final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), datasetAttributes);
+ datasetAttributes.getDatasetAccess().writeBlocks(posKva, Arrays.asList(dataBlocks));
+ } catch (final UncheckedIOException e) {
+ throw new N5IOException(
+ "Failed to write blocks into dataset " + datasetPath, e);
+ }
+ }
+
@Override
default void writeBlock(
final String path,
final DatasetAttributes datasetAttributes,
final DataBlock dataBlock) throws N5Exception {
- final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition());
- try (
- final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath);
- final OutputStream out = lock.newOutputStream()
- ) {
- datasetAttributes.getBlockCodec().encode(dataBlock).writeTo(out);
- } catch (final IOException | UncheckedIOException e) {
+ try {
+ final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);
+ datasetAttributes. getDatasetAccess().writeBlock(posKva, dataBlock);
+ } catch (final UncheckedIOException e) {
throw new N5IOException(
"Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path,
e);
}
+
}
@Override
@@ -271,12 +259,9 @@ default boolean deleteBlock(
final String path,
final long... gridPosition) throws N5Exception {
- final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), gridPosition);
- if (getKeyValueAccess().isFile(blockPath))
- getKeyValueAccess().delete(blockPath);
-
-
- /* an IOException should have occurred if anything had failed midway */
- return true;
+ final String normalPath = N5URI.normalizeGroupPath(path);
+ final DatasetAttributes datasetAttributes = getDatasetAttributes(normalPath);
+ final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);
+ return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition);
}
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java
index a0aab927a..91b3747d3 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java
@@ -31,6 +31,9 @@
import java.lang.reflect.Type;
import java.util.Map;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonParseException;
+
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonSyntaxException;
@@ -43,6 +46,14 @@ public interface GsonN5Reader extends N5Reader {
Gson getGson();
+ /**
+ * Get the key for the {@link KeyValueAccess}, that is used for storing attributes.
+ * The N5 format uses "attributes.json".
+ *
+ * @return the attributes key
+ */
+ String getAttributesKey();
+
@Override
default Map> listAttributes(final String pathName) throws N5Exception {
@@ -59,30 +70,15 @@ default DatasetAttributes getDatasetAttributes(final String pathName) throws N5E
default DatasetAttributes createDatasetAttributes(final JsonElement attributes) {
- try {
- final long[] dimensions = GsonUtils.readAttribute(attributes, DatasetAttributes.DIMENSIONS_KEY, long[].class, getGson());
- if (dimensions == null) {
- return null;
- }
-
- final DataType dataType = GsonUtils.readAttribute(attributes, DatasetAttributes.DATA_TYPE_KEY, DataType.class, getGson());
- if (dataType == null) {
- return null;
- }
+ final JsonDeserializationContext context = new JsonDeserializationContext() {
- final int[] blockSize = GsonUtils.readAttribute(attributes, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, getGson());
- final Compression compression = GsonUtils.readAttribute(attributes, DatasetAttributes.COMPRESSION_KEY, Compression.class, getGson());
+ @Override public T deserialize(JsonElement json, Type typeOfT) throws JsonParseException {
- /* version 0 */
- final String compressionVersion0Name = compression == null
- ? GsonUtils.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, getGson())
- : null;
+ return getGson().fromJson(json, typeOfT);
+ }
+ };
- return DatasetAttributes.from(dimensions, dataType, blockSize, compression, compressionVersion0Name);
- } catch (JsonSyntaxException | NumberFormatException | ClassCastException e) {
- /* We cannot create a dataset, so return null. */
- return null;
- }
+ return DatasetAttributes.getJsonAdapter().deserialize(attributes, DatasetAttributes.class, context);
}
@Override
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java
index 6970ae2d4..3d3db1aeb 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import java.io.IOException;
@@ -63,7 +38,6 @@
import java.util.regex.Matcher;
import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
@@ -71,7 +45,6 @@
import com.google.gson.JsonSyntaxException;
import com.google.gson.reflect.TypeToken;
import org.janelia.saalfeldlab.n5.N5Exception.N5JsonParseException;
-import org.janelia.saalfeldlab.n5.codec.CodecInfo;
/**
* Utility class for working with JSON.
@@ -80,15 +53,6 @@
*/
public interface GsonUtils {
- static Gson registerGson(final GsonBuilder gsonBuilder) {
-
- gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter());
- gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class));
- gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter());
- gsonBuilder.disableHtmlEscaping();
- return gsonBuilder.create();
- }
-
/**
* Reads the attributes json from a given {@link Reader}.
*
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java
index e7bc39355..a1462aa16 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java
@@ -158,7 +158,7 @@ public boolean exists(final String normalPath) {
public boolean isDirectory(final String normalPath) {
try {
- requireValidHttpResponse(getDirectoryPath(normalPath), HEAD, (code, msg,http) -> {
+ requireValidHttpResponse(getDirectoryPath(normalPath), HEAD, false, (code, msg,http) -> {
final N5Exception cause = validExistsResponse(code, "Error checking directory: " + normalPath, msg, true);
if (code >= 300 && code < 400) {
final String redirectLocation = http.getHeaderField("Location");
@@ -200,7 +200,7 @@ public boolean isFile(final String normalPath) {
/* Files must not end in `/` And Don't accept a redirect to a location ending in `/` */
try {
- requireValidHttpResponse(getFilePath(normalPath), HEAD, (code, msg, http) -> {
+ requireValidHttpResponse(getFilePath(normalPath), HEAD, false, (code, msg, http) -> {
final N5Exception cause = validExistsResponse(code, "Error accessing file: " + normalPath, msg, true);
if (code >= 300 && code < 400) {
final String redirectLocation = http.getHeaderField("Location");
@@ -305,7 +305,7 @@ private static N5Exception validExistsResponse(int code, String responseMsg, Str
if (code >= 200 && code < (allowRedirect ? 400 : 300)) return null;
final RuntimeException cause = new RuntimeException(message + "( "+ responseMsg + ")(" + code + ")");
- if (code == 404)
+ if (code == 404 | code == 410)
return new N5Exception.N5NoSuchKeyException(message, cause);
return new N5Exception(message, cause);
@@ -316,12 +316,17 @@ private HttpURLConnection requireValidHttpResponse(String uri, String method, St
}
private HttpURLConnection requireValidHttpResponse(String uri, String method, TriFunction filterCode) throws N5Exception {
+ return requireValidHttpResponse(uri, method, true, filterCode);
+ }
+
+ private HttpURLConnection requireValidHttpResponse(String uri, String method, boolean followRedirects, TriFunction filterCode) throws N5Exception {
final int code;
final HttpURLConnection http;
final String responseMsg;
try {
http = httpRequest(uri, method);
+ http.setInstanceFollowRedirects(followRedirects);
code = http.getResponseCode();
responseMsg = http.getResponseMessage();
} catch (IOException e) {
@@ -384,6 +389,7 @@ public InputStream newInputStream() throws N5IOException {
}
return conn.getInputStream();
} catch (FileNotFoundException e) {
+ /*default HttpURLConnection throws FileNotFoundException on 404 or 410 */
throw new N5Exception.N5NoSuchKeyException("Could not open stream for " + uri, e);
} catch (IOException e) {
throw new N5IOException("Could not open stream for " + uri, e);
@@ -397,7 +403,7 @@ private String rangeString() {
}
@Override
- public Reader newReader() throws N5IOException {
+ public Reader newReader() {
final InputStreamReader reader = new InputStreamReader(newInputStream(), StandardCharsets.UTF_8);
synchronized (resources) {
@@ -407,13 +413,13 @@ public Reader newReader() throws N5IOException {
}
@Override
- public OutputStream newOutputStream() throws N5IOException {
+ public OutputStream newOutputStream() {
throw new NonWritableChannelException();
}
@Override
- public Writer newWriter() throws N5IOException {
+ public Writer newWriter() {
throw new NonWritableChannelException();
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java
index 4bea296c2..915c9d65c 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
public class IntArrayDataBlock extends AbstractDataBlock {
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java
index 5ed6eec8a..832e5368a 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017--2021, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java
index 7184ea10d..f956937e4 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java
@@ -26,37 +26,11 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017--2021, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import java.io.Closeable;
-import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Reader;
@@ -77,7 +51,7 @@ public interface LockedChannel extends Closeable {
* @throws N5IOException
* if the reader could not be created
*/
- public Reader newReader() throws N5IOException;
+ Reader newReader() throws N5IOException;
/**
* Create a new {@link InputStream}.
@@ -86,7 +60,7 @@ public interface LockedChannel extends Closeable {
* @throws N5IOException
* if an input stream could not be created
*/
- public InputStream newInputStream() throws N5IOException;
+ InputStream newInputStream() throws N5IOException;
/**
* Create a new UTF-8 {@link Writer}.
@@ -95,7 +69,7 @@ public interface LockedChannel extends Closeable {
* @throws N5IOException
* if a writer could not be created
*/
- public Writer newWriter() throws N5IOException;
+ Writer newWriter() throws N5IOException;
/**
* Create a new {@link OutputStream}.
@@ -104,5 +78,5 @@ public interface LockedChannel extends Closeable {
* @throws N5IOException
* if an output stream could not be created
*/
- public OutputStream newOutputStream() throws N5IOException;
+ OutputStream newOutputStream() throws N5IOException;
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java
index bad39ec6d..be0d8883d 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
public class LongArrayDataBlock extends AbstractDataBlock {
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java
index 1039713b5..ea01879df 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java
@@ -31,12 +31,9 @@
import net.jpountz.lz4.LZ4BlockInputStream;
import net.jpountz.lz4.LZ4BlockOutputStream;
import org.janelia.saalfeldlab.n5.Compression.CompressionType;
-import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.serialization.NameConfig;
-import java.io.IOException;
-
@CompressionType("lz4")
@NameConfig.Name("lz4")
public class Lz4Compression implements Compression {
@@ -67,7 +64,7 @@ public boolean equals(final Object other) {
}
@Override
- public ReadData decode(final ReadData readData) throws N5IOException {
+ public ReadData decode(final ReadData readData) {
return ReadData.from(new LZ4BlockInputStream(readData.inputStream()));
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java
index 8d332098b..4397b4544 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017--2021, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import java.nio.file.FileSystems;
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java
index 589b566db..d856ebb22 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017--2021, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import java.nio.file.FileSystems;
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java
index e86b712ae..de60ebd5e 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java
@@ -127,9 +127,13 @@ protected N5KeyValueReader(
throws N5Exception {
this.keyValueAccess = keyValueAccess;
- this.gson = GsonUtils.registerGson(gsonBuilder);
+ this.gson = registerGson(gsonBuilder).create();
this.cacheMeta = cacheMeta;
- this.cache = newCache();
+
+ if (this.cacheMeta)
+ this.cache = newCache();
+ else
+ this.cache = null;
try {
uri = keyValueAccess.uri(basePath);
@@ -159,6 +163,21 @@ private boolean inferExistence(String path) {
return attributes != null || exists(path);
}
+ protected GsonBuilder registerGson(final GsonBuilder gsonBuilder) {
+
+ gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter());
+ gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter());
+ gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, DatasetAttributes.getJsonAdapter());
+ gsonBuilder.disableHtmlEscaping();
+ return gsonBuilder;
+ }
+
+ @Override
+ public String getAttributesKey() {
+
+ return ATTRIBUTES_JSON;
+ }
+
@Override
public Gson getGson() {
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java
index 367ac2954..c378dda68 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import java.io.ByteArrayInputStream;
@@ -252,8 +227,6 @@ default Version getVersion() throws N5Exception {
*
* @return the base path URI
*/
- // TODO: should this throw URISyntaxException or can we assume that this is
- // never possible if we were able to instantiate this N5Reader?
URI getURI();
/**
@@ -311,6 +284,8 @@ T getAttribute(
/**
* Reads a {@link DataBlock}.
*
+ * @param
+ * the DataBlock data type
* @param pathName
* dataset path
* @param datasetAttributes
@@ -321,11 +296,41 @@ T getAttribute(
* @throws N5Exception
* the exception
*/
- DataBlock readBlock(
+ DataBlock readBlock(
final String pathName,
final DatasetAttributes datasetAttributes,
final long... gridPosition) throws N5Exception;
+ /**
+ * Reads multiple {@link DataBlock}s.
+ *
+ * Implementations may optimize / batch read operations when possible, e.g.
+ * in the case that the datasets are sharded.
+ *
+ * @param
+ * the DataBlock data type
+ * @param pathName
+ * dataset path
+ * @param datasetAttributes
+ * the dataset attributes
+ * @param gridPositions
+ * a list of grid positions
+ * @return a list of data blocks
+ * @throws N5Exception
+ * the exception
+ */
+ default List> readBlocks(
+ final String pathName,
+ final DatasetAttributes datasetAttributes,
+ final List gridPositions) throws N5Exception {
+
+ final ArrayList> blocks = new ArrayList<>();
+ for( final long[] p : gridPositions )
+ blocks.add(readBlock(pathName, datasetAttributes, p));
+
+ return blocks;
+ }
+
/**
* Load a {@link DataBlock} as a {@link Serializable}. The offset is given
* in
@@ -345,13 +350,12 @@ DataBlock readBlock(
* @throws ClassNotFoundException
* the class not found exception
*/
- @SuppressWarnings("unchecked")
default T readSerializedBlock(
final String dataset,
final DatasetAttributes attributes,
final long... gridPosition) throws N5Exception, ClassNotFoundException {
- final DataBlock block = (DataBlock) readBlock(dataset, attributes, gridPosition);
+ final DataBlock block = readBlock(dataset, attributes, gridPosition);
if (block == null)
return null;
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java
index 1eac58a68..c086acc05 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java
@@ -454,10 +454,86 @@ public static String normalizeGroupPath(final String path) {
return normalizePath(path.startsWith("/") || path.startsWith("\\") ? path.substring(1) : path);
}
+ private enum N5UriPattern {
+
+ /**
+ * matches any `/` or `[N]` where `N` is non-negative
+ */
+ MULTI_PART_ATTRIBUTE(Pattern.compile(".*((?\\[[0-9]+])")),
+ /**
+ * matches `A[N]` where A is some non-empty preceding path
+ */
+ ATTRIBUTE_ARRAY_EXCEPT_START(Pattern.compile("((?\\[[0-9]+]))")),
+ /**
+ * The following Pattern has 4 possible matches.
+ * It is intended to be used to remove matching portions iteratively until no further matches are found:
+ *
+ * The first 3 matches can remove redundant separators of the form:
+ *
+ * - (?<=/)/+ : `a///b` -> `a/b`
+ * - (?<=(/|^))(\./)+ : `a/./b` -> `a/b`
+ * - ((/|(?<=/))\.)$ : `a/b/` -> `a/b`
+ *
+ * The next match avoids removing `/` when it is NOT redundant (e.g. only character, or escaped):
+ *
+ * - (? `/ , `/a/b/\\/` -> `/a/b/\\/`
+ *
+ * The last match resolves relative paths:
+ *
+ * - 5. ((?<=^/)|^|(?<=(/|^))[^/]+(?
+ *
+ * - `a/../b` -> `b`
+ * - `/a/../b` -> `/b`
+ * - `../a/../b` -> `b`
+ * - `/../a/../b` -> `/b`
+ * - `/../a/../../b` -> `/b`
+ *
+ *
+ *
+ */
+ RELATIVE_ATTRIBUTE_PARTS(Pattern.compile( "((?<=/)/+|(?<=(/|^))(\\./)+|((/|(?<=/))\\.)$|(?
- * Attribute paths have a few of special characters:
+ * Attribute paths have a few special characters:
*
* - "." which represents the current element
* - ".." which represent the previous elemnt
@@ -503,50 +579,20 @@ public static String normalizeAttributePath(final String attributePath) {
* Short circuit if there are no non-escaped `/` or array indices (e.g.
* [N] where N is a non-negative integer)
*/
- if (!attributePath.matches(".*((? `[10]/b` */
- final String attrPathPlusFirstIndexSeparator = attributePath.replaceAll("^(?\\[[0-9]+])", "${array}/");
+ final String attrPathPlusFirstIndexSeparator = N5UriPattern.appendSlashAfterArrayStart(attributePath);
+
/*
* Add separator before and after arrays not at the beginning `a[10]b`
* -> `a/[10]/b`
*/
- final String attrPathPlusIndexSeparators = attrPathPlusFirstIndexSeparator
- .replaceAll("((?\\[[0-9]+]))", "/${array}/");
+ final String attrPathPlusIndexSeparators = N5UriPattern.addSlashAroundArrayExceptStart(attrPathPlusFirstIndexSeparator);
- /*
- * The following has 4 possible matches, in each case it removes the
- * match:
- * The first 3 remove redundant separators of the form:
- * 1.`a///b` -> `a/b` : (?<=/)/+
- * 2.`a/./b` -> `a/b` : (?<=(/|^))(\./)+
- * 3.`a/b/` -> `a/b` : ((/|(?<=/))\.)$
- * The next avoids removing `/` when it is NOT redundant (e.g. only character, or escaped):
- * 4. `/` -> `/ , `/a/b/\\/` -> `/a/b/\\/` : (? `b`
- * - `/a/../b` -> `/b`
- * - `../a/../b` -> `b`
- * - `/../a/../b` -> `/b`
- * - `/../a/../../b` -> `/b`
- *
- * This is run iteratively, since earlier removals may cause later
- * removals to be valid,
- * as well as the need to match once per relative `../` pattern.
- */
- final Pattern relativePathPattern = Pattern.compile(
- "((?<=/)/+|(?<=(/|^))(\\./)+|((/|(?<=/))\\.)$|(? void writeBlock(
final DataBlock dataBlock) throws N5Exception;
/**
- * Deletes the block at {@code gridPosition}
+ * Write multiple data blocks, useful for request aggregation.
*
* @param datasetPath dataset path
- * @param gridPosition position of block to be deleted
+ * @param datasetAttributes the dataset attributes
+ * @param dataBlocks the data block
+ * @param the data block data type
* @throws N5Exception the exception
+ */
+ default void writeBlocks(
+ final String datasetPath,
+ final DatasetAttributes datasetAttributes,
+ final DataBlock... dataBlocks) throws N5Exception {
+
+ // default method is naive
+ for (DataBlock block : dataBlocks)
+ writeBlock(datasetPath, datasetAttributes, block);
+ }
+
+ /**
+ * Deletes the block at {@code gridPosition}.
*
- * @return {@code true} if the block at {@code gridPosition} is "empty"
- * after
- * deletion. The meaning of "empty" is implementation dependent. For
- * example "empty" means that no file exists on the file system for
- * the
- * deleted block in case of the file system implementation.
+ * @param datasetPath dataset path
+ * @param gridPosition position of block to be deleted
+ * @throws N5Exception if the block exists but could not be deleted
*
+ * @return {@code true} if the block at {@code gridPosition} existed and was deleted.
*/
boolean deleteBlock(
final String datasetPath,
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java
index 7fd05a262..2c7fa9b39 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java
@@ -87,6 +87,11 @@ public static synchronized void update(final NameConfigAdapter adapter) {
Class clazz;
try {
clazz = (Class)Class.forName(item.className());
+
+ final NameConfig.Serialize serialize = clazz.getAnnotation(NameConfig.Serialize.class);
+ if (serialize != null && !serialize.value())
+ continue;
+
final String name = clazz.getAnnotation(NameConfig.Name.class).value();
final String type = prefix + "." + name;
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java
index bb4dfe51b..6d3ebc86e 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java
@@ -26,43 +26,22 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import org.janelia.saalfeldlab.n5.Compression.CompressionType;
+import org.janelia.saalfeldlab.n5.codec.DeterministicSizeDataCodec;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.janelia.saalfeldlab.n5.serialization.NameConfig;
@CompressionType("raw")
-public class RawCompression implements Compression {
+@NameConfig.Name("raw")
+public class RawCompression implements Compression, DeterministicSizeDataCodec {
private static final long serialVersionUID = 7526445806847086477L;
@Override
public boolean equals(final Object other) {
+
return other != null && other.getClass() == RawCompression.class;
}
@@ -75,4 +54,9 @@ public ReadData encode(final ReadData readData) {
public ReadData decode(final ReadData readData) {
return readData;
}
+
+ @Override
+ public long encodedSize(final long size) {
+ return size;
+ }
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java
index eeff6fdc3..3a7fed420 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import java.lang.reflect.Field;
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java
index a181e589c..e247f76a6 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
public class ShortArrayDataBlock extends AbstractDataBlock {
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java
index 5b811b5e4..fe846ebd6 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java
@@ -1,31 +1,3 @@
-/*-
- * #%L
- * Not HDF5
- * %%
- * Copyright (C) 2017 - 2025 Stephan Saalfeld
- * %%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- * #L%
- */
/**
* Copyright (c) 2017, Stephan Saalfeld
* All rights reserved.
@@ -53,10 +25,58 @@
*/
package org.janelia.saalfeldlab.n5;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
public class StringDataBlock extends AbstractDataBlock {
- public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) {
+ protected static final Charset ENCODING = StandardCharsets.UTF_8;
+ protected static final String NULLCHAR = "\0";
+ protected byte[] serializedData = null;
+ protected String[] actualData = null;
+
+ public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) {
+ super(size, gridPosition, new String[0], a -> a.length);
+ actualData = data;
+ }
+
+ public StringDataBlock(final int[] size, final long[] gridPosition, final byte[] data) {
+ super(size, gridPosition, new String[0], a -> a.length);
+ serializedData = data;
+ }
+
+ public void readData(final ByteBuffer buffer) {
+
+ if (buffer.hasArray()) {
+ if (buffer.array() != serializedData)
+ buffer.get(serializedData);
+ actualData = deserialize(buffer.array());
+ } else
+ actualData = ENCODING.decode(buffer).toString().split(NULLCHAR);
+ }
+
+ protected byte[] serialize(String[] strings) {
+ final String flattenedArray = String.join(NULLCHAR, strings) + NULLCHAR;
+ return flattenedArray.getBytes(ENCODING);
+ }
+
+ protected String[] deserialize(byte[] rawBytes) {
+ final String rawChars = new String(rawBytes, ENCODING);
+ return rawChars.split(NULLCHAR);
+ }
+
+ @Override
+ public int getNumElements() {
+ if (serializedData == null)
+ serializedData = serialize(actualData);
+ return serializedData.length;
+ }
- super(size, gridPosition, data, a -> a.length);
- }
+ @Override
+ public String[] getData() {
+ if (actualData == null)
+ actualData = deserialize(serializedData);
+ return actualData;
+ }
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java
index 770cece13..2a60743a0 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java
@@ -28,6 +28,7 @@
*/
package org.janelia.saalfeldlab.n5.cache;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
@@ -183,6 +184,7 @@ public String[] list(final String normalPathKey) {
for (final String child : cacheInfo.children) {
children[i++] = child;
}
+ Arrays.sort(children);
return children;
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java
index 8eb889661..269daa107 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java
@@ -30,6 +30,7 @@
import org.janelia.saalfeldlab.n5.DataBlock;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.RawCompression;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
/**
@@ -43,4 +44,26 @@ public interface BlockCodec {
ReadData encode(DataBlock dataBlock) throws N5IOException;
DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException;
+
+ /**
+ * Given the {@code blockSize} of a {@code DataBlock} return the size of
+ * the encoded block in bytes.
+ *
+ * A {@code UnsupportedOperationException} is thrown, if this {@code
+ * BlockCodec} cannot determine encoded size independent of block content.
+ * For example, if the block type contains var-length elements or if the
+ * serializer uses a non-deterministic {@code DataCodec}.
+ *
+ * @param blockSize
+ * size of the block to be encoded
+ *
+ * @return size of the encoded block in bytes
+ *
+ * @throws UnsupportedOperationException
+ * if this {@code DataBlockSerializer} cannot determine encoded size independent of block content
+ */
+ default long encodedSize(int[] blockSize) throws UnsupportedOperationException {
+
+ throw new UnsupportedOperationException();
+ }
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java
index ef5f3825a..bd18f1759 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java
@@ -1,7 +1,7 @@
package org.janelia.saalfeldlab.n5.codec;
-import java.util.Arrays;
import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.DataType;
import org.janelia.saalfeldlab.n5.DatasetAttributes;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
@@ -11,15 +11,32 @@
* {@code BlockCodec}s encode {@link DataBlock}s into {@link ReadData} and
* decode {@link ReadData} into {@link DataBlock}s.
*/
-public interface BlockCodecInfo extends CodecInfo {
+public interface BlockCodecInfo extends CodecInfo, DeterministicSizeCodecInfo {
- BlockCodec create(final DatasetAttributes attributes, final DataCodec... codecs);
+ default long[] getKeyPositionForBlock(final DatasetAttributes attributes, final DataBlock datablock) {
- default BlockCodec create(final DatasetAttributes attributes, final DataCodecInfo... codecInfos) {
- final DataCodec[] codecs = new DataCodec[codecInfos.length];
- Arrays.setAll(codecs, i -> codecInfos[i].create());
- return create(attributes, codecs);
+ return datablock.getGridPosition();
+ }
+
+ default long[] getKeyPositionForBlock(final DatasetAttributes attributes, final long... blockPosition) {
+
+ return blockPosition;
+ }
+
+ @Override default long encodedSize(long size) {
+
+ return size;
}
- // TODO: Should we have both create() signatures?
+ @Override default long decodedSize(long size) {
+
+ return size;
+ }
+
+ BlockCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs);
+
+ default BlockCodec create(final DatasetAttributes attributes, final DataCodecInfo... codecInfos) {
+
+ return create(attributes.getDataType(), attributes.getBlockSize(), codecInfos);
+ }
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java
index 95b934643..a39032dab 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java
@@ -1,11 +1,10 @@
package org.janelia.saalfeldlab.n5.codec;
import java.io.Serializable;
-import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.serialization.NameConfig;
/**
- * {@code Codec}s can encode and decode {@link ReadData} objects.
+ * {@code CodecInfo}s are an untyped semantic layer for {@link BlockCodec}s and {@link DataCodec}s.
*
* Modeled after Codecs in
* Zarr.
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDeterministicSizeDataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDeterministicSizeDataCodec.java
new file mode 100644
index 000000000..99febb924
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDeterministicSizeDataCodec.java
@@ -0,0 +1,21 @@
+package org.janelia.saalfeldlab.n5.codec;
+
+class ConcatenatedDeterministicSizeDataCodec extends ConcatenatedDataCodec implements DeterministicSizeDataCodec {
+
+ private final DeterministicSizeDataCodec[] codecs;
+
+ ConcatenatedDeterministicSizeDataCodec(final DeterministicSizeDataCodec[] codecs) {
+
+ super(codecs);
+ this.codecs = codecs;
+ }
+
+ @Override
+ public long encodedSize(long size) {
+
+ for (DeterministicSizeDataCodec codec : codecs) {
+ size = codec.encodedSize(size);
+ }
+ return size;
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java
index be40d2e59..50a11cbbf 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java
@@ -1,5 +1,6 @@
package org.janelia.saalfeldlab.n5.codec;
+import java.util.Arrays;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
@@ -45,6 +46,9 @@ public interface DataCodec {
/**
* Create a {@code DataCodec} that sequentially applies {@code codecs} in
* the given order for encoding, and in reverse order for decoding.
+ *
+ * If all {@code codecs} implement {@code DeterministicSizeDataCodec}, the
+ * returned {@code DataCodec} will also be a {@code DeterministicSizeDataCodec}.
*
* @param codecs
* a list of DataCodecs
@@ -58,6 +62,21 @@ static DataCodec concatenate(final DataCodec... codecs) {
if (codecs.length == 1)
return codecs[0];
- return new ConcatenatedDataCodec(codecs);
+ if (Arrays.stream(codecs).allMatch(DeterministicSizeDataCodec.class::isInstance))
+ return new ConcatenatedDeterministicSizeDataCodec(Arrays.copyOf(codecs, codecs.length, DeterministicSizeDataCodec[].class));
+ else
+ return new ConcatenatedDataCodec(codecs);
+ }
+
+ static DataCodec create(final DataCodecInfo... codecInfos) {
+
+ if (codecInfos == null)
+ throw new NullPointerException();
+
+ final DataCodec[] codecs = new DataCodec[codecInfos.length];
+ Arrays.setAll(codecs, i -> codecInfos[i].create());
+
+ return DataCodec.concatenate(codecs);
}
+
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java
index ca3ab8c89..44d84662d 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java
@@ -1,11 +1,13 @@
package org.janelia.saalfeldlab.n5.codec;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.janelia.saalfeldlab.n5.serialization.NameConfig;
/**
* {@code DataCodec}s transform one {@link ReadData} into another,
* for example, compressing it.
*/
+@NameConfig.Prefix("data-codec")
public interface DataCodecInfo extends CodecInfo {
DataCodec create();
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java
new file mode 100644
index 000000000..9fd9f0178
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java
@@ -0,0 +1,13 @@
+package org.janelia.saalfeldlab.n5.codec;
+
+/**
+ * A {@link CodecInfo} that can deterministically determine the size of encoded data from the size of the raw data and vice versa from the data length alone (i.e. encoding is data
+ * independent).
+ */
+public interface DeterministicSizeCodecInfo extends CodecInfo {
+
+ public abstract long encodedSize(long size);
+
+ public abstract long decodedSize(long size);
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java
new file mode 100644
index 000000000..5c533b34a
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java
@@ -0,0 +1,38 @@
+package org.janelia.saalfeldlab.n5.codec;
+
+/**
+ * A {@link DataCodec} that can deterministically determine the size of encoded
+ * data from the size of the raw data (i.e. encoding is data independent).
+ */
+public interface DeterministicSizeDataCodec extends DataCodec {
+
+ /**
+ * Given {@code size} bytes of raw data, how many bytes will the encoded
+ * data have.
+ *
+ * @param size in bytes
+ * @return encoded size in bytes
+ */
+ long encodedSize(long size);
+
+ /**
+ * Create a {@code DeterministicSizeDataCodec} that sequentially applies
+ * {@code codecs} in the given order for encoding, and in reverse order for
+ * decoding.
+ *
+ * @param codecs
+ * a list of DeterministicSizeDataCodec
+ * @return the concatenated DeterministicSizeDataCodec
+ */
+ static DeterministicSizeDataCodec concatenate(final DeterministicSizeDataCodec... codecs) {
+
+ if (codecs == null)
+ throw new NullPointerException();
+
+ if (codecs.length == 1)
+ return codecs[0];
+
+ return new ConcatenatedDeterministicSizeDataCodec(codecs);
+ }
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java
new file mode 100644
index 000000000..046b99b67
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java
@@ -0,0 +1,41 @@
+package org.janelia.saalfeldlab.n5.codec;
+
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.janelia.saalfeldlab.n5.serialization.NameConfig;
+
+@NameConfig.Name(IdentityCodec.TYPE)
+public class IdentityCodec implements DeterministicSizeDataCodec, DataCodecInfo {
+
+ private static final long serialVersionUID = 8354269325800855621L;
+
+ public static final String TYPE = "id";
+
+ @Override
+ public String getType() {
+
+ return TYPE;
+ }
+
+ @Override
+ public ReadData decode(ReadData readData) {
+
+ return readData;
+ }
+
+ @Override
+ public ReadData encode(ReadData readData) {
+
+ return readData;
+ }
+
+ @Override public DataCodec create() {
+
+ return this;
+ }
+
+ @Override
+ public long encodedSize(long size) {
+
+ return size;
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IndexCodecAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IndexCodecAdapter.java
new file mode 100644
index 000000000..94bc66bbb
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IndexCodecAdapter.java
@@ -0,0 +1,45 @@
+package org.janelia.saalfeldlab.n5.codec;
+
+public class IndexCodecAdapter {
+
+ private final BlockCodecInfo blockCodecInfo;
+ private final DeterministicSizeCodecInfo[] dataCodecs;
+
+ public IndexCodecAdapter(final BlockCodecInfo blockCodecInfo, final DeterministicSizeCodecInfo... dataCodecs) {
+
+ this.blockCodecInfo = blockCodecInfo;
+ this.dataCodecs = dataCodecs;
+ }
+
+ public BlockCodecInfo getBlockCodecInfo() {
+
+ return blockCodecInfo;
+ }
+
+ public DataCodecInfo[] getDataCodecs() {
+
+ final DataCodecInfo[] dataCodecs = new DataCodecInfo[this.dataCodecs.length];
+ System.arraycopy(this.dataCodecs, 0, dataCodecs, 0, this.dataCodecs.length);
+ return dataCodecs;
+ }
+
+ public long encodedSize(long initialSize) {
+ long totalNumBytes = initialSize;
+ for (DeterministicSizeCodecInfo codec : dataCodecs) {
+ totalNumBytes = codec.encodedSize(totalNumBytes);
+ }
+ return totalNumBytes;
+ }
+
+ public static IndexCodecAdapter create(final CodecInfo... codecs) {
+ if (codecs == null || codecs.length == 0)
+ return new IndexCodecAdapter(new RawBlockCodecInfo());
+
+ if (codecs[0] instanceof BlockCodecInfo)
+ return new IndexCodecAdapter((BlockCodecInfo)codecs[0]);
+
+ final DeterministicSizeCodecInfo[] indexCodecs = new DeterministicSizeCodecInfo[codecs.length];
+ System.arraycopy(codecs, 0, indexCodecs, 0, codecs.length);
+ return new IndexCodecAdapter(new RawBlockCodecInfo(), indexCodecs);
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java
index 70c3555b3..b8946a2f4 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java
@@ -1,6 +1,8 @@
package org.janelia.saalfeldlab.n5.codec;
+import org.janelia.saalfeldlab.n5.DataBlock;
import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.DataType;
import org.janelia.saalfeldlab.n5.serialization.NameConfig;
@NameConfig.Name(value = N5BlockCodecInfo.TYPE)
@@ -10,6 +12,25 @@ public class N5BlockCodecInfo implements BlockCodecInfo {
public static final String TYPE = "n5bytes";
+ private transient DatasetAttributes attributes;
+
+ @Override public long[] getKeyPositionForBlock(DatasetAttributes attributes, DataBlock datablock) {
+
+ return datablock.getGridPosition();
+ }
+
+ @Override public long[] getKeyPositionForBlock(DatasetAttributes attributes, long... blockPosition) {
+
+ return blockPosition;
+ }
+
+ @Override public long encodedSize(long size) {
+
+ final int[] blockSize = attributes.getBlockSize();
+ int headerSize = new N5BlockCodecs.BlockHeader(blockSize, DataBlock.getNumElements(blockSize)).getSize();
+ return headerSize + size;
+ }
+
@Override
public String getType() {
@@ -17,8 +38,7 @@ public String getType() {
}
@Override
- public BlockCodec create(final DatasetAttributes attributes, final DataCodec... dataCodecs) {
- return N5BlockCodecs.create(attributes.getDataType(), DataCodec.concatenate(dataCodecs));
+ public BlockCodec create(final DataType dataType, final int[] blockSize, final DataCodecInfo... codecInfos) {
+ return N5BlockCodecs.create(dataType, DataCodec.create(codecInfos));
}
-
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java
index 03e058c6f..300190d6d 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java
@@ -51,6 +51,7 @@
import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.MODE_DEFAULT;
import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.MODE_OBJECT;
import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.MODE_VARLENGTH;
+import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.headerSizeInBytes;
public class N5BlockCodecs {
@@ -120,9 +121,9 @@ private interface BlockCodecFactory {
abstract static class N5AbstractBlockCodec implements BlockCodec {
- private final FlatArrayCodec dataCodec;
+ final FlatArrayCodec dataCodec;
private final DataBlockFactory dataBlockFactory;
- private final DataCodec codec;
+ final DataCodec codec;
N5AbstractBlockCodec(FlatArrayCodec dataCodec, DataBlockFactory dataBlockFactory, DataCodec codec) {
this.dataCodec = dataCodec;
@@ -189,6 +190,18 @@ protected BlockHeader decodeBlockHeader(final InputStream in) throws N5IOExcepti
return BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH);
}
+
+ @Override
+ public long encodedSize(final int[] blockSize) throws UnsupportedOperationException {
+ if (codec instanceof DeterministicSizeDataCodec) {
+ final int bytesPerElement = dataCodec.bytesPerElement();
+ final int numElements = DataBlock.getNumElements(blockSize);
+ final int headerSize = headerSizeInBytes(MODE_DEFAULT, blockSize.length);
+ return headerSize + ((DeterministicSizeDataCodec) codec).encodedSize((long) numElements * bytesPerElement);
+ } else {
+ throw new UnsupportedOperationException();
+ }
+ }
}
/**
@@ -315,6 +328,25 @@ private static void writeBlockSize(final int[] blockSize, final DataOutputStream
}
}
+ static int headerSizeInBytes(final short mode, final int numDimensions) {
+ switch (mode) {
+ case MODE_DEFAULT:
+ return 2 + // 1 short for mode
+ 2 + // 1 short for blockSize.length
+ 4 * numDimensions; // 1 int for each blockSize element
+ case MODE_VARLENGTH:
+ return 2 +// 1 short for mode
+ 2 + // 1 short for blockSize.length
+ 4 * numDimensions + // 1 int for each blockSize dimension
+ 4; // 1 int for numElements
+ case MODE_OBJECT:
+ return 2 + // 1 short for mode
+ 4; // 1 int for numElements
+ default:
+ throw new N5Exception("unexpected mode: " + mode);
+ }
+ }
+
void writeTo(final OutputStream out) throws N5IOException {
try {
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java
index ed93433f0..a9cec7003 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java
@@ -1,8 +1,15 @@
package org.janelia.saalfeldlab.n5.codec;
import java.nio.ByteOrder;
+
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
import org.janelia.saalfeldlab.n5.DataType;
-import org.janelia.saalfeldlab.n5.DatasetAttributes;
import org.janelia.saalfeldlab.n5.serialization.NameConfig;
@@ -11,7 +18,7 @@ public class RawBlockCodecInfo implements BlockCodecInfo {
private static final long serialVersionUID = 3282569607795127005L;
- public static final String TYPE = "rawbytes";
+ public static final String TYPE = "bytes";
@NameConfig.Parameter(value = "endian", optional = true)
private final ByteOrder byteOrder;
@@ -37,12 +44,12 @@ public ByteOrder getByteOrder() {
}
@Override
- public BlockCodec create(final DatasetAttributes attributes, final DataCodec... dataCodecs) {
- ensureValidByteOrder(attributes.getDataType(), getByteOrder());
- return RawBlockCodecs.create(attributes.getDataType(), byteOrder, attributes.getBlockSize(), DataCodec.concatenate(dataCodecs));
+ public BlockCodec create(final DataType dataType, final int[] blockSize, final DataCodecInfo... codecInfos) {
+ ensureValidByteOrder(dataType, getByteOrder());
+ return RawBlockCodecs.create(dataType, byteOrder, blockSize, DataCodec.create(codecInfos));
}
- private static void ensureValidByteOrder(final DataType dataType, final ByteOrder byteOrder) {
+ public static void ensureValidByteOrder(final DataType dataType, final ByteOrder byteOrder) {
switch (dataType) {
case INT8:
@@ -55,4 +62,32 @@ private static void ensureValidByteOrder(final DataType dataType, final ByteOrde
if (byteOrder == null)
throw new IllegalArgumentException("DataType (" + dataType + ") requires ByteOrder, but was null");
}
+
+ public static ByteOrderAdapter byteOrderAdapter = new ByteOrderAdapter();
+
+ public static class ByteOrderAdapter implements JsonDeserializer, JsonSerializer {
+
+ @Override
+ public JsonElement serialize(ByteOrder src, java.lang.reflect.Type typeOfSrc,
+ JsonSerializationContext context) {
+
+ if (src.equals(ByteOrder.LITTLE_ENDIAN))
+ return new JsonPrimitive("little");
+ else
+ return new JsonPrimitive("big");
+ }
+
+ @Override
+ public ByteOrder deserialize(JsonElement json, java.lang.reflect.Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+
+ if (json.getAsString().equals("little"))
+ return ByteOrder.LITTLE_ENDIAN;
+ if (json.getAsString().equals("big"))
+ return ByteOrder.BIG_ENDIAN;
+
+ return null;
+ }
+
+ }
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java
index 1fc606500..4a1ddb4ca 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java
@@ -115,5 +115,16 @@ public DataBlock decode(ReadData readData, long[] gridPosition) {
final T data = dataCodec.decode(decodeData, numElements);
return dataBlockFactory.createDataBlock(blockSize, gridPosition, data);
}
+
+ @Override
+ public long encodedSize(final int[] blockSize) throws UnsupportedOperationException {
+ if (codec instanceof DeterministicSizeDataCodec) {
+ final int bytesPerElement = dataCodec.bytesPerElement();
+ final int numElements = DataBlock.getNumElements(blockSize);
+ return ((DeterministicSizeDataCodec) codec).encodedSize((long) numElements * bytesPerElement);
+ } else {
+ throw new UnsupportedOperationException();
+ }
+ }
}
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java
new file mode 100644
index 000000000..68b12cd0c
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java
@@ -0,0 +1,109 @@
+package org.janelia.saalfeldlab.n5.codec.checksum;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.zip.CheckedInputStream;
+import java.util.zip.CheckedOutputStream;
+import java.util.zip.Checksum;
+
+import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.codec.CodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DataCodec;
+import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+
+/**
+ * A {@link CodecInfo} that appends a checksum to data when encoding and can validate against that checksum when decoding.
+ */
+public abstract class ChecksumCodec implements DataCodec, DataCodecInfo, DeterministicSizeCodecInfo {
+
+ private static final long serialVersionUID = 3141427377277375077L;
+
+ private int numChecksumBytes;
+
+ private Checksum checksum;
+
+ public ChecksumCodec(Checksum checksum, int numChecksumBytes) {
+
+ this.checksum = checksum;
+ this.numChecksumBytes = numChecksumBytes;
+ }
+
+ public Checksum getChecksum() {
+
+ return checksum;
+ }
+
+ public int numChecksumBytes() {
+
+ return numChecksumBytes;
+ }
+
+ private CheckedOutputStream createStream(OutputStream out) {
+ return new CheckedOutputStream(out, getChecksum()) {
+
+ private boolean closed = false;
+ @Override public void close() throws IOException {
+
+ if (!closed) {
+ writeChecksum(out);
+ closed = true;
+ out.close();
+ }
+ }
+ };
+ }
+
+ @Override public ReadData encode(ReadData readData) {
+
+ return readData.encode(this::createStream);
+
+ }
+
+ @Override public ReadData decode(ReadData readData) throws N5IOException {
+
+
+ return ReadData.from(new CheckedInputStream(readData.inputStream(), getChecksum()));
+ }
+
+ @Override
+ public long encodedSize(final long size) {
+
+ return size + numChecksumBytes();
+ }
+
+ @Override
+ public long decodedSize(final long size) {
+
+ return size - numChecksumBytes();
+ }
+
+ protected boolean valid(InputStream in) throws IOException {
+
+ return readChecksum(in) == getChecksum().getValue();
+ }
+
+ protected long readChecksum(InputStream in) throws IOException {
+
+ final byte[] checksum = new byte[numChecksumBytes()];
+ in.read(checksum);
+ return ByteBuffer.wrap(checksum).getLong();
+ }
+
+ /**
+ * Return the value of the checksum as a {@link ByteBuffer} to be serialized.
+ *
+ * @return a ByteBuffer representing the checksum value
+ */
+ public abstract ByteBuffer getChecksumValue();
+
+ public void writeChecksum(OutputStream out) throws IOException {
+
+ out.write(getChecksumValue().array());
+ }
+
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumException.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumException.java
new file mode 100644
index 000000000..034343c42
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumException.java
@@ -0,0 +1,12 @@
+package org.janelia.saalfeldlab.n5.codec.checksum;
+
+public class ChecksumException extends Exception {
+
+ private static final long serialVersionUID = 905130066386622561L;
+
+ public ChecksumException(final String message) {
+
+ super(message);
+ }
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java
new file mode 100644
index 000000000..93c701ab1
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java
@@ -0,0 +1,52 @@
+package org.janelia.saalfeldlab.n5.codec.checksum;
+
+import org.apache.commons.lang3.NotImplementedException;
+import org.janelia.saalfeldlab.n5.codec.DataCodec;
+import org.janelia.saalfeldlab.n5.serialization.NameConfig;
+
+import java.nio.ByteBuffer;
+import java.util.zip.CRC32;
+
+@NameConfig.Name(Crc32cChecksumCodec.TYPE)
+public class Crc32cChecksumCodec extends ChecksumCodec {
+
+ private static final long serialVersionUID = 7424151868725442500L;
+
+ public static final String TYPE = "crc32c";
+
+ public Crc32cChecksumCodec() {
+
+ super(new CRC32(), 4);
+ }
+
+ @Override
+ public long encodedSize(final long size) {
+
+ return size + numChecksumBytes();
+ }
+
+ @Override
+ public long decodedSize(final long size) {
+
+ return size - numChecksumBytes();
+ }
+
+ @Override
+ public ByteBuffer getChecksumValue() {
+
+ final ByteBuffer buf = ByteBuffer.allocate(numChecksumBytes());
+ buf.putInt((int)getChecksum().getValue());
+ return buf;
+ }
+
+ @Override
+ public String getType() {
+
+ return TYPE;
+ }
+
+ @Override public DataCodec create() {
+
+ return this;
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java
index a2305d655..cbd5caf56 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java
@@ -24,9 +24,14 @@ interface SegmentsAndData {
}
/**
- * Wrap {@code readData} and create one segment comprising the entire {@code
+ * Wrap a {@link ReadData} and create one segment comprising the entire
+ * {@code
* readData}. The segment can be retrieved as the first (and only) element
* of {@link SegmentedReadData#segments()}.
+ *
+ * @param readData
+ * the ReadData to wrap
+ * @return the SegmentedReadData
*/
static SegmentedReadData wrap(ReadData readData) {
return new DefaultSegmentedReadData(readData);
@@ -35,8 +40,15 @@ static SegmentedReadData wrap(ReadData readData) {
/**
* Wrap {@code readData} and create segments at the given locations. The
* order of segments in the returned {@link SegmentsAndData#segments()} list
- * matches the order of the given {@code locations} (while the {@link
- * #segments} in the {@link SegmentsAndData#data()} are ordered by offset).
+ * matches the order of the given {@code locations} (while the
+ * {@link #segments} in the {@link SegmentsAndData#data()} are ordered by
+ * offset).
+ *
+ * @param readData
+ * the ReadData to wrap
+ * @param locations
+ * the ranges for segments
+ * @return the SegmentsAndData
*/
static SegmentsAndData wrap(ReadData readData, Range... locations) {
return wrap(readData, Arrays.asList(locations));
@@ -45,8 +57,15 @@ static SegmentsAndData wrap(ReadData readData, Range... locations) {
/**
* Wrap {@code readData} and create segments at the given locations. The
* order of segments in the returned {@link SegmentsAndData#segments()} list
- * matches the order of the given {@code locations} (while the {@link
- * #segments} in the {@link SegmentsAndData#data()} are ordered by offset).
+ * matches the order of the given {@code locations} (while the
+ * {@link #segments} in the {@link SegmentsAndData#data()} are ordered by
+ * offset).
+ *
+ * @param readData
+ * the ReadData to wrap
+ * @param locations
+ * the ranges for segments
+ * @return the SegmentsAndData
*/
static SegmentsAndData wrap(ReadData readData, List locations) {
return DefaultSegmentedReadData.wrap(readData, locations);
@@ -57,41 +76,45 @@ static SegmentsAndData wrap(ReadData readData, List locations) {
* given {@code readDatas}. The concatenation contains the segments of all
* concatenated {@code readData}s with appropriately offset locations.
*
- * In particular, it is also possible to concatenate {@code SegmentedReadData}s
- * with (yet) unknown length. (This is useful for postponing compression of
- * DataBlocks until they are actually written.) In that case, segment locations
- * are only available after all lengths become known. This happens when
- * concatenation (or all its constituents) is {@link #materialize()
- * materialized} or {@link #writeTo(OutputStream) written}.
+ * In particular, it is also possible to concatenate
+ * {@code SegmentedReadData}s with (yet) unknown length. (This is useful for
+ * postponing compression of DataBlocks until they are actually written.) In
+ * that case, segment locations are only available after all lengths become
+ * known. This happens when concatenation (or all its constituents) is
+ * {@link #materialize() materialized} or {@link #writeTo(OutputStream)
+ * written}.
+ *
+ * @param readDatas
+ * a list of ReadDatra to concatenate
+ * @return the SegmentedReadData comprising all the input readDatas
*/
static SegmentedReadData concatenate(List readDatas) {
return new ConcatenatedReadData(readDatas);
}
-
-
/**
* Returns the location of {@code segment} in this {@code ReadData}.
*
- * Note that this {@code ReadData} is not necessarily the source of the segment.
+ * Note that this {@code ReadData} is not necessarily the source of the
+ * segment.
*
* The returned {@code Range} may be {@code {offset=0, length=-1}}, which
- * means that the segment comprises this whole {@code ReadData} (and the length of
- * this {@code ReadData} is not yet known).
+ * means that the segment comprises this whole {@code ReadData} (and the
+ * length of this {@code ReadData} is not yet known).
*
* @param segment
- * the segment id
+ * the segment id
*
* @return location of the segment, or null
*
* @throws IllegalArgumentException
- * if the segment is not contained in this ReadData
+ * if the segment is not contained in this ReadData
*/
Range location(Segment segment) throws IllegalArgumentException;
/**
- * Return all segments (fully) contained in this {@code ReadData}, ordered by location
- * (that is, sorted by {@link Range#COMPARATOR}).
+ * Return all segments (fully) contained in this {@code ReadData}, ordered
+ * by location (that is, sorted by {@link Range#COMPARATOR}).
*
* @return all segments contained in this {@code ReadData}.
*/
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java b/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java
index f19776906..184837b55 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java
@@ -63,6 +63,20 @@ public interface NameConfig extends Serializable {
String value();
}
+ /**
+ * Controls whether a class should be serializable as a {@code NameConfig}.
+ *
+ * This annotation allows explicitly enabling or disabling serialization for a class.
+ *
+ * By default, classes are serialized.
+ */
+ @Retention(RetentionPolicy.RUNTIME)
+ @Inherited
+ @Target(ElementType.TYPE)
+ @Indexable @interface Serialize {
+ boolean value() default true;
+ }
+
/**
* Marks a field as a parameter to be serialized.
*
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java
new file mode 100644
index 000000000..a7a2f6d99
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java
@@ -0,0 +1,68 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;
+import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.TreeMap;
+
+/**
+ * Wrap an instantiated DataBlock/shard codec hierarchy to implement (single and
+ * batch) DataBlock read/write methods.
+ *
+ * @param
+ * type of the data contained in the DataBlock
+ */
+public interface DatasetAccess {
+
+ DataBlock readBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException;
+
+ void writeBlock(PositionValueAccess kva, DataBlock dataBlock) throws N5IOException;
+
+ boolean deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException;
+
+ List> readBlocks(PositionValueAccess kva, List positions);
+
+ void writeBlocks(PositionValueAccess kva, List> blocks);
+
+ NestedGrid getGrid();
+
+ /**
+ * Sort a list of {@link NestedPosition}s by their parent level {@link NestedPosition}.
+ * nestedPositions are grouped at `outerLevel`.
+ * If {@code NestedPosition.level()} level is already equivalent to {@code NestedGrid.numLevels() - 1}, nestedPositions is returned.
+ *
+ * @param grid to sort the blocky by
+ * @param innerPositions to group per shard.
+ * @param outerLevel of the outerLevel shard position to group by. must be in range {@code [1, NestedGrid.numLevels() - 1]}
+ * @return map of outerLevel shard positions to inner level block positions
+ */
+ static Collection> groupInnerPositions(final NestedGrid grid, final List innerPositions, final int outerLevel) {
+
+ if (outerLevel < 1 || outerLevel >= grid.numLevels())
+ throw new IllegalArgumentException("outerLevel must be in range [1, grid.numLevels() - 1]");
+
+ if (innerPositions.isEmpty())
+ return Collections.emptyList();
+
+ final TreeMap> blocksPerShard = new TreeMap<>();
+ for (T nestedPosition : innerPositions) {
+ final NestedPosition outerNestedPosition;
+ if (nestedPosition.level() == outerLevel)
+ outerNestedPosition = nestedPosition;
+ else
+ outerNestedPosition = new NestedPosition(grid, nestedPosition.absolute(0), outerLevel);
+
+
+ final List blocks = blocksPerShard.computeIfAbsent(outerNestedPosition, it -> new ArrayList<>());
+ blocks.add(nestedPosition);
+ }
+ return blocksPerShard.values();
+ }
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java
new file mode 100644
index 000000000..43029582d
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java
@@ -0,0 +1,352 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.N5Exception;
+import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException;
+import org.janelia.saalfeldlab.n5.codec.BlockCodec;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;
+import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition;
+
+import static org.janelia.saalfeldlab.n5.shard.DatasetAccess.*;
+
+public class DefaultDatasetAccess implements DatasetAccess {
+
+ private final NestedGrid grid;
+ private final BlockCodec[] codecs;
+
+ public DefaultDatasetAccess(final NestedGrid grid, final BlockCodec[] codecs) {
+ this.grid = grid;
+ this.codecs = codecs;
+ }
+
+ public NestedGrid getGrid() {
+ return grid;
+ }
+
+ @Override
+ public DataBlock readBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException {
+ final NestedPosition position = new NestedPosition(grid, gridPosition);
+ return readBlockRecursive(kva.get(position.key()), position, grid.numLevels() - 1);
+ }
+
+ private DataBlock readBlockRecursive(
+ final ReadData readData,
+ final NestedPosition position,
+ final int level) {
+ if (readData == null) {
+ return null;
+ } else if (level == 0) {
+ @SuppressWarnings("unchecked")
+ final BlockCodec codec = (BlockCodec) codecs[0];
+ return codec.decode(readData, position.absolute(0));
+ } else {
+ @SuppressWarnings("unchecked")
+ final BlockCodec codec = (BlockCodec) codecs[level];
+ final RawShard shard = codec.decode(readData, position.absolute(level)).getData();
+ return readBlockRecursive(shard.getElementData(position.relative(level - 1)), position, level - 1);
+ }
+ }
+
+ @Override
+ public List> readBlocks(PositionValueAccess pva, List positions) {
+
+ if (grid.numLevels() == 1) {
+ return positions.stream().map(it -> readBlock(pva, it)).collect(Collectors.toList());
+ }
+
+ final List blockPositions = positions.stream().map(it -> new NestedPosition(grid, it)).collect(Collectors.toList());
+
+ final int outermostLevel = grid.numLevels() - 1;
+ final Collection> blocksPerOutermostShard = groupInnerPositions(grid, blockPositions, outermostLevel);
+
+ final ArrayList> blocks = new ArrayList<>(blockPositions.size());
+ for (List blocksInSingleShard : blocksPerOutermostShard) {
+ if (blocksInSingleShard.isEmpty())
+ continue;
+
+ final NestedPosition firstBlock = blocksInSingleShard.get(0);
+ final ReadData readData = pva.get(firstBlock.key());
+ final List> shardBlocks;
+ try {
+ shardBlocks = readShardRecursive(readData, blocksInSingleShard, outermostLevel);
+ } catch (N5NoSuchKeyException e) {
+ continue;
+ }
+
+ blocks.addAll(shardBlocks);
+ }
+
+ //TODO Caleb: No guarantee of order; If we want that, we need to sort the result.
+ return blocks;
+ }
+
+ /**
+ * Bulk Read operation on a shard. `positions` MUST all be in the same shard.
+ * That is, for each `position` in `positions`, `position.absolute(level)` must be the same.
+ *
+ * @param readData for the corresponding shard
+ * @param positions of blocks within the shard to be read
+ * @param level of the shard
+ * @return list of blocks read from a single shard
+ */
+ private List> readShardRecursive(
+ final ReadData readData,
+ final List positions,
+ final int level
+ ) {
+ // Cannot have a shard at level 0
+ if (readData == null || level == 0) {
+ return null;
+ }
+
+ if (positions.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ final NestedPosition firstBlock = positions.get(0);
+ final long[] shardPosition = firstBlock.absolute(level);
+
+ final BlockCodec codec = (BlockCodec) codecs[level];
+ final RawShard shard = codec.decode(readData, shardPosition).getData();
+
+ final ArrayList> blocks = new ArrayList<>(positions.size());
+ if (level == 1) {
+ final int innerMostLevel = 0;
+ //Base case; read the blocks
+ for (NestedPosition blockPosition : positions) {
+ final long[] elementPos = blockPosition.relative(innerMostLevel);
+ final ReadData elementData = shard.getElementData(elementPos);
+ final DataBlock block = readBlockRecursive(elementData, blockPosition, innerMostLevel);
+ blocks.add(block);
+ }
+ } else {
+ // group the blocks by shard for next level, and call again for each nested shard
+ final Collection> nextLevelShards = groupInnerPositions(grid, positions, level - 1);
+ for (List innerPositions : nextLevelShards) {
+ final List> innerBlocks = readShardRecursive(readData, innerPositions, level - 1);
+ blocks.addAll(innerBlocks);
+ }
+ }
+
+ return blocks;
+ }
+
+ @Override
+ public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException {
+ final NestedPosition position = new NestedPosition(grid, dataBlock.getGridPosition());
+ final long[] key = position.key();
+
+ final ReadData existingData = getExistingReadData(pva, key);
+ final ReadData modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1);
+ pva.put(key, modifiedData);
+ }
+
+ private ReadData writeBlockRecursive(
+ final ReadData existingReadData,
+ final DataBlock dataBlock,
+ final NestedPosition position,
+ final int level) {
+ if (level == 0) {
+ @SuppressWarnings("unchecked")
+ final BlockCodec codec = (BlockCodec) codecs[0];
+ return codec.encode(dataBlock);
+ } else {
+ @SuppressWarnings("unchecked")
+ final BlockCodec codec = (BlockCodec) codecs[level];
+ final long[] gridPos = position.absolute(level);
+ final RawShard shard = existingReadData == null ?
+ new RawShard(grid.relativeBlockSize(level)) :
+ codec.decode(existingReadData, gridPos).getData();
+ final long[] elementPos = position.relative(level - 1);
+ final ReadData existingElementData = (level == 1)
+ ? null // if level == 1, we don't need to extract the nested (DataBlock) ReadData because it will be overridden anyway
+ : shard.getElementData(elementPos);
+ final ReadData modifiedElementData = writeBlockRecursive(existingElementData, dataBlock, position, level - 1);
+ shard.setElementData(modifiedElementData, elementPos);
+ return codec.encode(new RawShardDataBlock(gridPos, shard));
+ }
+ }
+
+ @Override public void writeBlocks(PositionValueAccess pva, List> dataBlocks) {
+
+ if (grid.numLevels() == 1) {
+ dataBlocks.forEach(it -> writeBlock(pva, it));
+ return;
+ }
+
+ final List> nestedPositionDataBlocks = dataBlocks.stream()
+ .map(it -> new NestedPositionDataBlock<>(grid, it))
+ .collect(Collectors.toList());
+
+ final int outermostLevel = grid.numLevels() - 1;
+ final Collection>> dataBlocksPerShard = groupInnerPositions(grid, nestedPositionDataBlocks, outermostLevel);
+
+ for (List> dataBlocksForShard : dataBlocksPerShard) {
+ if (dataBlocksForShard.isEmpty())
+ continue;
+
+ final NestedPositionDataBlock firstDataBlock = dataBlocksForShard.get(0);
+ final long[] shardKey = firstDataBlock.key();
+
+ //TODO Caleb: When writing all blocks in a shard, we don't need to read existing data.
+ // Also, could only materialize the index and skip reading if we are overwriting all existing blocks.
+ final ReadData existingReadData = getExistingReadData(pva, shardKey);
+ final ReadData writeShardReadData = writeShardRecursive(existingReadData, dataBlocksForShard, outermostLevel);
+
+ pva.put(shardKey, writeShardReadData);
+ }
+ }
+
+ private ReadData writeShardRecursive(
+ final ReadData existingShard,
+ final List> dataBlocks,
+ final int level) {
+
+ // cannot have shard level 0, or nothing to write
+ if (level == 0 || dataBlocks.isEmpty()) {
+ return null;
+ }
+
+ final BlockCodec codec = (BlockCodec)codecs[level];
+ final NestedPositionDataBlock firstBlock = dataBlocks.get(0);
+ final long[] shardPosition = firstBlock.absolute(level);
+
+ final RawShard shard;
+ if (existingShard == null)
+ shard = new RawShard(grid.relativeBlockSize(level));
+ else
+ shard = codec.decode(existingShard, shardPosition).getData();
+
+ if (level == 1) {
+ final int innerMostLevel = 0;
+ // Base case, write the blocks
+ for (NestedPositionDataBlock nestedPosDataBlock : dataBlocks) {
+ final DataBlock dataBlock = nestedPosDataBlock.getDataBlock();
+
+ final long[] blockRelativePos = nestedPosDataBlock.relative(innerMostLevel);
+ final ReadData blockReadData = shard.getElementData(blockRelativePos);
+ final ReadData modifiedShardBlock = writeBlockRecursive(blockReadData, dataBlock, nestedPosDataBlock, innerMostLevel);
+ shard.setElementData(modifiedShardBlock, blockRelativePos);
+ }
+ } else {
+ final Collection>> dataBlocksForInnerShards = groupInnerPositions(grid, dataBlocks, level - 1);
+ for (List> innerShardDataBlocks : dataBlocksForInnerShards) {
+ if (innerShardDataBlocks.isEmpty())
+ continue;
+
+ final ReadData innerShardReadData = writeShardRecursive(existingShard, innerShardDataBlocks, level - 1);
+
+ final NestedPositionDataBlock firstInnerNestedPosBlock = innerShardDataBlocks.get(0);
+ final long[] relPosInShard = firstInnerNestedPosBlock.relative(level - 1);
+ shard.setElementData(innerShardReadData, relPosInShard);
+ }
+ }
+ return codec.encode(new RawShardDataBlock(shardPosition, shard));
+ }
+
+ @Override
+ public boolean deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException {
+ final NestedPosition position = new NestedPosition(grid, gridPosition);
+ final long[] key = position.key();
+ if (grid.numLevels() == 1) {
+ // for non-sharded dataset, don't bother getting the value, just remove the key.
+ try {
+ return kva.remove(key);
+ } catch (final Exception e) {
+ throw new N5Exception("The shard at " + Arrays.toString(key) + " could not be deleted.", e);
+ }
+ } else {
+ final ReadData existingData = kva.get(key);
+ final ReadData modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1);
+ if (existingData != null && modifiedData == null) {
+ return kva.remove(key);
+ } else if (modifiedData != existingData) {
+ kva.put(key, modifiedData);
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ private ReadData deleteBlockRecursive(
+ final ReadData existingReadData,
+ final NestedPosition position,
+ final int level) {
+ if (level == 0 || existingReadData == null) {
+ return null;
+ } else {
+ @SuppressWarnings("unchecked")
+ final BlockCodec codec = (BlockCodec) codecs[level];
+ final long[] gridPos = position.absolute(level);
+ final RawShard shard = codec.decode(existingReadData, gridPos).getData();
+ final long[] elementPos = position.relative(level - 1);
+ final ReadData existingElementData = shard.getElementData(elementPos);
+ if (existingElementData == null) {
+ // The DataBlock (or the whole nested shard containing it) does not exist.
+ // This shard remains unchanged.
+ return existingReadData;
+ } else {
+ final ReadData modifiedElementData = deleteBlockRecursive(existingElementData, position, level - 1);
+ if (modifiedElementData == existingElementData) {
+ // The nested shard was not modified.
+ // This shard remains unchanged.
+ return existingReadData;
+ }
+ shard.setElementData(modifiedElementData, elementPos);
+ if (modifiedElementData == null) {
+ // The DataBlock or nested shard was removed.
+ // Check whether this shard becomes empty.
+ if (shard.index().allElementsNull()) {
+ // This shard is empty and should be removed.
+ return null;
+ }
+ }
+ return codec.encode(new RawShardDataBlock(gridPos, shard));
+ }
+ }
+ }
+
+ private static ReadData getExistingReadData(final PositionValueAccess pva, final long[] key) {
+ // need to read the shard anyway, and currently (Sept 24 2025)
+ // have no way to tell if they key exist from what is in this method except to attempt
+ // to materialize and catch the N5NoSuchKeyException
+ try {
+ ReadData existingData = pva.get(key);
+ if (existingData != null)
+ existingData.materialize();
+ return existingData;
+ } catch (N5NoSuchKeyException e) {
+ return null;
+ }
+ }
+
+ /**
+ * NestedPosition wrapper for a DataBlock. Useful for grouping DataBlock by nested shard position.
+ *
+ * @param type of the datablock
+ */
+ private static class NestedPositionDataBlock extends NestedPosition {
+
+ private final DataBlock dataBlock;
+
+ private NestedPositionDataBlock(NestedGrid grid, DataBlock dataBlock) {
+
+ super(grid, dataBlock.getGridPosition(), 0);
+ this.dataBlock = dataBlock;
+ }
+
+ private DataBlock getDataBlock() {
+
+ return dataBlock;
+ }
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java
new file mode 100644
index 000000000..7b7cc0473
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java
@@ -0,0 +1,136 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import java.util.Arrays;
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.codec.BlockCodec;
+import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.CodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
+import org.janelia.saalfeldlab.n5.serialization.N5Annotations;
+import org.janelia.saalfeldlab.n5.serialization.NameConfig;
+import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation;
+
+/**
+ * Default (and probably only) implementation of {@link ShardCodecInfo}.
+ */
+@NameConfig.Name(value = "sharding_indexed")
+public class DefaultShardCodecInfo implements ShardCodecInfo {
+
+ @Override
+ public String getType() {
+ return "sharding_indexed";
+ }
+
+ @N5Annotations.ReverseArray
+ @NameConfig.Parameter(value = "chunk_shape")
+ private final int[] innerBlockSize;
+
+ @NameConfig.Parameter(value = "index_location")
+ private final IndexLocation indexLocation;
+
+ @NameConfig.Parameter
+ private CodecInfo[] codecs;
+
+ @NameConfig.Parameter(value = "index_codecs")
+ private CodecInfo[] indexCodecs;
+
+ private transient final BlockCodecInfo innerBlockCodecInfo;
+
+ private transient final DataCodecInfo[] innerDataCodecInfos;
+
+ private transient final BlockCodecInfo indexBlockCodecInfo;
+
+ private transient final DataCodecInfo[] indexDataCodecInfos;
+
+ DefaultShardCodecInfo() {
+ // for serialization
+ this(null, null, null, null, null, null);
+ }
+
+ public DefaultShardCodecInfo(
+ final int[] innerBlockSize,
+ final BlockCodecInfo innerBlockCodecInfo,
+ final DataCodecInfo[] innerDataCodecInfos,
+ final BlockCodecInfo indexBlockCodecInfo,
+ final DataCodecInfo[] indexDataCodecInfos,
+ final IndexLocation indexLocation) {
+
+ this.innerBlockSize = innerBlockSize;
+ this.innerBlockCodecInfo = innerBlockCodecInfo;
+ this.innerDataCodecInfos = innerDataCodecInfos;
+ this.indexBlockCodecInfo = indexBlockCodecInfo;
+ this.indexDataCodecInfos = indexDataCodecInfos;
+ this.indexLocation = indexLocation;
+
+ codecs = concatenateCodecs(innerBlockCodecInfo, innerDataCodecInfos);
+ indexCodecs = concatenateCodecs(indexBlockCodecInfo, indexDataCodecInfos);
+ }
+
+ @Override
+ public int[] getInnerBlockSize() {
+ return innerBlockSize;
+ }
+
+ @Override
+ public BlockCodecInfo getInnerBlockCodecInfo() {
+ return innerBlockCodecInfo;
+ }
+
+ @Override
+ public DataCodecInfo[] getInnerDataCodecInfos() {
+ return innerDataCodecInfos;
+ }
+
+ @Override
+ public BlockCodecInfo getIndexBlockCodecInfo() {
+ return indexBlockCodecInfo;
+ }
+
+ @Override
+ public DataCodecInfo[] getIndexDataCodecInfos() {
+ return indexDataCodecInfos;
+ }
+
+ @Override
+ public IndexLocation getIndexLocation() {
+ return indexLocation;
+ }
+
+ public CodecInfo[] getCodecs() {
+ return codecs;
+ }
+
+ public CodecInfo[] getIndexCodecs() {
+ return indexCodecs;
+ }
+
+ @Override
+ public RawShardCodec create(final int[] blockSize, final DataCodecInfo... codecs) {
+
+ // Number of elements (DataBlocks, nested shards) in each dimension per shard.
+ final int[] size = new int[blockSize.length];
+ // blockSize argument is number of pixels in the shard
+ // innerBlockSize is number of pixels in each shard element (nested shard or DataBlock)
+ Arrays.setAll(size, d -> blockSize[d] / innerBlockSize[d]);
+
+ final BlockCodec indexCodec = indexBlockCodecInfo.create(
+ DataType.UINT64,
+ ShardIndex.blockSizeFromIndexSize(size),
+ indexDataCodecInfos);
+
+ return new RawShardCodec(size, indexLocation, indexCodec);
+ }
+
+ private static CodecInfo[] concatenateCodecs(BlockCodecInfo blkInfo, DataCodecInfo[] dataInfos) {
+
+ if (dataInfos == null) {
+ return new CodecInfo[]{blkInfo};
+ }
+
+ final CodecInfo[] allCodecs = new CodecInfo[dataInfos.length + 1];
+ allCodecs[0] = blkInfo;
+ System.arraycopy(dataInfos, 0, allCodecs, 1, dataInfos.length);
+
+ return allCodecs;
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java
new file mode 100644
index 000000000..892900ae4
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java
@@ -0,0 +1,347 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+
+import java.util.ArrayList;
+
+// TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// [ ] NestedGrid
+// [ ] validation in constructor
+// [ ] test for that validation
+// [ ] javadoc
+//
+// [ ] NestedPosition interface
+// [+] LazyNestedPosition class
+// [+] fields: NestedGrid, long[] position, int level
+// [+] construct with source level 0
+// [+] minimal abs/rel access methods
+// [+] toString()
+// [-] extract NestedPosition interface
+// ==> postpone until necessary
+// [ ] equals / hashcode
+// [ ] should we have prefix()? suffix()? head()? tail()?
+// [+] Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...).
+// For nested = {X,Y,Z} compare by Z, then Y, then X.
+// For X = {x,y,z} compare by z, then y, then x. (flattening order)
+//
+// TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.janelia.saalfeldlab.n5.util.GridIterator;
+
+public class Nesting {
+
+ public static void main(String[] args) {
+ final int[][] blockSizes = {{1,1,1}, {3,2,2}, {6,4,4}, {24,24,24}};
+ final NestedGrid grid = new NestedGrid(blockSizes);
+ final NestedPosition pos = new NestedPosition(grid, new long[] {38, 7, 129});
+ System.out.println("pos = " + pos);
+ System.out.println("key = " + Arrays.toString(pos.key()));
+ }
+
+ public static class NestedPosition implements Comparable {
+
+ private final NestedGrid grid;
+ private final long[] position;
+ private final int level;
+
+ public NestedPosition(final NestedGrid grid, final long[] position, final int level) {
+ this.grid = grid;
+ this.position = position;
+ this.level = level;
+ }
+
+ public NestedPosition(final NestedGrid grid, final long[] position) {
+ this(grid, position, 0);
+ }
+
+ /**
+ * Get the nesting level of this position.
+ *
+ * Positions with {@code level=0} refer to DataBlocks, positions with
+ * {@code level=1} refer to first-level shards (containing DataBlocks),
+ * and so on.
+ *
+ * @return nesting level
+ */
+ public int level() {
+ return level;
+ }
+
+ public int numDimensions() {
+ return grid.numDimensions();
+ }
+
+ /**
+ * Get the relative grid position at {@code level}, that is, relative
+ * offset within containing the {@code (level+1)} element.
+ *
+ * @param level
+ * requested nesting level
+ *
+ * @return relative grid position
+ */
+ public long[] relative(final int level) {
+ return grid.relativePosition(position, level);
+ }
+
+ /**
+ * Get the absolute grid position at {@code level}.
+ *
+ * @param level
+ * requested nesting level
+ *
+ * @return absolute grid position
+ */
+ public long[] absolute(final int level) {
+ return grid.absolutePosition(position, level);
+ }
+
+ public long[] key() {
+ return relative(grid.numLevels() - 1);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append('{');
+ for ( int l = level; l < grid.numLevels(); ++l ) {
+ if ( l > level ) {
+ sb.append(" / ");
+ }
+ sb.append(Arrays.toString(relative(l)));
+ }
+ sb.append(" (level ").append(level).append(")}");
+ return sb.toString();
+ }
+
+ @Override public int compareTo(NestedPosition o) {
+
+ final int dimensionInequality = Integer.compare(numDimensions(), o.numDimensions());
+ if (dimensionInequality != 0)
+ return dimensionInequality;
+
+ final int levelInequality = Integer.compare(level, o.level);
+ if (levelInequality != 0)
+ return levelInequality;
+
+ final long[] otherAbsPos = o.absolute(level);
+ final long[] absPos = absolute(level);
+
+ for (int i = absPos.length - 1; i >= 0; --i) {
+ final long diff = absPos[i] - otherAbsPos[i];
+ if (diff != 0)
+ return (int)diff;
+ }
+
+ return 0;
+ }
+
+ // TODO: equals() and hashCode()
+ // TODO: should we have prefix()? suffix()? head()? tail()?
+ }
+
+ /**
+ * A nested grid of blocks used to coordinate the relationships of shards and the blocks / chunks they contain.
+ */
+ public static class NestedGrid {
+
+ private final int numLevels;
+
+ private final int numDimensions;
+
+ // relativeToBase[i][d] is block size at level i relative to level 0
+ private final int[][] relativeToBase;
+
+ // relativeToAdjacent[i][d] is block size at level i relative to level i-1
+ private final int[][] relativeToAdjacent;
+
+ private final int[][] blockSizes;
+
+ /**
+ * {@code blockSizes[l][d]} is the block size at level {@code l} in dimension {@code d}.
+ * Level 0 contains the smallest blocks. blockSizes[l+1][d] must be a multiple of blockSizes[l][d].
+ *
+ * @param blockSizes
+ * block sizes for all levels and dimensions.
+ */
+ public NestedGrid(int[][] blockSizes) {
+
+ if (blockSizes == null)
+ throw new IllegalArgumentException("blockSizes is null");
+
+ if (blockSizes[0] == null)
+ throw new IllegalArgumentException("blockSizes[0] is null");
+
+ this.blockSizes = blockSizes;
+
+ numLevels = blockSizes.length;
+ numDimensions = blockSizes[0].length;
+ relativeToBase = new int[numLevels][numDimensions];
+ relativeToAdjacent = new int[numLevels][numDimensions];
+ for (int l = 0; l < numLevels; ++l) {
+ final int k = Math.max(0, l - 1);
+
+ if (blockSizes[l] == null)
+ throw new IllegalArgumentException("blockSizes[" + l + "] null");
+
+ if (blockSizes[l].length != numDimensions)
+ throw new IllegalArgumentException(
+ String.format("Block size at level %d has a different length (%d vs %d)", l, numDimensions, blockSizes[l].length));
+
+ for (int d = 0; d < numDimensions; ++d) {
+
+ if (blockSizes[l][d] <= 0 ) {
+ throw new IllegalArgumentException(
+ String.format("Block sizes at level %d (%d) is negative for dimension %d.",
+ l, blockSizes[l][d], d));
+ }
+
+ if (blockSizes[l][d] < blockSizes[k][d]) {
+ throw new IllegalArgumentException(
+ String.format("Block sizes at level %d (%d) is smaller than previous level (%d) "
+ + " for dimension %d.",
+ l, blockSizes[l][d], blockSizes[k][d], d));
+ }
+
+ if (blockSizes[l][d] % blockSizes[k][d] != 0) {
+ throw new IllegalArgumentException(
+ String.format("Block sizes at level %d (%d) not a multiple of previous level (%d) "
+ + " for dimension %d.",
+ l, blockSizes[l][d], blockSizes[k][d], d));
+ }
+
+ relativeToBase[l][d] = blockSizes[l][d] / blockSizes[0][d];
+ relativeToAdjacent[l][d] = blockSizes[l][d] / blockSizes[k][d];
+ }
+ }
+ }
+
+ public int numLevels() {
+ return numLevels;
+ }
+
+ public int numDimensions() {
+ return numDimensions;
+ }
+
+ public int[] getBlockSize(int level) {
+ return blockSizes[level];
+ }
+
+ public void absolutePosition(
+ final long[] sourcePos,
+ final int sourceLevel,
+ final long[] targetPos,
+ final int targetLevel) {
+ final int[] sk = relativeToBase[sourceLevel];
+ final int[] si = relativeToBase[targetLevel];
+ for (int d = 0; d < numDimensions; ++d) {
+ targetPos[d] = sourcePos[d] * sk[d] / si[d];
+ }
+ }
+
+ public void relativePosition(
+ final long[] sourcePos,
+ final int sourceLevel,
+ final long[] targetPos,
+ final int targetLevel) {
+ absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel);
+ if (targetLevel < numLevels - 1) {
+ final int[] rj = relativeToAdjacent[targetLevel + 1];
+ for (int d = 0; d < numDimensions; ++d) {
+ targetPos[d] %= rj[d];
+ }
+ }
+ }
+
+ /**
+ * The absolute position of * source position for the given source level at the target level.
+ *
+ * @param sourcePos the source position j
+ * @param sourceLevel the source level
+ * @param targetLevel the target level
+ * @return absolute position at the target level
+ */
+ public long[] absolutePosition(
+ final long[] sourcePos,
+ final int sourceLevel,
+ final int targetLevel) {
+ final long[] targetPos = new long[numDimensions];
+ absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel);
+ return targetPos;
+ }
+
+ /**
+ * The absolute position of the level 0 source position at
+ * the target level.
+ *
+ * @param sourcePos the source position j
+ * @param targetLevel the target level
+ * @return absolute position at the target level
+ */
+ public long[] absolutePosition(
+ final long[] sourcePos,
+ final int targetLevel) {
+ return absolutePosition(sourcePos, 0, targetLevel);
+ }
+
+ public long[] relativePosition(
+ final long[] sourcePos,
+ final int sourceLevel,
+ final int targetLevel) {
+ final long[] targetPos = new long[numDimensions];
+ relativePosition(sourcePos, sourceLevel, targetPos, targetLevel);
+ return targetPos;
+ }
+
+ public long[] relativePosition(
+ final long[] sourcePos,
+ final int targetLevel) {
+ return relativePosition(sourcePos, 0, targetLevel);
+ }
+
+ public int[] relativeBlockSize(final int level) {
+ return relativeToAdjacent[level];
+ }
+
+ public int[] absoluteBlockSize(final int level) {
+ return relativeToBase[level];
+ }
+
+ /**
+ * Given a block position at a particular level, returns a list of
+ * positions of all sub-blocks at a particular subLevel.
+ *
+ * Can be used to get a list of chunk positions for a shard with a
+ * particular position.
+ *
+ * @param position
+ * a position
+ * @param level
+ * the nesting level of the given position
+ * @param subLevel
+ * the nesting sub-level of positions to return
+ * @return the sub-block positions
+ */
+ public List positionInSubGrid(long[] position, int level, int subLevel) {
+
+ final long[] subPosition = new long[numDimensions()];
+ absolutePosition(position, level, subPosition, subLevel);
+
+ final int[] numElementsInSubGrid = absoluteBlockSize(numLevels() - 1);
+ final GridIterator git = new GridIterator(GridIterator.int2long(numElementsInSubGrid), subPosition);
+
+ // TODO return NestedPositions instead?
+ final ArrayList positions = new ArrayList<>();
+ while (git.hasNext())
+ positions.add(git.next().clone());
+
+ return positions;
+ }
+
+ }
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java
new file mode 100644
index 000000000..a9cd70319
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java
@@ -0,0 +1,103 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+
+import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.KeyValueAccess;
+import org.janelia.saalfeldlab.n5.LockedChannel;
+import org.janelia.saalfeldlab.n5.N5Exception;
+import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+
+/**
+ * Idea is to wrap a KeyValueAccess and a dataset URI to be able to get/put values (ReadData) by {@code long[]} key
+ */
+public interface PositionValueAccess {
+
+ /**
+ * Gets the {@link ReadData} for the DataBlock (or shard) at the given
+ * position in the block (or shard) grid.
+ *
+ * @param key
+ * The position of the data block or shard
+ * @return ReadData for the given key or {@code null} if the key doesn't
+ * exist
+ * @throws N5Exception.N5IOException
+ * if an error occurs while reading
+ */
+ ReadData get(long[] key) throws N5Exception.N5IOException;
+
+ void put(long[] key, ReadData data) throws N5Exception.N5IOException;
+
+ boolean remove(long[] key) throws N5Exception.N5IOException;
+
+ public static PositionValueAccess fromKva(
+ final KeyValueAccess kva,
+ final URI uri,
+ final String normalPath,
+ final DatasetAttributes attributes) {
+
+ return new KvaPositionValueAccess(kva, uri, normalPath, attributes);
+ }
+
+ class KvaPositionValueAccess implements PositionValueAccess {
+
+ private final KeyValueAccess kva;
+ private final URI uri;
+ private final String normalPath;
+ private final DatasetAttributes attributes;
+
+ KvaPositionValueAccess(final KeyValueAccess kva,
+ final URI uri,
+ final String normalPath,
+ final DatasetAttributes attributes) {
+
+ this.kva = kva;
+ this.uri = uri;
+ this.normalPath = normalPath;
+ this.attributes = attributes;
+ }
+
+ /**
+ * Constructs the absolute path for a data block (or shard) at a given grid
+ * position.
+ *
+ * @param gridPosition
+ * to the target data block
+ * @return the absolute path to the data block ad gridPosition
+ */
+ protected String absolutePath( final long... gridPosition) {
+ return kva.compose(uri, normalPath, attributes.relativeBlockPath(gridPosition));
+ }
+
+ @Override
+ public ReadData get(long[] key) throws N5IOException {
+ return kva.createReadData(absolutePath(key));
+ }
+
+ @Override
+ public void put(long[] key, ReadData data) throws N5IOException {
+
+ try ( final LockedChannel ch = kva.lockForWriting(absolutePath(key));
+ final OutputStream outputStream = ch.newOutputStream();) {
+ data.writeTo(outputStream);
+ } catch (IOException e) {
+ throw new N5IOException(e);
+ }
+ }
+
+ @Override
+ public boolean remove(long[] gridPosition) throws N5IOException {
+
+ final String key = absolutePath(gridPosition);
+ if (!kva.isFile(key))
+ return false;
+
+ kva.delete(key);
+ return true;
+ }
+
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java
new file mode 100644
index 000000000..0a8cb5a7d
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java
@@ -0,0 +1,57 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.janelia.saalfeldlab.n5.readdata.segment.Segment;
+import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData;
+import org.janelia.saalfeldlab.n5.shard.ShardIndex.NDArray;
+
+public class RawShard {
+
+ private final SegmentedReadData sourceData;
+
+ private final NDArray index;
+
+ RawShard(final int[] size) {
+ sourceData = null;
+ index = new NDArray<>(size, Segment[]::new);
+ }
+
+ RawShard(final SegmentedReadData sourceData, final NDArray index) {
+ this.sourceData = sourceData;
+ this.index = index;
+ }
+
+ RawShard(final ShardIndex.SegmentIndexAndData segmentIndexAndData) {
+ this(segmentIndexAndData.data(), segmentIndexAndData.index());
+ }
+
+ /**
+ * The ReadData from which the shard was constructed, or {@code null} for a
+ * new empty shard.
+ *
+ * @return this shard's source ReadData, or null.
+ */
+ public SegmentedReadData sourceData() {
+ return sourceData;
+ }
+
+ /**
+ * Maps grid position of shard elements to {@link Segment}s that give the
+ * byte range for the blocks in this shard.
+ *
+ * @return an NDArray of segments
+ */
+ public NDArray index() {
+ return index;
+ }
+
+ public ReadData getElementData(final long[] pos) {
+ final Segment segment = index.get(pos);
+ return segment == null ? null : segment.source().slice(segment);
+ }
+
+ public void setElementData(final ReadData data, final long[] pos) {
+ final Segment segment = data == null ? null : SegmentedReadData.wrap(data).segments().get(0);
+ index.set(segment, pos);
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java
new file mode 100644
index 000000000..8ba859336
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java
@@ -0,0 +1,83 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import static org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation.START;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.N5Exception;
+import org.janelia.saalfeldlab.n5.codec.BlockCodec;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.janelia.saalfeldlab.n5.readdata.segment.Segment;
+import org.janelia.saalfeldlab.n5.readdata.Range;
+import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData;
+import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation;
+import org.janelia.saalfeldlab.n5.shard.ShardIndex.NDArray;
+
+public class RawShardCodec implements BlockCodec {
+
+ /**
+ * Number of elements (DataBlocks, nested shards) in each dimension per shard.
+ */
+ private final int[] size;
+ private final IndexLocation indexLocation;
+ private final BlockCodec indexCodec;
+ private final long indexBlockSizeInBytes;
+
+ RawShardCodec(final int[] size, final IndexLocation indexLocation, final BlockCodec indexCodec) {
+
+ this.size = size;
+ this.indexLocation = indexLocation;
+ this.indexCodec = indexCodec;
+ indexBlockSizeInBytes = indexCodec.encodedSize(ShardIndex.blockSizeFromIndexSize(size));
+ }
+
+ @Override
+ public ReadData encode(final DataBlock shard) throws N5Exception.N5IOException {
+
+ // concatenate slices for all non-null segments in shard.getData().index()
+ final NDArray index = shard.getData().index();
+ final List readDatas = new ArrayList<>();
+ // TODO: Any clever ReadData grouping, slice merging, etc. should go here
+ // This basic implementation just slices ReadData for all non-null
+ // elements and concatenates in flat index order.
+ for (Segment segment : index.data) {
+ if (segment != null) {
+ readDatas.add(segment.source().slice(segment));
+ }
+ }
+ final SegmentedReadData data = SegmentedReadData.concatenate(readDatas);
+
+ final ReadData.OutputStreamWriter writer;
+ if (indexLocation == START) {
+ data.materialize();
+ final NDArray locations = ShardIndex.locations(index, data);
+ final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, indexBlockSizeInBytes);
+ final ReadData indexReadData = indexCodec.encode(indexDataBlock);
+ writer = out -> {
+ indexReadData.writeTo(out);
+ data.writeTo(out);
+ };
+ } else { // indexLocation == END
+ writer = out -> {
+ data.writeTo(out);
+ final NDArray locations = ShardIndex.locations(index, data);
+ final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, 0);
+ final ReadData indexReadData = indexCodec.encode(indexDataBlock);
+ indexReadData.writeTo(out);
+ };
+ }
+ return ReadData.from(writer);
+ }
+
+ @Override
+ public DataBlock decode(final ReadData readData, final long[] gridPosition) throws N5Exception.N5IOException {
+
+ final long indexOffset = (indexLocation == START) ? 0 : (readData.requireLength() - indexBlockSizeInBytes);
+ final ReadData indexReadData = readData.slice(indexOffset, indexBlockSizeInBytes);
+ final DataBlock indexDataBlock = indexCodec.decode(indexReadData, new long[size.length]);
+ final NDArray locations = ShardIndex.fromDataBlock(indexDataBlock);
+ final ShardIndex.SegmentIndexAndData segments = ShardIndex.segments(locations, readData);
+ return new RawShardDataBlock(gridPosition, new RawShard(segments));
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java
new file mode 100644
index 000000000..e36017ffd
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java
@@ -0,0 +1,42 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import org.janelia.saalfeldlab.n5.DataBlock;
+
+/**
+ * Wrap a RawShard as a DataBlock.
+ * This basically just adds a gridPosition for the shard.
+ */
+public class RawShardDataBlock implements DataBlock {
+
+ private final long[] gridPosition;
+
+ private final RawShard shard;
+
+ RawShardDataBlock(final long[] gridPosition, final RawShard shard) {
+ this.gridPosition = gridPosition;
+ this.shard = shard;
+ }
+
+ // TODO: should this be the number of elements in the Shard (number of
+ // sub-shards / datablock) along each dimension, or the number of
+ // pixels alon each dimension?
+ @Override
+ public int[] getSize() {
+ return shard.index().size();
+ }
+
+ @Override
+ public long[] getGridPosition() {
+ return gridPosition;
+ }
+
+ @Override
+ public int getNumElements() {
+ return shard.index().numElements();
+ }
+
+ @Override
+ public RawShard getData() {
+ return shard;
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java
new file mode 100644
index 000000000..dcc40f3d7
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java
@@ -0,0 +1,53 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
+import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation;
+
+public interface ShardCodecInfo extends BlockCodecInfo {
+
+ /**
+ * Chunk size of each shard element (either nested shard or DataBlock)
+ *
+ * @return the size of each shard element
+ */
+ int[] getInnerBlockSize();
+
+ /**
+ * BlockCodecInfo for shard elements (either nested shard or DataBlock)
+ *
+ * @return the BlockCodecInfo for DataBlocks in this shard
+ */
+ BlockCodecInfo getInnerBlockCodecInfo();
+
+ /**
+ * @return the collection of DataCodecInfos applied to data blocks for this
+ * shard.
+ */
+ DataCodecInfo[] getInnerDataCodecInfos();
+
+ /**
+ * BlockCodec for shard index
+ *
+ * @return the BlockCodecInfo for this shard's index
+ */
+ BlockCodecInfo getIndexBlockCodecInfo();
+
+ /**
+ * Deterministic-size DataCodecs for index BlockCodec
+ *
+ * @return the collection of DataCodecInfos for this shard's index
+ */
+ DataCodecInfo[] getIndexDataCodecInfos();
+
+ IndexLocation getIndexLocation();
+
+ @SuppressWarnings("unchecked")
+ @Override
+ default RawShardCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs) {
+ return create(blockSize, codecs);
+ }
+
+ RawShardCodec create(int[] blockSize, DataCodecInfo... codecs);
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java
new file mode 100644
index 000000000..d5d25eda7
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java
@@ -0,0 +1,238 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.IntFunction;
+import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.LongArrayDataBlock;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.janelia.saalfeldlab.n5.readdata.segment.Segment;
+import org.janelia.saalfeldlab.n5.readdata.Range;
+import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData;
+import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData.SegmentsAndData;
+
+import com.google.gson.annotations.SerializedName;
+
+public class ShardIndex {
+
+ private ShardIndex() {
+ // utility class. should not be instantiated.
+ }
+
+ public enum IndexLocation {
+ @SerializedName("start") START,
+ @SerializedName("end") END
+ }
+
+ /**
+ * Access flat {@code T[]} array as n-dimensional array.
+ *
+ * @param
+ * element type
+ */
+ public static class NDArray {
+
+ final int[] size;
+ private final int[] stride;
+ final T[] data;
+
+ NDArray(final int[] size, final IntFunction createArray) {
+ this.size = size;
+ stride = getStrides(size);
+ data = createArray.apply(getNumElements(size));
+ }
+
+ NDArray(final int[] size, final T[] data) {
+ this.size = size;
+ stride = getStrides(size);
+ this.data = data;
+ }
+
+ T get(long... position) {
+ return data[index(position)];
+ }
+
+ void set(T value, long... position) {
+ data[index(position)] = value;
+ }
+
+ private int index(long... position) {
+ int index = 0;
+ for (int i = 0; i < stride.length; i++) {
+ index += stride[i] * position[i];
+ }
+ return index;
+ }
+
+ public int[] size() {
+ return size;
+ }
+
+ public int numElements() {
+ return data.length;
+ }
+
+ public boolean allElementsNull() {
+ for (T t : data) {
+ if (t != null) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+
+ static int getNumElements(final int[] size) {
+ int numElements = 1;
+ for (int s : size) {
+ numElements *= s;
+ }
+ return numElements;
+ }
+
+ static int[] getStrides(final int[] size) {
+ final int n = size.length;
+ final int[] stride = new int[n];
+ stride[0] = 1;
+ for (int i = 1; i < n; i++) {
+ stride[i] = stride[i - 1] * size[i - 1];
+ }
+ return stride;
+ }
+
+ /**
+ * Special value indicating an empty block entry in the index.
+ * Used for both offset and length when a block doesn't exist.
+ */
+ static final long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL;
+
+ /**
+ * Size of first dimension of the {@code DataBlock} representation of the shard index.
+ */
+ private static final int LONGS_PER_BLOCK = 2;
+
+ static NDArray fromDataBlock( final DataBlock block ) {
+
+ final long[] blockData = block.getData();
+ final int[] size = indexSizeFromBlockSize(block.getSize());
+ final int n = getNumElements(size);
+ final Range[] locations = new Range[n];
+
+ for (int i = 0; i < n; i++) {
+ long offset = blockData[i * LONGS_PER_BLOCK];
+ long length = blockData[i * LONGS_PER_BLOCK + 1];
+ if (offset != EMPTY_INDEX_NBYTES && length != EMPTY_INDEX_NBYTES) {
+ locations[i] = Range.at(offset, length);
+ }
+ }
+ return new NDArray<>(size, locations);
+ }
+
+ static DataBlock toDataBlock( final NDArray locations, final long offset ) {
+
+ final Range[] data = locations.data;
+
+ final int[] blockSize = blockSizeFromIndexSize(locations.size);
+ final long[] blockData = new long[data.length * 2];
+
+ for (int i = 0; i < data.length; ++i) {
+ if (data[i] != null) {
+ blockData[i * LONGS_PER_BLOCK] = data[i].offset() + offset;
+ blockData[i * LONGS_PER_BLOCK + 1] = data[i].length();
+ } else {
+ blockData[i * LONGS_PER_BLOCK] = EMPTY_INDEX_NBYTES;
+ blockData[i * LONGS_PER_BLOCK + 1] = EMPTY_INDEX_NBYTES;
+ }
+ }
+ return new LongArrayDataBlock(blockSize, new long[blockSize.length], blockData);
+ }
+
+ /**
+ * Prepends a value to an array.
+ *
+ * @param value the value to prepend
+ * @param array the original array
+ * @return a new array with the value prepended
+ */
+ private static int[] prepend(final int value, final int[] array) {
+
+ final int[] indexBlockSize = new int[array.length + 1];
+ indexBlockSize[0] = value;
+ System.arraycopy(array, 0, indexBlockSize, 1, array.length);
+ return indexBlockSize;
+ }
+
+ /**
+ * Prepends {@code LONGS_PER_BLOCK} to the {@code indexSize} array.
+ */
+ static int[] blockSizeFromIndexSize(final int[] indexSize) {
+ return prepend(LONGS_PER_BLOCK, indexSize);
+ }
+
+ /**
+ * Strips first element (should be {@code LONGS_PER_BLOCK} from the {@code blockSize} array.
+ */
+ static int[] indexSizeFromBlockSize(final int[] blockSize) {
+ assert blockSize[ 0 ] == LONGS_PER_BLOCK;
+ return Arrays.copyOfRange(blockSize, 1, blockSize.length);
+ }
+
+ /**
+ * Retrieves the {@code SegmentLocation} of each non-null {@code Segment} in
+ * {@code segments}. Returns a {@code NDArray} with entries
+ * corresponding tho the {@code segments} entries.
+ */
+ static NDArray locations(final NDArray segments, final SegmentedReadData readData) {
+
+ final Segment[] data = segments.data;
+ final Range[] locations = new Range[data.length];
+ for (int i = 0; i < data.length; ++i) {
+ final Segment segment = data[i];
+ if ( segment != null ) {
+ locations[i] = readData.location(segment);
+ }
+ }
+ return new NDArray<>(segments.size, locations);
+ }
+
+ interface SegmentIndexAndData {
+ NDArray index();
+ SegmentedReadData data();
+ }
+
+ /**
+ * Puts a {@code Segment} at each non-null {@code SegmentLocation} in {@code
+ * locations} on the given {@code readData}. Returns both the {@code
+ * SegmentedReadData} with these segments and a {@code NDArray}
+ * with segment entries corresponding to the {@code locations} entries.
+ */
+ static SegmentIndexAndData segments(final NDArray locations, final ReadData readData) {
+
+ final Range[] locationsData = locations.data;
+ final Segment[] segmentsData = new Segment[locationsData.length];
+
+ final List presentLocations = new ArrayList<>();
+ for (int i = 0; i < locationsData.length; i++) {
+ if (locationsData[i] != null) {
+ presentLocations.add(locationsData[i]);
+ }
+ }
+
+ final SegmentsAndData segmentsAndData = SegmentedReadData.wrap(readData, presentLocations);
+ final Iterator presentSegments = segmentsAndData.segments().iterator();
+ for (int i = 0; i < locationsData.length; i++) {
+ if (locationsData[i] != null) {
+ segmentsData[i] = presentSegments.next();
+ }
+ }
+
+ final NDArray index = new NDArray<>(locations.size, segmentsData);
+ final SegmentedReadData data = segmentsAndData.data();
+ return new SegmentIndexAndData() {
+ @Override public NDArray index() {return index;}
+ @Override public SegmentedReadData data() {return data;}
+ };
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java b/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java
new file mode 100644
index 000000000..1b7076d54
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java
@@ -0,0 +1,38 @@
+package org.janelia.saalfeldlab.n5.util;
+
+/*
+ * An immutable {@Position}.
+ */
+public class FinalPosition implements Position {
+
+ public final long[] position;
+
+ public FinalPosition(long[] position) {
+ this.position = position;
+ }
+
+ public FinalPosition(Position p) {
+ this.position = p.get().clone();
+ }
+
+ @Override
+ public long[] get() {
+ return position;
+ }
+
+ @Override
+ public long get(int i) {
+ return position[i];
+ }
+
+ @Override
+ public String toString() {
+ return Position.toString(this);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return Position.equals(this, obj);
+ }
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java
new file mode 100644
index 000000000..43258efaf
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java
@@ -0,0 +1,178 @@
+package org.janelia.saalfeldlab.n5.util;
+
+import java.util.Iterator;
+
+/**
+ * Essentially imglib2's IntervalIterator, but N5 does not depend on imglib2.
+ */
+public class GridIterator implements Iterator {
+
+ final protected long[] dimensions;
+
+ final protected long[] steps;
+
+ final protected long[] position;
+
+ final protected int[] intPosition;
+
+ final protected long[] min;
+
+ final protected int lastIndex;
+
+ protected int index = -1;
+
+ public GridIterator(final long[] dimensions, final long[] min) {
+
+ final int n = dimensions.length;
+ this.dimensions = new long[n];
+ this.position = new long[n];
+ this.intPosition = new int[n];
+ this.min = min;
+ steps = new long[n];
+
+ final int m = n - 1;
+ long k = steps[0] = 1;
+ for (int d = 0; d < m; ) {
+ final long dimd = dimensions[d];
+ this.dimensions[d] = dimd;
+ k *= dimd;
+ steps[++d] = k;
+ }
+ final long dimm = dimensions[m];
+ this.dimensions[m] = dimm;
+ lastIndex = (int)(k * dimm - 1);
+ }
+
+ public GridIterator(final long[] dimensions) {
+
+ this(dimensions, new long[dimensions.length]);
+ }
+
+ public GridIterator(final int[] dimensions) {
+
+ this(int2long(dimensions));
+ }
+
+ public void fwd() {
+
+ ++index;
+ }
+
+ public void reset() {
+
+ index = -1;
+ }
+
+ @Override
+ public boolean hasNext() {
+
+ return index < lastIndex;
+ }
+
+ @Override
+ public long[] next() {
+
+ fwd();
+ indexToPosition(index, dimensions, min, position);
+ return position;
+ }
+
+ public int[] nextInt() {
+
+ next();
+ long2int(position, intPosition);
+ return intPosition;
+ }
+
+ public int getIndex() {
+
+ return index;
+ }
+
+ public static void indexToPosition(long index, final long[] dimensions, final long[] offset,
+ final long[] position) {
+
+ for (int dim = 0; dim < dimensions.length; dim++) {
+ position[dim] = (index % dimensions[dim]) + offset[dim];
+ index /= dimensions[dim];
+ }
+ }
+
+ public static void indexToPosition(long index, final int[] dimensions, final long[] offset,
+ final long[] position) {
+
+ for (int dim = 0; dim < dimensions.length; dim++) {
+ position[dim] = (index % dimensions[dim]) + offset[dim];
+ index /= dimensions[dim];
+ }
+ }
+
+ final static public long positionToIndex(final long[] dimensions, final long[] position) {
+ long idx = 0;
+ int cumulativeSize = 1;
+ for (int i = 0; i < position.length; i++) {
+ idx += position[i] * cumulativeSize;
+ cumulativeSize *= dimensions[i];
+ }
+ return idx;
+ }
+
+ final static public long positionToIndex(final long[] dimensions, final int[] position) {
+ long idx = 0;
+ int cumulativeSize = 1;
+ for (int i = 0; i < position.length; i++) {
+ idx += position[i] * cumulativeSize;
+ cumulativeSize *= dimensions[i];
+ }
+ return idx;
+ }
+
+ final static public long positionToIndex(final int[] dimensions, final long[] position) {
+ long idx = 0;
+ int cumulativeSize = 1;
+ for (int i = 0; i < position.length; i++) {
+ idx += position[i] * cumulativeSize;
+ cumulativeSize *= dimensions[i];
+ }
+ return idx;
+ }
+
+ final static public long positionToIndex(final int[] dimensions, final int[] position) {
+ long idx = 0;
+ int cumulativeSize = 1;
+ for (int i = 0; i < position.length; i++) {
+ idx += position[i] * cumulativeSize;
+ cumulativeSize *= dimensions[i];
+ }
+ return idx;
+ }
+
+ public static int[] long2int(final long[] src, final int[] tgt) {
+
+ for (int d = 0; d < tgt.length; ++d)
+ tgt[d] = (int)src[d];
+
+ return tgt;
+ }
+
+ public static int[] long2int(final long[] a) {
+
+ final int[] i = new int[a.length];
+
+ for (int d = 0; d < a.length; ++d)
+ i[d] = (int)a[d];
+
+ return i;
+ }
+
+ public static long[] int2long(final int[] i) {
+
+ final long[] l = new long[i.length];
+
+ for (int d = 0; d < l.length; ++d)
+ l[d] = i[d];
+
+ return l;
+ }
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java
new file mode 100644
index 000000000..2403835b9
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java
@@ -0,0 +1,66 @@
+package org.janelia.saalfeldlab.n5.util;
+
+import java.util.Arrays;
+
+/*
+ * A wrapper around a primitive long array that is lexicographically {@link Comparable}
+ * and for which we can test equality.
+ */
+public interface Position extends Comparable {
+
+ long[] get();
+
+ long get(int i);
+
+ default int numDimensions() {
+ return get().length;
+ }
+
+ @Override
+ default int compareTo(Position other) {
+
+ // use Arrays.compare when we update to Java 9+
+ final int N = numDimensions() > other.numDimensions() ? numDimensions() : other.numDimensions();
+ for (int i = 0; i < N; i++) {
+ final long diff = get(i) - other.get(i);
+ if (diff != 0)
+ return (int) diff;
+ }
+ return 0;
+ }
+
+ static boolean equals(final Position a, final Object b) {
+
+ if (a == null && b == null)
+ return true;
+
+ if (b == null)
+ return false;
+
+ if (!(b instanceof Position))
+ return false;
+
+ final Position other = (Position) b;
+ if (other.numDimensions() != a.numDimensions())
+ return false;
+
+ for (int i = 0; i < a.numDimensions(); i++)
+ if (other.get(i) != a.get(i))
+ return false;
+
+ return true;
+ }
+
+ static String toString(Position p) {
+ return "Position: " + Arrays.toString(p.get());
+ }
+
+ static Position wrap(final long[] p) {
+ return new FinalPosition(p);
+ }
+
+ static Position wrap(final int[] p) {
+ return new FinalPosition(GridIterator.int2long(p));
+ }
+
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java
index f6b332654..60025e453 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017--2021, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import static org.junit.Assert.assertArrayEquals;
@@ -60,6 +35,7 @@
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
import java.net.URI;
@@ -82,7 +58,9 @@
import org.janelia.saalfeldlab.n5.N5Reader.Version;
import org.janelia.saalfeldlab.n5.url.UriAttributeTest;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.gson.GsonBuilder;
@@ -107,8 +85,8 @@ public abstract class AbstractN5Test {
static protected final String groupName = "/test/group";
static protected final String[] subGroupNames = new String[]{"a", "b", "c"};
static protected final String datasetName = "/test/group/dataset";
- static protected final long[] dimensions = new long[]{100, 200, 300};
- static protected final int[] blockSize = new int[]{44, 33, 22};
+ static protected final long[] dimensions = new long[]{6, 15, 35};
+ static protected final int[] blockSize = new int[]{3, 5, 7};
static protected final int blockNumElements = blockSize[0] * blockSize[1] * blockSize[2];
static protected byte[] byteBlock;
@@ -132,7 +110,7 @@ public static URI createTempUri(String prefix, String suffix, URI base) {
return N5URI.getAsUri(name);
}
- protected final N5Writer createTempN5Writer() {
+ public N5Writer createTempN5Writer() {
try {
return createTempN5Writer(tempN5Location());
@@ -141,7 +119,7 @@ protected final N5Writer createTempN5Writer() {
}
}
- protected final N5Writer createTempN5Writer(String location) {
+ public final N5Writer createTempN5Writer(String location) {
return createTempN5Writer(location, new GsonBuilder());
}
@@ -160,6 +138,7 @@ protected final N5Writer createTempN5Writer(String location, GsonBuilder gson) {
@After
public void removeTempWriters() {
+
synchronized (tempWriters) {
for (final N5Writer writer : tempWriters) {
try {
@@ -208,7 +187,7 @@ protected Compression[] getCompressions() {
@Before
public void setUpOnce() {
- final Random rnd = new Random();
+ final Random rnd = new Random(111);
byteBlock = new byte[blockNumElements];
shortBlock = new short[blockNumElements];
intBlock = new int[blockNumElements];
@@ -252,22 +231,107 @@ public void testSetAttributeDoesntCreateGroup() {
}
@Test
- public void testCreateDataset() {
+ public void testCreateDataset() {
- final DatasetAttributes info;
- try (N5Writer writer = createTempN5Writer()) {
- writer.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression());
+ final DatasetAttributes info;
+ try (N5Writer writer = createTempN5Writer()) {
+ writer.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression());
+
+ assertTrue("Dataset does not exist", writer.exists(datasetName));
+
+ info = writer.getDatasetAttributes(datasetName);
+ }
+ assertArrayEquals(dimensions, info.getDimensions());
+ assertArrayEquals(blockSize, info.getBlockSize());
+ assertEquals(DataType.UINT64, info.getDataType());
+ }
+
+ @Test
+ public void testBlocksLargerThanDimensions() {
+
+ // Test case where block size is larger than dataset dimensions
+ final long[] smallDimensions = new long[]{2, 3, 4};
+ final int[] largeBlockSize = new int[]{5, 7, 10};
+
+ try (final N5Writer n5 = createTempN5Writer()) {
+ n5.createDataset(datasetName, smallDimensions, largeBlockSize, DataType.UINT8, new RawCompression());
+ final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName);
+
+ // Create a block that is larger than the dataset dimensions
+ final int numElements = largeBlockSize[0] * largeBlockSize[1] * largeBlockSize[2];
+ final byte[] data = new byte[numElements];
+ for (int i = 0; i < numElements; i++) {
+ data[i] = (byte)(i % 256);
+ }
+
+ final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(largeBlockSize, new long[]{0, 0, 0}, data);
+ n5.writeBlock(datasetName, attributes, dataBlock);
+
+ // Read the block back
+ final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0);
+ assertNotNull("Block should be readable", loadedDataBlock);
+ assertArrayEquals("Block size should match", largeBlockSize, loadedDataBlock.getSize());
+ assertArrayEquals("Block data should match", data, (byte[])loadedDataBlock.getData());
+ }
+ }
+
+ @Test
+ public void testUnalignedBlocksTruncatedAtEnd() {
+
+ // Test case where dimensions don't evenly divide by block size
+ final long[] unalignedDimensions = new long[]{5, 14, 33};
+ final int[] testBlockSize = new int[]{3, 5, 7};
+
+ try (final N5Writer n5 = createTempN5Writer()) {
+ n5.createDataset(datasetName, unalignedDimensions, testBlockSize, DataType.INT32, new RawCompression());
+ final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName);
- assertTrue("Dataset does not exist", writer.exists(datasetName));
+ // Test writing to the last block in dimension 0 (should be truncated to size 2 instead of 3)
+ final int[] truncatedBlockSize0 = new int[]{2, 5, 7}; // [3-4] in dim 0
+ final int numElements0 = truncatedBlockSize0[0] * truncatedBlockSize0[1] * truncatedBlockSize0[2];
+ final int[] data0 = new int[numElements0];
+ for (int i = 0; i < numElements0; i++) {
+ data0[i] = i + 1000;
+ }
+ final IntArrayDataBlock dataBlock0 = new IntArrayDataBlock(truncatedBlockSize0, new long[]{1, 0, 0}, data0);
+ n5.writeBlock(datasetName, attributes, dataBlock0);
+
+ final DataBlock loadedBlock0 = n5.readBlock(datasetName, attributes, 1, 0, 0);
+ assertNotNull("Truncated block should be readable", loadedBlock0);
+ assertArrayEquals("Truncated block data should match", data0, (int[])loadedBlock0.getData());
+
+ // Test writing to the last block in dimension 1 (should be truncated to size 4 instead of 5)
+ final int[] truncatedBlockSize1 = new int[]{3, 4, 7}; // [10-13] in dim 1
+ final int numElements1 = truncatedBlockSize1[0] * truncatedBlockSize1[1] * truncatedBlockSize1[2];
+ final int[] data1 = new int[numElements1];
+ for (int i = 0; i < numElements1; i++) {
+ data1[i] = i + 2000;
+ }
+ final IntArrayDataBlock dataBlock1 = new IntArrayDataBlock(truncatedBlockSize1, new long[]{0, 2, 0}, data1);
+ n5.writeBlock(datasetName, attributes, dataBlock1);
+
+ final DataBlock loadedBlock1 = n5.readBlock(datasetName, attributes, 0, 2, 0);
+ assertNotNull("Truncated block should be readable", loadedBlock1);
+ assertArrayEquals("Truncated block data should match", data1, (int[])loadedBlock1.getData());
+
+ // Test writing to the last block in dimension 2 (should be truncated to size 5 instead of 7)
+ final int[] truncatedBlockSize2 = new int[]{3, 5, 5}; // [28-32] in dim 2
+ final int numElements2 = truncatedBlockSize2[0] * truncatedBlockSize2[1] * truncatedBlockSize2[2];
+ final int[] data2 = new int[numElements2];
+ for (int i = 0; i < numElements2; i++) {
+ data2[i] = i + 3000;
+ }
+ final IntArrayDataBlock dataBlock2 = new IntArrayDataBlock(truncatedBlockSize2, new long[]{0, 0, 4}, data2);
+ n5.writeBlock(datasetName, attributes, dataBlock2);
- info = writer.getDatasetAttributes(datasetName);
+ final DataBlock loadedBlock2 = n5.readBlock(datasetName, attributes, 0, 0, 4);
+ assertNotNull("Truncated block should be readable", loadedBlock2);
+ assertArrayEquals("Truncated block data should match", data2, (int[])loadedBlock2.getData());
}
- assertArrayEquals(dimensions, info.getDimensions());
- assertArrayEquals(blockSize, info.getBlockSize());
- assertEquals(DataType.UINT64, info.getDataType());
- assertTrue(info.getCompression() instanceof RawCompression);
}
+
+
@Test
public void testWriteReadByteBlock() {
@@ -283,7 +347,6 @@ public void testWriteReadByteBlock() {
final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0);
assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData());
-
}
}
}
@@ -460,11 +523,63 @@ public void testWriteReadSerializableBlock() throws ClassNotFoundException {
}
}
+ @Test
+ @Ignore // TODO
+ public void testWriteInvalidBlock() {
+
+ final Compression compression = getCompressions()[0];
+ final DataType dataType = DataType.UINT8;
+
+ final int[] biggerBlockSize = Arrays.stream(blockSize).map(x -> x + 2).toArray();
+ int nBigger = Arrays.stream(biggerBlockSize).reduce(1, (x, y) -> x * y);
+
+ final int[] smallerBlockSize = Arrays.stream(blockSize).map(x -> x - 2).toArray();
+ int nSmaller = Arrays.stream(smallerBlockSize).reduce(1, (x, y) -> x * y);
+
+ int N = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y);
+
+ final Random rnd = new Random(7560);
+ final byte[] biggerData = new byte[nBigger];
+ rnd.nextBytes(biggerData);
+
+ final byte[] smallerData = new byte[nSmaller];
+ rnd.nextBytes(smallerData);
+
+ final float[] floatData = new float[N];
+
+ try (final N5Writer n5 = createTempN5Writer()) {
+
+ n5.createDataset(datasetName, dimensions, blockSize, dataType, compression);
+ final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName);
+
+ // write a block that is too large
+ final ByteArrayDataBlock bigDataBlock = new ByteArrayDataBlock(biggerBlockSize, new long[]{0, 0, 0}, biggerData);
+ n5.writeBlock(datasetName, attributes, bigDataBlock);
+
+ final DataBlock loadedBigDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0);
+ assertArrayEquals(biggerData, (byte[])loadedBigDataBlock.getData());
+
+ // write a block that is too small
+ final ByteArrayDataBlock smallDataBlock = new ByteArrayDataBlock(smallerBlockSize, new long[]{0, 0, 0}, smallerData);
+ n5.writeBlock(datasetName, attributes, smallDataBlock);
+
+ final DataBlock loadedSmallDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0);
+ assertArrayEquals(smallerData, (byte[])loadedSmallDataBlock.getData());
+
+ // write a block of the wrong type
+ final FloatArrayDataBlock floatDataBlock = new FloatArrayDataBlock(blockSize, new long[]{0, 0, 0}, floatData);
+ assertThrows(ClassCastException.class, () -> {
+ n5.writeBlock(datasetName, attributes, floatDataBlock);
+ });
+ }
+ }
+
@Test
public void testOverwriteBlock() {
+ final Compression compression = getCompressions()[0];
try (final N5Writer n5 = createTempN5Writer()) {
- n5.createDataset(datasetName, dimensions, blockSize, DataType.INT32, new GzipCompression());
+ n5.createDataset(datasetName, dimensions, blockSize, DataType.INT32, compression);
final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName);
final IntArrayDataBlock randomDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock);
@@ -472,16 +587,17 @@ public void testOverwriteBlock() {
final DataBlock loadedRandomDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0);
assertArrayEquals(intBlock, (int[])loadedRandomDataBlock.getData());
- // test the case where the resulting file becomes shorter
- final IntArrayDataBlock emptyDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, new int[DataBlock.getNumElements(blockSize)]);
+ // test the case where the resulting file becomes shorter (because the data compresses better)
+ final int[] emptyBlock = new int[DataBlock.getNumElements(blockSize)];
+ final IntArrayDataBlock emptyDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, emptyBlock);
n5.writeBlock(datasetName, attributes, emptyDataBlock);
final DataBlock loadedEmptyDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0);
- assertArrayEquals(new int[DataBlock.getNumElements(blockSize)], (int[])loadedEmptyDataBlock.getData());
+ assertArrayEquals(emptyBlock, (int[])loadedEmptyDataBlock.getData());
}
}
@Test
- public void testAttributeParsingPrimitive() {
+ public void testAttributeParsingPrimitive() {
try (final N5Writer n5 = createTempN5Writer()) {
@@ -557,7 +673,7 @@ public void testAttributeParsingPrimitive() {
}
@Test
- public void testAttributes() {
+ public void testAttributes() {
try (final N5Writer n5 = createTempN5Writer()) {
assertNull(n5.getAttribute(groupName, "test", String.class));
@@ -623,7 +739,6 @@ public void testAttributes() {
}
}
-
@Test
public void testNullAttributes() throws URISyntaxException, IOException {
@@ -847,7 +962,7 @@ public void testUri() throws IOException, URISyntaxException {
}
@Test
- public void testRemoveGroup() {
+ public void testRemoveGroup() {
try (final N5Writer n5 = createTempN5Writer()) {
n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression());
@@ -907,7 +1022,7 @@ public void testDeepList() throws ExecutionException, InterruptedException {
for (final String subGroup : subGroupNames)
assertTrue("deepList contents", Arrays.asList(n5.deepList("")).contains(groupName.replaceFirst("/", "") + "/" + subGroup));
- final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64, new RawCompression());
+ final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64);
final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize, new long[]{0, 0, 0}, new long[blockNumElements]);
n5.createDataset(datasetName, datasetAttributes);
n5.writeBlock(datasetName, datasetAttributes, dataBlock);
@@ -1009,7 +1124,7 @@ public void testDeepList() throws ExecutionException, InterruptedException {
}
@Test
- public void testExists() {
+ public void testExists() {
final String groupName2 = groupName + "-2";
final String datasetName2 = datasetName + "-2";
@@ -1030,7 +1145,7 @@ public void testExists() {
}
@Test
- public void testListAttributes() {
+ public void testListAttributes() {
try (N5Writer n5 = createTempN5Writer()) {
final String groupName2 = groupName + "-2";
@@ -1137,14 +1252,14 @@ public void testReaderCreation() throws IOException, URISyntaxException {
writer.setAttribute("/", N5Reader.VERSION_KEY, invalidVersion);
assertThrows("Incompatible version throws error", N5Exception.class, () -> {
try (final N5Reader ignored = createN5Reader(location)) {
- /*Only try with resource to ensure `close()` is called.*/
+ /*Only try with resource to ensure `close()` is called.*/
}
});
}
}
@Test
- public void testDelete() {
+ public void testDelete() {
try (N5Writer n5 = createTempN5Writer()) {
final String datasetName = AbstractN5Test.datasetName + "-test-delete";
@@ -1154,8 +1269,7 @@ public void testDelete() {
final long[] position2 = {0, 1, 2};
// no blocks should exist to begin with
- assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1)));
- assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2)));
+ assertNull(n5.readBlock(datasetName, attributes, position1));
final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, position1, byteBlock);
n5.writeBlock(datasetName, attributes, dataBlock);
@@ -1165,24 +1279,17 @@ public void testDelete() {
assertNotNull(readBlock);
assertTrue(readBlock instanceof ByteArrayDataBlock);
assertArrayEquals(byteBlock, ((ByteArrayDataBlock)readBlock).getData());
- assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2)));
- // deletion should report true in all cases
- assertTrue(n5.deleteBlock(datasetName, position1));
- assertTrue(n5.deleteBlock(datasetName, position1));
- assertTrue(n5.deleteBlock(datasetName, position2));
+ assertTrue("deleting existing block should return true", n5.deleteBlock(datasetName, position1));
+ assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position1));
+ assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position2));
// no block should exist anymore
- assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1)));
- assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2)));
+ assertNull(n5.readBlock(datasetName, attributes, position1));
+ assertNull(n5.readBlock(datasetName, attributes, position2));
}
}
- protected boolean testDeleteIsBlockDeleted(final DataBlock dataBlock) {
-
- return dataBlock == null;
- }
-
public static class TestData {
public String groupPath;
@@ -1287,7 +1394,7 @@ public void customObjectTest() {
}
@Test
- public void testAttributePaths() {
+ public void testAttributePaths() {
try (final N5Writer writer = createTempN5Writer()) {
@@ -1393,13 +1500,13 @@ public void testAttributePaths() {
* to try and grab the value as a json structure. I should grab the root, and match the empty string case */
assertEquals(writer.getAttribute(testGroup, "", JsonObject.class), writer.getAttribute(testGroup, "/", JsonObject.class));
- /* Lastly, ensure grabing nonsense returns null */
+ /* Lastly, ensure grabbing nonsense returns null */
assertNull(writer.getAttribute(testGroup, "/this/key/does/not/exist", Object.class));
}
}
@Test
- public void testAttributePathEscaping() {
+ public void testAttributePathEscaping() {
final JsonObject emptyObj = new JsonObject();
@@ -1485,8 +1592,7 @@ private String jsonKeyVal(final String key, final String val) {
}
@Test
- public void
- testRootLeaves() {
+ public void testRootLeaves() {
/* Test retrieving non-JsonObject root leaves */
try (final N5Writer n5 = createTempN5Writer()) {
@@ -1593,13 +1699,17 @@ public void testWriterSeparation() {
}
}
+ protected String[] illegalChars() {
+ return new String[]{" ", "#", "%"};
+ }
+
@Test
public void testPathsWithIllegalUriCharacters() throws IOException, URISyntaxException {
try (N5Writer writer = createTempN5Writer()) {
try (N5Reader reader = createN5Reader(writer.getURI().toString())) {
- final String[] illegalChars = {" ", "#", "%"};
+ final String[] illegalChars = illegalChars();
for (final String illegalChar : illegalChars) {
final String groupWithIllegalChar = "test" + illegalChar + "group";
assertThrows("list over group should throw prior to create", N5Exception.N5IOException.class, () -> writer.list(groupWithIllegalChar));
@@ -1625,10 +1735,40 @@ public void testPathsWithIllegalUriCharacters() throws IOException, URISyntaxExc
}
}
+ public static void assertBlockEquals(final DataBlock expected, final DataBlock actual) {
+ assertEquals("Datablocks are different type", expected.getClass(), actual.getClass());
+
+ Assert.assertArrayEquals("read block position should be same as block position when unsharded", expected.getGridPosition(), actual.getGridPosition());
+ Assert.assertArrayEquals("read block size should equal block size when unsharded", expected.getSize(), actual.getSize());
+
+ final Object expectedData = expected.getData();
+ final Object actualData = actual.getData();
+
+ final String dataEqualsMsg = "block written through shard should be identical";
+ if (expectedData instanceof byte[])
+ assertArrayEquals(dataEqualsMsg, (byte[])expectedData, (byte[])expectedData);
+ else if (expectedData instanceof short[])
+ assertArrayEquals(dataEqualsMsg, (short[])expectedData, (short[])actualData);
+ else if (expectedData instanceof int[])
+ assertArrayEquals(dataEqualsMsg, (int[])expectedData, (int[])actualData);
+ else if (expectedData instanceof long[])
+ assertArrayEquals(dataEqualsMsg, (long[])expectedData, (long[])actualData);
+ else if (expectedData instanceof float[])
+ assertArrayEquals(dataEqualsMsg, (float[])expectedData, (float[])actualData, 0f);
+ else if (expectedData instanceof double[])
+ assertArrayEquals(dataEqualsMsg, (double[])expectedData, (double[])actualData, 0d);
+ else if (expectedData instanceof String[])
+ assertArrayEquals(dataEqualsMsg, (String[])expectedData, (String[])actualData);
+ else
+ fail("Unsupported data type for block data: " + expectedData.getClass());
+ }
+
protected void assertDatasetAttributesEquals(final DatasetAttributes expected, final DatasetAttributes actual) {
assertArrayEquals(expected.getDimensions(), actual.getDimensions());
assertArrayEquals(expected.getBlockSize(), actual.getBlockSize());
assertEquals(expected.getDataType(), actual.getDataType());
- assertEquals(expected.getCompression(), actual.getCompression());
+
+ // TODO would be nice to check this somehow maybe make a DatasetAttributes.equals method?
+// assertArrayEquals(expected.getDataCodecInfos(), actual.getDataCodecInfos());
}
}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java
new file mode 100644
index 000000000..63d3d7e5d
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java
@@ -0,0 +1,154 @@
+/*-
+ * #%L
+ * Not HDF5
+ * %%
+ * Copyright (C) 2017 - 2025 Stephan Saalfeld
+ * %%
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * #L%
+ */
+package org.janelia.saalfeldlab.n5;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo;
+import org.janelia.saalfeldlab.n5.shard.DefaultShardCodecInfo;
+import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;
+import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation;
+import org.junit.Test;
+
+/**
+ * Unit tests for DatasetAttributes class.
+ */
+public class DatasetAttributesTest {
+
+ /**
+ * Test that validateBlockShardSizes method accepts valid shard and block size combinations.
+ */
+ @Test
+ public void testValidateBlockShardSizesValid() {
+
+ // Test case 1: shard size equals block size
+ long[] dimensions = new long[]{100, 200, 300};
+ int[] shardSize = new int[]{64, 64, 64};
+ int[] blockSize = new int[]{64, 64, 64};
+ DataType dataType = DataType.UINT8;
+
+ // This should not throw any exception
+ DatasetAttributes attrs = shardDatasetAttributes(dimensions, shardSize, blockSize, dataType);
+ assertEquals(blockSize, attrs.getBlockSize());
+ NestedGrid grid = attrs.getNestedBlockGrid();
+ assertEquals(blockSize, grid.getBlockSize(0));
+ assertEquals(shardSize, grid.getBlockSize(1));
+
+ // Test case 2: shard size is a multiple of block size
+ shardSize = new int[]{128};
+ blockSize = new int[]{64};
+ attrs = shardDatasetAttributes(new long[]{128}, shardSize, blockSize, dataType);
+ assertEquals(blockSize, attrs.getBlockSize());
+ grid = attrs.getNestedBlockGrid();
+ assertEquals(blockSize, grid.getBlockSize(0));
+ assertEquals(shardSize, grid.getBlockSize(1));
+
+ // Test case 3: different multiples per dimension
+ shardSize = new int[]{128, 256, 32, 2};
+ blockSize = new int[]{32, 64, 32, 1};
+ attrs = shardDatasetAttributes(new long[]{128, 128, 128, 128}, shardSize, blockSize, dataType );
+ assertEquals(blockSize, attrs.getBlockSize());
+ grid = attrs.getNestedBlockGrid();
+ assertEquals(blockSize, grid.getBlockSize(0));
+ assertEquals(shardSize, grid.getBlockSize(1));
+
+ // Test case 4: large multiples
+ shardSize = new int[]{1024, 2048, 512};
+ blockSize = new int[]{32, 64, 16};
+ attrs = shardDatasetAttributes(dimensions, shardSize, blockSize, dataType);
+ assertEquals(blockSize, attrs.getBlockSize());
+ grid = attrs.getNestedBlockGrid();
+ assertEquals(blockSize, grid.getBlockSize(0));
+ assertEquals(shardSize, grid.getBlockSize(1));
+ }
+
+ private static DatasetAttributes shardDatasetAttributes(
+ long[] dimensions, int[] shardSize, int[] blockSize, DataType dataType) {
+
+ DefaultShardCodecInfo blockCodecInfo = new DefaultShardCodecInfo(
+ blockSize,
+ new N5BlockCodecInfo(),
+ new DataCodecInfo[]{new RawCompression()},
+ new RawBlockCodecInfo(),
+ new DataCodecInfo[]{new RawCompression()},
+ IndexLocation.END);
+
+ return new DatasetAttributes(dimensions, shardSize, dataType, blockCodecInfo);
+ }
+
+ /**
+ * Test that validateBlockShardSizes method rejects invalid shard and block size combinations.
+ */
+ @Test
+ public void testValidateBlockShardSizesInvalid() {
+
+ final long[] dimensions = new long[]{100, 200, 300};
+ final DataType dataType = DataType.UINT8;
+
+ // Block size too small
+ IllegalArgumentException ex0 = assertThrows(
+ IllegalArgumentException.class,
+ () -> shardDatasetAttributes(dimensions, new int[]{1, 1, 1}, new int[]{1, 0, -1}, dataType));
+ assertTrue(ex0.getMessage().contains("negative"));
+
+ // Different number of dimensions
+ IllegalArgumentException ex1 = assertThrows(
+ IllegalArgumentException.class,
+ () -> shardDatasetAttributes(dimensions, new int[]{64, 64}, new int[]{32, 32, 32}, dataType));
+ assertTrue(ex1.getMessage().contains("different length"));
+
+ // Shard size smaller than block size
+ IllegalArgumentException ex2 = assertThrows(
+ IllegalArgumentException.class,
+ () -> shardDatasetAttributes(dimensions, new int[]{32, 64, 64}, new int[]{64, 64, 64}, dataType));
+ assertTrue(ex2.getMessage().contains("is smaller than previous"));
+
+ // Shard size not a multiple of block size
+ IllegalArgumentException ex3 = assertThrows(
+ IllegalArgumentException.class,
+ () -> shardDatasetAttributes(dimensions, new int[]{100, 100, 100}, new int[]{64, 64, 64}, dataType));
+ assertTrue(ex3.getMessage().contains("not a multiple of previous level"));
+
+ // Multiple violations - shard smaller than block in one dimension
+ IllegalArgumentException ex4 = assertThrows(
+ IllegalArgumentException.class,
+ () -> shardDatasetAttributes(dimensions, new int[]{128, 32, 128}, new int[]{64, 64, 64}, dataType));
+ assertTrue(ex4.getMessage().contains("is smaller than previous"));
+
+ // Edge case - shard size of 0
+ assertThrows(
+ IllegalArgumentException.class,
+ () -> shardDatasetAttributes(dimensions, new int[]{0, 64, 64}, new int[]{64, 64, 64}, dataType));
+ }
+
+}
\ No newline at end of file
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java
index 5f069f7b2..546a4072f 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java
@@ -26,31 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- * Copyright (c) 2017--2021, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
package org.janelia.saalfeldlab.n5;
import static org.junit.Assert.fail;
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java
index 262d2ee18..070516be5 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java
@@ -136,8 +136,8 @@ public void cacheGroupDatasetTest() throws IOException, URISyntaxException {
final String groupName = "gg";
final String tmpLocation = tempN5Location();
- try (N5KeyValueWriter w1 = (N5KeyValueWriter) createN5Writer(tmpLocation);
- N5KeyValueWriter w2 = (N5KeyValueWriter) createN5Writer(tmpLocation);) {
+ try (GsonKeyValueN5Writer w1 = (GsonKeyValueN5Writer) createN5Writer(tmpLocation);
+ GsonKeyValueN5Writer w2 = (GsonKeyValueN5Writer) createN5Writer(tmpLocation);) {
// create a group, both writers know it exists
w1.createGroup(groupName);
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java
new file mode 100644
index 000000000..4cdc1a272
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java
@@ -0,0 +1,141 @@
+package org.janelia.saalfeldlab.n5.backward;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.file.Files;
+import java.util.Arrays;
+
+import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader;
+import org.janelia.saalfeldlab.n5.KeyValueAccess;
+import org.janelia.saalfeldlab.n5.LockedChannel;
+import org.janelia.saalfeldlab.n5.N5FSReader;
+import org.janelia.saalfeldlab.n5.N5FSWriter;
+import org.janelia.saalfeldlab.n5.RawCompression;
+import org.junit.Test;
+
+import com.google.gson.JsonElement;
+
+public class CompatibilityTest {
+
+ String[][] readVersionsDataset = {
+ {"data-1.5.0.n5", "raw"},
+ {"data-2.5.1.n5", "raw"},
+ {"data-3.1.3.n5", "raw"} };
+
+ String writeVersion = "data-3.1.3.n5";
+ String writeDataset = "raw";
+ String[] writePathsToTest = {"0/0", "0/1", "1/0", "1/1"};
+
+ @Test
+ public void testBackwardReads() throws NumberFormatException, IOException {
+
+ for (String[] versionDset : readVersionsDataset)
+ backwardReadHelper(versionDset[0], versionDset[1]);
+ }
+
+ public void backwardReadHelper(final String base, final String dsetPath) throws NumberFormatException, IOException {
+
+ final N5FSReader n5 = new N5FSReader("src/test/resources/backward/" + base);
+ assertTrue(n5.datasetExists(dsetPath));
+ final DatasetAttributes attrs = n5.getDatasetAttributes(dsetPath);
+
+ // equivalent to the assertTrue above, but be extra sure
+ assertNotNull(attrs);
+
+ byte value = 0;
+ long[] p = new long[2];
+
+ DataBlock b00 = n5.readBlock(dsetPath, attrs, p);
+ assertNotNull(b00);
+ assertArrayEquals(new int[]{5,4}, b00.getSize());
+ assertArrayEquals(expectedData(20, value), b00.getData());
+
+ p[0] = 1;
+ p[1] = 0;
+ value++;
+ DataBlock b10 = n5.readBlock(dsetPath, attrs, p);
+ assertNotNull(b10);
+ assertArrayEquals(new int[]{2,4}, b10.getSize());
+ assertArrayEquals(expectedData(8, value), b10.getData());
+
+ p[0] = 0;
+ p[1] = 1;
+ value++;
+ DataBlock b01 = n5.readBlock(dsetPath, attrs, p);
+ assertNotNull(b01);
+ assertArrayEquals(new int[]{5,1}, b01.getSize());
+ assertArrayEquals(expectedData(5, value), b01.getData());
+
+ p[0] = 1;
+ p[1] = 1;
+ value++;
+ DataBlock b11 = n5.readBlock(dsetPath, attrs, p);
+ assertNotNull(b11);
+ assertArrayEquals(new int[]{2,1}, b11.getSize());
+ assertArrayEquals(expectedData(2, value), b11.getData());
+
+ n5.close();
+ }
+
+ @Test
+ public void testBlockData() throws IOException {
+
+ final N5FSReader n5Legacy = new N5FSReader("src/test/resources/backward/" + writeVersion);
+ final URI uriLegacy = n5Legacy.getURI();
+
+ final File basePath = Files.createTempDirectory("n5-blockDataTest-").toFile();
+ basePath.delete();
+ basePath.mkdir();
+ basePath.deleteOnExit();
+
+ N5FSWriter n5My = CreateSampleData.createSampleData(
+ basePath.getCanonicalPath(), writeDataset, new RawCompression());
+ URI uriMy = n5My.getURI();
+
+ // check attributes
+ final JsonElement attrsLegacy = ((GsonKeyValueN5Reader)n5Legacy).getAttributes(writeDataset);
+ final JsonElement attrsMy = ((GsonKeyValueN5Reader)n5My).getAttributes(writeDataset);
+ assertEquals(attrsLegacy, attrsMy);
+
+ final KeyValueAccess kva = n5My.getKeyValueAccess();
+ for (final String path : writePathsToTest) {
+ final byte[] dataMy = read(kva, kva.compose(uriMy, writeDataset, path));
+ final byte[] dataLegacy = read(kva, kva.compose(uriLegacy, writeDataset, path));
+ assertArrayEquals(dataLegacy, dataMy);
+ }
+
+ n5My.remove();
+ n5My.close();
+ n5Legacy.close();
+ }
+
+ private byte[] read(KeyValueAccess kva, String path) {
+
+ int N = (int)kva.size(path);
+ byte[] data = new byte[N];
+ try (LockedChannel ch = kva.lockForReading(path);
+ InputStream is = ch.newInputStream();) {
+
+ is.read(data);
+ } catch (IOException e) {
+ return null;
+ }
+ return data;
+ }
+
+ private static byte[] expectedData(int size, byte value ) {
+ byte[] data = new byte[size];
+ Arrays.fill(data, value);
+ return data;
+ }
+
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java
new file mode 100644
index 000000000..73334cdcd
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java
@@ -0,0 +1,69 @@
+package org.janelia.saalfeldlab.n5.backward;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.janelia.saalfeldlab.n5.ByteArrayDataBlock;
+import org.janelia.saalfeldlab.n5.Compression;
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.N5FSWriter;
+import org.janelia.saalfeldlab.n5.RawCompression;
+
+public class CreateSampleData {
+
+ public static void main(String[] args) throws IOException {
+
+ File f = new File("src/test/resources/data-4.0.0-alpha-X.n5");
+ System.out.println(f.getCanonicalPath());
+ createSampleData(f.getCanonicalPath(), "raw", new RawCompression());
+ }
+
+ public static N5FSWriter createSampleData(String baseDir, String dataset, Compression compression) throws IOException {
+
+ N5FSWriter n5 = new N5FSWriter(baseDir);
+ final String dsetPath = compression.getType();
+
+ long[] dimensions = new long[]{7, 5};
+ int[] blkSizeDset = new int[]{5, 4};
+ int[] blkSize = new int[]{5, 4};
+
+ final DatasetAttributes attrs = new DatasetAttributes(dimensions, blkSizeDset, DataType.UINT8, compression);
+ n5.createDataset(dsetPath, attrs);
+
+ byte val = 0;
+ long[] pos = new long[]{0, 0};
+ n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val));
+
+ pos[0] = 1;
+ pos[1] = 0;
+ blkSize[0] = 2;
+ blkSize[1] = 4;
+ val++;
+ n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val));
+
+ pos[0] = 0;
+ pos[1] = 1;
+ blkSize[0] = 5;
+ blkSize[1] = 1;
+ val++;
+ n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val));
+
+ pos[0] = 1;
+ pos[1] = 1;
+ blkSize[0] = 2;
+ blkSize[1] = 1;
+ val++;
+ n5.writeBlock(dsetPath, attrs, createDataBlock( blkSize, pos, val ));
+
+ return n5;
+ }
+
+ public static ByteArrayDataBlock createDataBlock(int[] size, long[] gridPosition, byte value) throws IOException {
+ int N = Arrays.stream(size).reduce(1, (x,y) -> x*y);
+ final byte[] data = new byte[N];
+ Arrays.fill(data, value);
+ return new ByteArrayDataBlock(size, gridPosition, data);
+ }
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java
new file mode 100644
index 000000000..199563b98
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java
@@ -0,0 +1,136 @@
+package org.janelia.saalfeldlab.n5.benchmarks;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess;
+import org.janelia.saalfeldlab.n5.KeyValueAccess;
+import org.janelia.saalfeldlab.n5.LockedChannel;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@State(Scope.Benchmark)
+@Warmup(iterations = 10, time = 100, timeUnit = TimeUnit.MICROSECONDS)
+@Measurement(iterations = 100, time = 100, timeUnit = TimeUnit.MICROSECONDS)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Fork(1)
+public class ReadDataBenchmarks {
+
+ @Param(value = { "10000000" })
+ protected int objectSizeBytes;
+
+ protected Path basePath;
+ protected ArrayList tmpPaths;
+ protected KeyValueAccess kva;
+ protected Random random;
+
+ public ReadDataBenchmarks() {}
+
+ public static void main(String... args) throws RunnerException {
+
+ final Options options = new OptionsBuilder().include(ReadDataBenchmarks.class.getSimpleName() + "\\.")
+ .build();
+
+ new Runner(options).run();
+ }
+
+ @Benchmark
+ public void run(Blackhole hole) throws IOException {
+
+ hole.consume(read().materialize());
+ }
+
+ public ReadData read() throws IOException {
+
+ return kva.createReadData(getPath().toString());
+ }
+
+ protected Path getPath() {
+
+ return basePath.resolve("tmp-" + objectSizeBytes);
+ }
+
+ @Setup(Level.Trial)
+ public void setup() throws IOException {
+
+ random = new Random();
+ kva = new FileSystemKeyValueAccess(FileSystems.getDefault());
+
+ basePath = Files.createTempDirectory("ReadDataBenchmark-");
+ tmpPaths = new ArrayList<>();
+ for (final int sz : sizes()) {
+ Path p = basePath.resolve("tmp-"+sz);
+ write(p, sz);
+ tmpPaths.add(p);
+ }
+ }
+
+ protected void write(Path path, int numBytes) {
+
+ final byte[] data = new byte[numBytes];
+ random.nextBytes(data);
+
+ System.out.println(path.toAbsolutePath().toString());
+ System.out.println(numBytes);
+ try (final LockedChannel ch = kva.lockForWriting(path.toAbsolutePath().toString())) {
+ final OutputStream os = ch.newOutputStream();
+ os.write(data);
+ os.flush();
+ os.close();
+ } catch (final IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @TearDown(Level.Trial)
+ public void teardown() {
+
+ for ( Path p : tmpPaths ) {
+ p.toFile().delete();
+ }
+ basePath.toFile().delete();
+ }
+
+ public int[] sizes() {
+
+ try {
+ final Param ann = ReadDataBenchmarks.class.getDeclaredField("objectSizeBytes").getAnnotation(Param.class);
+ System.out.println(Arrays.toString(ann.value()));
+ return Arrays.stream(ann.value()).mapToInt(Integer::parseInt).toArray();
+
+ } catch (final NoSuchFieldException e) {
+ e.printStackTrace();
+ } catch (final SecurityException e) {
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java
new file mode 100644
index 000000000..da0bf9958
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java
@@ -0,0 +1,43 @@
+package org.janelia.saalfeldlab.n5.benchmarks;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess;
+
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@State(Scope.Benchmark)
+@Warmup(iterations = 10, time = 100, timeUnit = TimeUnit.MICROSECONDS)
+@Measurement(iterations = 100, time = 100, timeUnit = TimeUnit.MICROSECONDS)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Fork(1)
+public class ReadDataBenchmarksKvaReadData extends ReadDataBenchmarks {
+
+ public static void main(String... args) throws RunnerException {
+
+ final Options options = new OptionsBuilder().include(ReadDataBenchmarksKvaReadData.class.getSimpleName() + "\\.")
+ .build();
+
+ new Runner(options).run();
+ }
+
+ public ReadData read() throws IOException {
+
+ return ((FileSystemKeyValueAccess)kva).createReadData(getPath().toString());
+ }
+
+}
\ No newline at end of file
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java
index ed6259f2d..d3f996ede 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java
@@ -209,7 +209,7 @@ public void testChildManagement() {
// Test addChildIfPresent on cached parent without children list
cache.exists("parent2", null);
- children = cache.list("parent2"); // initialize children array
+ children = cache.list("parent2"); // create children array
cache.addChildIfPresent("parent2", "child");
children = cache.list("parent2");
assertTrue(Arrays.asList(children).contains("child"));
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java
new file mode 100644
index 000000000..3ff34059f
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java
@@ -0,0 +1,291 @@
+package org.janelia.saalfeldlab.n5.codec;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.ByteOrder;
+import java.util.Arrays;
+import java.util.Random;
+
+import org.janelia.saalfeldlab.n5.ByteArrayDataBlock;
+import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock;
+import org.janelia.saalfeldlab.n5.FloatArrayDataBlock;
+import org.janelia.saalfeldlab.n5.GzipCompression;
+import org.janelia.saalfeldlab.n5.IntArrayDataBlock;
+import org.janelia.saalfeldlab.n5.LongArrayDataBlock;
+import org.janelia.saalfeldlab.n5.RawCompression;
+import org.janelia.saalfeldlab.n5.ShortArrayDataBlock;
+import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec;
+import org.janelia.saalfeldlab.n5.shard.DatasetAccess;
+import org.janelia.saalfeldlab.n5.shard.PositionValueAccess;
+import org.janelia.saalfeldlab.n5.shard.RawShardTest;
+import org.janelia.saalfeldlab.n5.shard.TestPositionValueAccess;
+import org.junit.Test;
+
+public class BlockCodecTests {
+
+ static Random random = new Random(12345);
+
+ final int[] blockSize = {11, 7, 5};
+ private final BitShiftBytesCodec shiftCodec = new BitShiftBytesCodec(3);
+ private final GzipCompression compressor = new GzipCompression();
+ private final DataCodecInfo[][] dataCodecInfos = new DataCodecInfo[][]{
+ {}, // empty: "raw" compression
+ {compressor},
+ {shiftCodec},
+ {shiftCodec, compressor}
+ };
+
+ private final DataType[] dataTypes = {
+ DataType.INT8, DataType.UINT8,
+ DataType.INT16, DataType.UINT16,
+ DataType.INT32, DataType.UINT32,
+ DataType.INT64, DataType.UINT64,
+ DataType.FLOAT32, DataType.FLOAT64
+ };
+
+ @Test
+ public void testN5BlockCodec() throws Exception {
+ for (DataType dataType : dataTypes) {
+ for (DataCodecInfo[] dataCodecInfo : dataCodecInfos) {
+
+ final DatasetAttributes attributes = new DatasetAttributes(
+ new long[]{32, 32, 32},
+ blockSize,
+ dataType,
+ new N5BlockCodecInfo(),
+ dataCodecInfo);
+
+ testBlockCodecHelper(attributes);
+ }
+ }
+ }
+
+ @Test
+ public void testRawBytesBlockCodec() throws Exception {
+ // Test RawBlockCodecInfo codec with different byte orders and DataTypes
+ final ByteOrder[] byteOrders = {ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN};
+ for (DataType dataType : dataTypes) {
+ for (ByteOrder byteOrder : byteOrders) {
+ for (DataCodecInfo[] codecs : dataCodecInfos) {
+
+ final RawBlockCodecInfo codec = new RawBlockCodecInfo(byteOrder);
+ final DatasetAttributes attributes = new DatasetAttributes(
+ new long[]{32, 32, 32},
+ blockSize,
+ dataType,
+ codec,
+ codecs);
+
+ testBlockCodecHelper(attributes);
+ }
+ }
+ }
+ }
+
+ private void testBlockCodecHelper(DatasetAttributes attributes) throws Exception {
+
+ // TODO
+// final int[] blockSize = attributes.getBlockSize();
+// final DataType dataType = attributes.getDataType();
+// final long[] gridPosition = {3, 2, 1};
+//
+// // Create appropriate data block based on type
+// DataBlock originalBlock = ((DataBlock)createRandomDataBlock(dataType, blockSize, gridPosition));
+// final BlockCodec codec = attributes.getBlockCodec();
+//
+// // Test encode/decode roundtrip
+// final ReadData encoded = codec.encode(originalBlock);
+// assertNotNull(encoded);
+//
+// final DataBlock decoded = codec.decode(encoded, gridPosition);
+// assertNotNull(decoded);
+//
+// assertArrayEquals("Block size should match", blockSize, decoded.getSize());
+// assertArrayEquals("Grid position should match", gridPosition, decoded.getGridPosition());
+// assertDataEquals(originalBlock, decoded);
+// verifyCompatibleDataType(dataType, decoded);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testEmptyBlock() throws Exception {
+ // Test handling of empty blocks
+ final int[] blockSize = {0, 0};
+ final long[] gridPosition = {0, 0};
+ final N5BlockCodecInfo blockCodecInfo = new N5BlockCodecInfo();
+ final RawShardTest.TestDatasetAttributes attributes = new RawShardTest.TestDatasetAttributes(
+ new long[]{64, 64},
+ new int[]{8, 8},
+ DataType.UINT8,
+ blockCodecInfo,
+ new RawCompression());
+
+ final PositionValueAccess store = new TestPositionValueAccess();
+ DatasetAccess access = attributes.datasetAccess();
+
+ // Test encode/decode
+ final ByteArrayDataBlock emptyBlock = new ByteArrayDataBlock(blockSize, gridPosition, new byte[0]);
+
+ access.writeBlock(store, emptyBlock);
+ final DataBlock decoded = access.readBlock(store, gridPosition);
+
+ assertEquals("Empty block should have 0 elements", 0, decoded.getNumElements());
+ }
+
+ @Test
+ public void testEncodedSizeCalculation() throws Exception {
+
+ // TODO
+
+ // Test that encoded size calculations are correct
+// final int[] blockSize = {64, 64};
+// final DatasetAttributes n5ArrayAttrs = new DatasetAttributes(
+// new long[]{512, 512},
+// blockSize,
+// blockSize,
+// DataType.INT16,
+// new N5BlockCodecInfo());
+//
+//
+// final DatasetAttributes rawArrayAttrs = new DatasetAttributes(
+// new long[]{512, 512},
+// blockSize,
+// blockSize,
+// DataType.INT16,
+// new RawBlockCodecInfo());
+//
+// // Calculate expected sizes
+// final long rawDataSize = blockSize[0] * blockSize[1] * 2; // INT16 has 2 bytes per element
+//
+// // N5BlockCodecInfo adds a header
+// // the estimate of the encoded size
+// final long n5EncodedSize = n5ArrayAttrs.getBlockCodecInfo().encodedSize(rawDataSize);
+// assertTrue("N5 encoded size should be larger than raw size", n5EncodedSize > rawDataSize);
+//
+// DataBlock dataBlock = ((DataBlock)createRandomDataBlock(n5ArrayAttrs.getDataType(), blockSize, new long[]{0, 0}));
+// ReadData n5EncodedDataBlock = n5ArrayAttrs.getBlockCodec().encode(dataBlock);
+// assertEquals("N5 actual encoded size should equal estimated size", n5EncodedSize, n5EncodedDataBlock.length());
+//
+// // RawBlockCodecInfo should not change size
+// final long rawEncodedSize = rawArrayAttrs.getBlockCodecInfo().encodedSize(rawDataSize);
+// assertEquals("Raw encoded size should equal input size", rawDataSize, rawEncodedSize);
+//
+// ReadData rawEncodedDataBlock = rawArrayAttrs.getBlockCodec().encode(dataBlock);
+// assertEquals("Raw actual encoded size should equal estimated size", rawEncodedSize, rawEncodedDataBlock.length());
+ }
+
+ private static DataBlock createRandomDataBlock(DataType dataType, int[] blockSize, long[] gridPosition) {
+ final int numElements = Arrays.stream(blockSize).reduce(1, (a, b) -> a * b);
+
+ switch (dataType) {
+ case INT8:
+ case UINT8:
+ byte[] uint8Data = new byte[numElements];
+ for (int i = 0; i < numElements; i++) {
+ uint8Data[i] = (byte) random.nextInt(256);
+ }
+ return new ByteArrayDataBlock(blockSize, gridPosition, uint8Data);
+
+ case INT16:
+ case UINT16:
+ short[] uint16Data = new short[numElements];
+ for (int i = 0; i < numElements; i++) {
+ uint16Data[i] = (short) random.nextInt(65536);
+ }
+ return new ShortArrayDataBlock(blockSize, gridPosition, uint16Data);
+
+ case INT32:
+ case UINT32:
+ int[] uint32Data = new int[numElements];
+ for (int i = 0; i < numElements; i++) {
+ uint32Data[i] = random.nextInt();
+ }
+ return new IntArrayDataBlock(blockSize, gridPosition, uint32Data);
+
+ case INT64:
+ case UINT64:
+ long[] uint64Data = new long[numElements];
+ for (int i = 0; i < numElements; i++) {
+ uint64Data[i] = random.nextLong();
+ }
+ return new LongArrayDataBlock(blockSize, gridPosition, uint64Data);
+
+ case FLOAT32:
+ float[] floatData = new float[numElements];
+ for (int i = 0; i < numElements; i++) {
+ floatData[i] = random.nextFloat();
+ }
+ return new FloatArrayDataBlock(blockSize, gridPosition, floatData);
+
+ case FLOAT64:
+ double[] doubleData = new double[numElements];
+ for (int i = 0; i < numElements; i++) {
+ doubleData[i] = random.nextDouble();
+ }
+ return new DoubleArrayDataBlock(blockSize, gridPosition, doubleData);
+
+ default:
+ throw new IllegalArgumentException("Unsupported data type: " + dataType);
+ }
+ }
+
+ private static void verifyCompatibleDataType(DataType expectedType, DataBlock block) {
+
+ Object data = block.getData();
+ switch (expectedType) {
+ case INT8:
+ case UINT8:
+ assertTrue("Expected byte array for " + expectedType, data instanceof byte[]);
+ break;
+ case INT16:
+ case UINT16:
+ assertTrue("Expected short array for " + expectedType, data instanceof short[]);
+ break;
+ case INT32:
+ case UINT32:
+ assertTrue("Expected int array for " + expectedType, data instanceof int[]);
+ break;
+ case INT64:
+ case UINT64:
+ assertTrue("Expected long array for " + expectedType, data instanceof long[]);
+ break;
+ case FLOAT32:
+ assertTrue("Expected float array for " + expectedType, data instanceof float[]);
+ break;
+ case FLOAT64:
+ assertTrue("Expected double array for " + expectedType, data instanceof double[]);
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported data type: " + expectedType);
+ }
+ }
+
+ private static void assertDataEquals(DataBlock expected, DataBlock actual) {
+
+ Object expectedData = expected.getData();
+ Object actualData = actual.getData();
+
+ if (expectedData instanceof byte[]) {
+ assertArrayEquals((byte[]) expectedData, (byte[]) actualData);
+ } else if (expectedData instanceof short[]) {
+ assertArrayEquals((short[]) expectedData, (short[]) actualData);
+ } else if (expectedData instanceof int[]) {
+ assertArrayEquals((int[]) expectedData, (int[]) actualData);
+ } else if (expectedData instanceof long[]) {
+ assertArrayEquals((long[]) expectedData, (long[]) actualData);
+ } else if (expectedData instanceof float[]) {
+ assertArrayEquals((float[]) expectedData, (float[]) actualData, 0.0f);
+ } else if (expectedData instanceof double[]) {
+ assertArrayEquals((double[]) expectedData, (double[]) actualData, 0.0);
+ } else {
+ throw new IllegalArgumentException("Unknown data type");
+ }
+ }
+
+
+}
\ No newline at end of file
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java
new file mode 100644
index 000000000..90692c710
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java
@@ -0,0 +1,208 @@
+package org.janelia.saalfeldlab.n5.codec;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Random;
+import java.util.function.IntUnaryOperator;
+
+import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.janelia.saalfeldlab.n5.readdata.ReadData.OutputStreamOperator;
+import org.janelia.saalfeldlab.n5.serialization.NameConfig;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class BytesCodecTests {
+
+ static Random random;
+
+ @BeforeClass
+ public static void setup() {
+ random = new Random(7777);
+ }
+
+ @Test
+ public void testEncodeDecodeBytes() {
+
+ // Create a BitShiftBytesCodec with shift value
+ final BitShiftBytesCodec originalCodec = new BitShiftBytesCodec(3);
+
+ // Test encode/decode roundtrip
+ final byte[] testData = new byte[12];
+ random.nextBytes(testData);
+
+ final ReadData original = ReadData.from(testData);
+ final ReadData encoded = originalCodec.encode(original);
+ final ReadData decoded = originalCodec.decode(encoded);
+
+ final byte[] result = decoded.allBytes();
+ assertEquals("Length should match", testData.length, result.length);
+ assertArrayEquals("encoded-decoded bytes should match original", testData, result);
+ }
+
+ @Test
+ public void concatenatedBytesCodecTest() throws IOException {
+
+ int N = 16;
+ ReadData data = ReadData.from( new InputStream() {
+ @Override
+ public int read() throws IOException {
+ return Math.abs(random.nextInt()) % 32;
+ }
+ }, N ).materialize();
+
+ final byte[] bytes = data.allBytes();
+ final byte[] expected = new byte[bytes.length];
+ for (int i = 0; i < bytes.length; i++) {
+ expected[i] = (byte)(2 * bytes[i] + 3);
+ }
+
+ final DataCodec a = new ByteFunctionCodec(x -> 2 * x, x -> x / 2);
+ final DataCodec b = new ByteFunctionCodec(x -> x + 3, x -> x - 3 );
+ final ConcatenatedDataCodec ab = new ConcatenatedDataCodec(new DataCodec[]{a, b});
+
+ final ReadData encodedData = ab.encode(data).materialize();
+ assertArrayEquals(expected, encodedData.allBytes());
+
+ final ReadData decodedData = ab.decode(encodedData).materialize();
+ assertArrayEquals(bytes, decodedData.allBytes());
+ }
+
+ public static class ByteFunctionCodec implements DataCodec, DataCodecInfo {
+
+ IntUnaryOperator encoder;
+ IntUnaryOperator decoder;
+
+ public ByteFunctionCodec( IntUnaryOperator encoder, IntUnaryOperator decoder ) {
+ this.encoder = encoder;
+ this.decoder = decoder;
+ }
+
+ @Override
+ public String getType() {
+ return "byteFunction";
+ }
+
+ public ReadData decode(ReadData data) {
+ return data.encode(new ByteFun(decoder));
+ }
+
+ public ReadData encode(ReadData data) {
+ return data.encode(new ByteFun(encoder));
+ }
+
+ @Override public DataCodec create() {
+
+ return this;
+ }
+ }
+
+ private static class ByteFun implements OutputStreamOperator {
+
+ IntUnaryOperator fun;
+ public ByteFun(IntUnaryOperator fun) {
+ this.fun = fun;
+ }
+
+ @Override
+ public OutputStream apply(OutputStream o) {
+ return new OutputStream() {
+ @Override
+ public void write(int b) throws IOException {
+ o.write(fun.applyAsInt(b));
+ }
+ };
+ }
+ }
+
+ @NameConfig.Name(BitShiftBytesCodec.TYPE)
+ public static class BitShiftBytesCodec implements DataCodec, DataCodecInfo {
+ @Override public DataCodec create() {
+
+ return this;
+ }
+
+ private static final String TYPE = "bitshift";
+
+ @NameConfig.Parameter
+ private int shift;
+
+ public BitShiftBytesCodec() {
+
+ shift = 0;
+ }
+
+ public BitShiftBytesCodec(int shift) {
+
+ this.shift = shift;
+ }
+
+ @Override
+ public String getType() {
+
+ return TYPE;
+ }
+
+ @Override
+ public ReadData decode(ReadData readData) throws N5IOException {
+
+ if (shift == 0) {
+ return readData;
+ }
+
+ final byte[] data = readData.allBytes();
+ final byte[] decoded = new byte[data.length];
+
+ // Apply inverse bit shift (right rotate) to decode
+ for (int i = 0; i < data.length; i++) {
+ int b = data[i] & 0xFF;
+ decoded[i] = (byte)((b >>> shift) | (b << (8 - shift)));
+ }
+
+ return ReadData.from(decoded);
+ }
+
+ @Override
+ public ReadData encode(ReadData readData) throws N5IOException {
+
+ if (shift == 0) {
+ return readData;
+ }
+
+ byte[] data = readData.allBytes();
+ byte[] encoded = new byte[data.length];
+
+ // Apply bit shift (left rotate) to encode
+ for (int i = 0; i < data.length; i++) {
+ int b = data[i] & 0xFF;
+ encoded[i] = (byte)((b << shift) | (b >>> (8 - shift)));
+ }
+ return ReadData.from(encoded);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ BitShiftBytesCodec other = (BitShiftBytesCodec)obj;
+ return shift == other.shift;
+ }
+
+ @Override
+ public int hashCode() {
+
+ return Integer.hashCode(shift);
+ }
+
+ }
+
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/compression/CompressionTypesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/compression/CompressionTypesTest.java
index 25ecea36f..af467c856 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/compression/CompressionTypesTest.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/compression/CompressionTypesTest.java
@@ -26,9 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- *
- */
package org.janelia.saalfeldlab.n5.compression;
import java.lang.reflect.Field;
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java
new file mode 100644
index 000000000..20b69082a
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java
@@ -0,0 +1,93 @@
+package org.janelia.saalfeldlab.n5.demo;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.RawCompression;
+import org.janelia.saalfeldlab.n5.codec.CodecInfo;
+import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo;
+import org.janelia.saalfeldlab.n5.util.GridIterator;
+
+public class BlockIterators {
+
+ public static void main(String[] args) {
+
+// blockIterator();
+// shardBlockIterator();
+ }
+
+ public static void shardBlockIterator() {
+
+// final DatasetAttributes attrs = new DatasetAttributes(
+// new long[] {12, 8}, // image size
+// new int[] {6, 4}, // shard size
+// new int[] {2, 2}, // block size
+// DataType.UINT8,
+// new ShardingCodec(
+// new int[] {2, 2},
+// new CodecInfo[] { new N5BlockCodecInfo() },
+// new DeterministicSizeCodecInfo[] { new RawBlockCodecInfo() },
+// IndexLocation.END
+// ));
+//
+// shardPositions(attrs)
+// .forEach(x -> System.out.println(Arrays.toString(x)));
+ }
+
+// public static void blockIterator() {
+//
+// final DatasetAttributes attrs = new DatasetAttributes(
+// new long[] {12, 8},
+// new int[] {2, 2},
+// DataType.UINT8,
+// new RawCompression());
+//
+// blockPositions(attrs).forEach(x -> System.out.println(Arrays.toString(x)));
+// }
+//
+// public static long[] blockGridSize(final DatasetAttributes attrs ) {
+// // this could be a nice method for DatasetAttributes
+//
+// return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getBlockSize()[i])).toArray();
+//
+// }
+//
+// public static long[] shardGridSize(final DatasetAttributes attrs ) {
+// // this could be a nice method for DatasetAttributes
+//
+// return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getShardSize()[i])).toArray();
+//
+// }
+//
+// public static Stream blockPositions( DatasetAttributes attrs ) {
+// return toStream(new GridIterator(blockGridSize(attrs)));
+// }
+//
+// public static Stream shardPositions( DatasetAttributes attrs ) {
+//
+// final int[] blocksPerShard = attrs.getBlocksPerShard();
+// return toStream( new GridIterator(shardGridSize(attrs)))
+// .flatMap( shardPosition -> {
+//
+// final int nd = attrs.getNumDimensions();
+// final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]);
+// return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min));
+// });
+// }
+//
+// public static Stream toStream( final Iterator it ) {
+// return StreamSupport.stream( Spliterators.spliteratorUnknownSize(
+// it, Spliterator.ORDERED),
+// false);
+// }
+
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java
index 68edb9d3f..aba51a4d0 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java
@@ -29,9 +29,9 @@
package org.janelia.saalfeldlab.n5.http;
import com.google.gson.Gson;
+import com.google.gson.JsonElement;
import org.janelia.saalfeldlab.n5.CachedGsonKeyValueN5Reader;
import org.janelia.saalfeldlab.n5.CachedGsonKeyValueN5Writer;
-import org.janelia.saalfeldlab.n5.Compression;
import org.janelia.saalfeldlab.n5.DataBlock;
import org.janelia.saalfeldlab.n5.DataType;
import org.janelia.saalfeldlab.n5.DatasetAttributes;
@@ -39,6 +39,7 @@
import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer;
import org.janelia.saalfeldlab.n5.KeyValueAccess;
import org.janelia.saalfeldlab.n5.N5Exception;
+import org.janelia.saalfeldlab.n5.N5KeyValueReader;
import java.io.Serializable;
import java.lang.reflect.Field;
@@ -50,6 +51,8 @@
import java.util.concurrent.ExecutorService;
import java.util.function.Predicate;
+import static org.janelia.saalfeldlab.n5.N5KeyValueReader.ATTRIBUTES_JSON;
+
public class HttpReaderFsWriter implements GsonKeyValueN5Writer {
private final GsonKeyValueN5Writer writer;
@@ -66,8 +69,13 @@ public HttpRead
if (cachedReader.cacheMeta()) {
/* Hack necessary to test HTTP reader caching without creating the data entirely first */
try {
- // Access the private 'cache' field in the reader
- final Field cacheField = reader.getClass().getDeclaredField("cache");
+ // Access the private 'cache' field in the reader (or the N5KeyValueReader as a fallback)
+ Field cacheField;
+ try {
+ cacheField = reader.getClass().getDeclaredField("cache");
+ } catch (NoSuchFieldException e) {
+ cacheField = N5KeyValueReader.class.getDeclaredField("cache");
+ }
cacheField.setAccessible(true);
// Set the value of 'cache' to the one from writer.getCache()
@@ -81,6 +89,11 @@ public HttpRead
}
+ @Override public String getAttributesKey() {
+
+ return writer.getAttributesKey();
+ }
+
@Override public Version getVersion() throws N5Exception {
return reader.getVersion();
@@ -255,11 +268,6 @@ public HttpRead
writer.createDataset(datasetPath, datasetAttributes);
}
- @Override public void createDataset(String datasetPath, long[] dimensions, int[] blockSize, DataType dataType, Compression compression) throws N5Exception {
-
- writer.createDataset(datasetPath, dimensions, blockSize, dataType, compression);
- }
-
@Override public void writeBlock(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception {
writer.writeBlock(datasetPath, datasetAttributes, dataBlock);
}
@@ -273,4 +281,29 @@ public HttpRead
writer.writeSerializedBlock(object, datasetPath, datasetAttributes, gridPosition);
}
+
+ @Override public void setVersion(String path) {
+
+ writer.setVersion(path);
+ }
+
+ @Override public void writeAttributes(String normalGroupPath, JsonElement attributes) throws N5Exception {
+
+ writer.writeAttributes(normalGroupPath, attributes);
+ }
+
+ @Override public void setAttributes(String path, JsonElement attributes) throws N5Exception {
+
+ writer.setAttributes(path, attributes);
+ }
+
+ @Override public void writeAttributes(String normalGroupPath, Map attributes) throws N5Exception {
+
+ writer.writeAttributes(normalGroupPath, attributes);
+ }
+
+ @Override public void writeBlocks(String datasetPath, DatasetAttributes datasetAttributes, DataBlock... dataBlocks) throws N5Exception {
+
+ writer.writeBlocks(datasetPath, datasetAttributes, dataBlocks);
+ }
}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java
index 31f3e01af..ca65fd766 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java
@@ -26,9 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- *
- */
package org.janelia.saalfeldlab.n5.kva;
import org.janelia.saalfeldlab.n5.KeyValueAccess;
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java
index 7e0c7a912..ff1bf517f 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java
@@ -26,9 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- *
- */
package org.janelia.saalfeldlab.n5.kva;
import static org.junit.Assert.assertArrayEquals;
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java
index 19e439799..486ae34f3 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java
@@ -26,9 +26,6 @@
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
-/**
- *
- */
package org.janelia.saalfeldlab.n5.kva;
import org.janelia.saalfeldlab.n5.AbstractN5Test;
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java
new file mode 100644
index 000000000..b478c5189
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java
@@ -0,0 +1,92 @@
+package org.janelia.saalfeldlab.n5.serialization;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.janelia.saalfeldlab.n5.Compression;
+import org.janelia.saalfeldlab.n5.CompressionAdapter;
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.GsonUtils;
+import org.janelia.saalfeldlab.n5.GzipCompression;
+import org.janelia.saalfeldlab.n5.NameConfigAdapter;
+import org.janelia.saalfeldlab.n5.codec.DataCodec;
+import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec;
+import org.janelia.saalfeldlab.n5.codec.CodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.IdentityCodec;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+public class CodecSerialization {
+
+ private Gson gson;
+
+ @Before
+ public void before() {
+
+ final GsonBuilder gsonBuilder = new GsonBuilder();
+ gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter());
+ gsonBuilder.registerTypeHierarchyAdapter(DataCodecInfo.class, NameConfigAdapter.getJsonAdapter(DataCodecInfo.class));
+ gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class));
+ gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter());
+ gsonBuilder.disableHtmlEscaping();
+ gson = gsonBuilder.create();
+ }
+
+ @Test
+ public void testCodecSerialization() {
+
+ final IdentityCodec id = new IdentityCodec();
+ final JsonObject jsonId = gson.toJsonTree(id).getAsJsonObject();
+ final JsonElement expected = gson.fromJson("{\"name\":\"id\"}", JsonElement.class);
+ assertEquals("identity json", expected, jsonId.getAsJsonObject());
+
+ final BitShiftBytesCodec codec = new BitShiftBytesCodec(3);
+ final JsonObject bitShiftJson = gson.toJsonTree(codec).getAsJsonObject();
+ final JsonElement expectedBitShift = gson.fromJson(
+ "{\"name\":\"bitshift\",\"configuration\":{\"shift\":3}}",
+ JsonElement.class);
+ assertEquals("bitshift json", expectedBitShift, bitShiftJson);
+
+ final DataCodecInfo deserializedCodecInfo = gson.fromJson(bitShiftJson, DataCodecInfo.class);
+ // Verify deserialized codec
+ assertEquals("Deserialized codec should equal original", codec, deserializedCodecInfo);
+ }
+
+ @Test
+ public void testSerializeCodecArray() {
+
+ CodecInfo[] codecs = new CodecInfo[]{
+ new IdentityCodec()
+ };
+ JsonArray jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray();
+ JsonElement expected = gson.fromJson(
+ "[{\"name\":\"id\"}]",
+ JsonElement.class);
+ assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray());
+
+ CodecInfo[] codecsDeserialized = gson.fromJson(expected, CodecInfo[].class);
+ assertEquals("codecs length not 1", 1, codecsDeserialized.length);
+ assertTrue("first codec not identity", codecsDeserialized[0] instanceof IdentityCodec);
+
+ codecs = new CodecInfo[]{
+ new GzipCompression()
+ };
+ jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray();
+ expected = gson.fromJson(
+ "[{\"name\":\"gzip\",\"configuration\":{\"level\":-1,\"useZlib\":false}}]",
+ JsonElement.class);
+ assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray());
+
+ codecsDeserialized = gson.fromJson(expected, CodecInfo[].class);
+ assertEquals("codecs length not 1", 1, codecsDeserialized.length);
+ assertTrue("second codec not gzip", codecsDeserialized[0] instanceof GzipCompression);
+ }
+
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java
new file mode 100644
index 000000000..d213c0713
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java
@@ -0,0 +1,88 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertThrows;
+
+import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class NestedGridTest {
+
+ private static long absPosition1D(final NestedGrid grid, final int sourcePos, final int targetLevel) {
+ return grid.absolutePosition(new long[] {sourcePos}, targetLevel)[0];
+ }
+
+ @Test
+ public void testValidateInput() {
+ int[][] blockSizes = {{3}, {7}, {11}};
+ assertThrows(IllegalArgumentException.class, () -> new NestedGrid(blockSizes));
+ }
+
+ @Test
+ public void testAbsolutePosition() {
+ int[][] blockSizes = {{1}, {3}, {6}, {24}};
+ NestedGrid grid = new NestedGrid(blockSizes);
+
+ Assert.assertEquals(38, absPosition1D(grid, 38, 0));
+
+ Assert.assertEquals(12, absPosition1D(grid, 36, 1));
+ Assert.assertEquals(12, absPosition1D(grid, 37, 1));
+ Assert.assertEquals(12, absPosition1D(grid, 38, 1));
+
+ Assert.assertEquals(6, absPosition1D(grid, 38, 2));
+ Assert.assertEquals(1, absPosition1D(grid, 38, 3));
+ }
+
+ @Test
+ public void testAbsolutePositionChunkSize() {
+ int[][] blockSizes = {{10}, {30}, {60}, {240}};
+ NestedGrid grid = new NestedGrid(blockSizes);
+
+ Assert.assertEquals(38, absPosition1D(grid, 38, 0));
+ Assert.assertEquals(12, absPosition1D(grid, 38, 1));
+ Assert.assertEquals(6, absPosition1D(grid, 38, 2));
+ Assert.assertEquals(1, absPosition1D(grid, 38, 3));
+ }
+
+ private static long relPosition1D(final NestedGrid grid, final int sourcePos, final int targetLevel) {
+ return grid.relativePosition(new long[] {sourcePos}, targetLevel)[0];
+ }
+
+ @Test
+ public void testRelativePosition() {
+ int[][] blockSizes = {{1}, {3}, {6}, {24}};
+ NestedGrid grid = new NestedGrid(blockSizes);
+
+ Assert.assertEquals(2, relPosition1D(grid, 38, 0));
+ Assert.assertEquals(0, relPosition1D(grid, 38, 1));
+ Assert.assertEquals(2, relPosition1D(grid, 38, 2));
+ Assert.assertEquals(1, relPosition1D(grid, 38, 3));
+
+ }
+
+ @Test
+ public void testRelativePositionChunkSize() {
+ int[][] blockSizes = {{10}, {30}, {60}, {240}};
+ NestedGrid grid = new NestedGrid(blockSizes);
+
+ Assert.assertEquals(2, relPosition1D(grid, 38, 0));
+ Assert.assertEquals(0, relPosition1D(grid, 38, 1));
+ Assert.assertEquals(2, relPosition1D(grid, 38, 2));
+ Assert.assertEquals(1, relPosition1D(grid, 38, 3));
+ }
+
+ @Test
+ public void testNd() {
+
+ int[][] blockSizes = {{5, 7}, {5*3, 7*2}};
+ NestedGrid grid = new NestedGrid(blockSizes);
+ System.out.println(grid);
+ assertArrayEquals(new long[]{1, 2}, grid.absolutePosition(new long[]{1, 2}, 0));
+ assertArrayEquals(new long[]{99, 99}, grid.absolutePosition(new long[]{99, 99}, 0));
+
+ assertArrayEquals(new long[]{0, 0}, grid.absolutePosition(new long[]{0, 1}, 1));
+ assertArrayEquals(new long[]{0, 1}, grid.absolutePosition(new long[]{0, 2}, 1));
+ assertArrayEquals(new long[]{1, 1}, grid.absolutePosition(new long[]{3, 2}, 1));
+ }
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java
new file mode 100644
index 000000000..eaa895420
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java
@@ -0,0 +1,142 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import java.util.Arrays;
+import org.janelia.saalfeldlab.n5.ByteArrayDataBlock;
+import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.RawCompression;
+import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo;
+import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation;
+
+public class RawShardTest {
+
+
+ public static void main(String[] args) {
+
+ int[] datablockSize = {3, 3, 3};
+ int[] level1ShardSize = {6, 6, 6};
+ int[] level2ShardSize = {24, 24, 24};
+
+ // DataBlocks are 3x3x3
+ // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks)
+ // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards)
+ final BlockCodecInfo c0 = new N5BlockCodecInfo();
+ final ShardCodecInfo c1 = new DefaultShardCodecInfo(
+ datablockSize,
+ c0,
+ new DataCodecInfo[] {new RawCompression()},
+ new RawBlockCodecInfo(),
+ new DataCodecInfo[] {new RawCompression()},
+ IndexLocation.END
+ );
+ final ShardCodecInfo c2 = new DefaultShardCodecInfo(
+ level1ShardSize,
+ c1,
+ new DataCodecInfo[] {new RawCompression()},
+ new RawBlockCodecInfo(),
+ new DataCodecInfo[] {new RawCompression()},
+ IndexLocation.START
+ );
+
+ TestDatasetAttributes attributes = new TestDatasetAttributes(
+ new long[] {},
+ level2ShardSize,
+ DataType.INT8,
+ c2,
+ new RawCompression());
+
+ final DatasetAccess datasetAccess = attributes.datasetAccess();
+
+ // TODO: N5Reader/Writer needs to provide a PositionValueAccess implementation on top of its KVA.
+ // The read/write/deleteBlock methods would getDataAccess() from the DatasetAttributes and call it with that PositionValueAccess.
+ final PositionValueAccess store = new TestPositionValueAccess();
+
+
+ // ---------------------------------------------------------------
+ // Some "tests"
+ // TODO: Turn into unit tests
+ // ---------------------------------------------------------------
+
+ // write some blocks, filled with constant values
+ final int[] dataBlockSize = c1.getInnerBlockSize();
+ datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 0, 0}, 1));
+ datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 0, 0}, 2));
+ datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 1, 0}, 3));
+ datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 1, 0}, 4));
+ datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {3, 2, 1}, 5));
+ datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {8, 4, 1}, 6));
+
+ // verify that the written blocks can be read back with the correct values
+ checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), true, 1);
+ checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2);
+ checkBlock(datasetAccess.readBlock(store, new long[] {0, 1, 0}), true, 3);
+ checkBlock(datasetAccess.readBlock(store, new long[] {1, 1, 0}), true, 4);
+ checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5);
+ checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6);
+
+ // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present)
+ datasetAccess.deleteBlock(store, new long[] {0, 0, 0});
+ checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1);
+ checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2);
+
+ // if a shard becomes empty the corresponding key should be deleted
+ if ( store.get(new long[] {1, 0, 0}) == null ) {
+ throw new IllegalStateException("expected non-null readData");
+ }
+ datasetAccess.deleteBlock(store, new long[] {8, 4, 1});
+ if ( store.get(new long[] {1, 0, 0}) != null ) {
+ throw new IllegalStateException("expected null readData");
+ }
+
+ // deleting a non-existent block should not fail
+ datasetAccess.deleteBlock(store, new long[] {0, 0, 8});
+
+ System.out.println("all good");
+ }
+
+ private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) {
+
+ if (dataBlock == null) {
+ if (expectedNonNull) {
+ throw new IllegalStateException("expected non-null dataBlock");
+ }
+ } else {
+ if (!expectedNonNull) {
+ throw new IllegalStateException("expected null dataBlock");
+ }
+ final byte[] bytes = dataBlock.getData();
+ for (byte b : bytes) {
+ if (b != (byte) expectedFillValue) {
+ throw new IllegalStateException("expected all values to be " + expectedFillValue);
+ }
+ }
+ }
+ }
+
+ private static DataBlock createDataBlock(int[] size, long[] gridPosition, int fillValue) {
+ final byte[] bytes = new byte[DataBlock.getNumElements(size)];
+ Arrays.fill(bytes, (byte) fillValue);
+ return new ByteArrayDataBlock(size, gridPosition, bytes);
+ }
+
+ public static class TestDatasetAttributes extends DatasetAttributes {
+
+ public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType dataType, BlockCodecInfo blockCodecInfo,
+ DataCodecInfo... dataCodecInfos) {
+
+ super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos);
+ }
+
+ public DatasetAccess datasetAccess() {
+
+ // to make this accessible for the test
+ return createDatasetAccess();
+ }
+
+ }
+
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java
new file mode 100644
index 000000000..ea1406d83
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java
@@ -0,0 +1,105 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import org.janelia.saalfeldlab.n5.KeyValueAccess;
+import org.janelia.saalfeldlab.n5.LockedChannel;
+import org.janelia.saalfeldlab.n5.N5FSTest;
+import org.janelia.saalfeldlab.n5.N5KeyValueWriter;
+import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter;
+import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec;
+import org.janelia.saalfeldlab.n5.util.GridIterator;
+import org.junit.After;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Paths;
+
+import static org.junit.Assert.assertEquals;
+
+public class ShardIndexTest {
+
+ private static final N5FSTest tempN5Factory = new N5FSTest();
+
+ @After
+ public void removeTempWriters() {
+
+ tempN5Factory.removeTempWriters();
+ }
+
+ @Test
+ @Ignore
+ public void testOffsetIndex() {
+
+ // TODO
+// int[] shardBlockGridSize = new int[]{5, 4, 3};
+// ShardIndex index = new ShardIndex(
+// shardBlockGridSize,
+// IndexLocation.END, new RawBlockCodecInfo());
+//
+// GridIterator it = new GridIterator(shardBlockGridSize);
+// int i = 0;
+// while (it.hasNext()) {
+// int j = index.getOffsetIndex(GridIterator.long2int(it.next()));
+// assertEquals(i, j);
+// i += 2;
+// }
+//
+// shardBlockGridSize = new int[]{5, 4, 3, 13};
+// index = new ShardIndex(
+// shardBlockGridSize,
+// IndexLocation.END, new RawBlockCodecInfo());
+//
+// it = new GridIterator(shardBlockGridSize);
+// i = 0;
+// while (it.hasNext()) {
+// int j = index.getOffsetIndex(GridIterator.long2int(it.next()));
+// assertEquals(i, j);
+// i += 2;
+// }
+
+ }
+
+ @Test
+ @Ignore
+ public void writeReadTest() throws IOException {
+
+ // TODO
+
+// final N5KeyValueWriter writer = (N5KeyValueWriter)tempN5Factory.createTempN5Writer();
+// final KeyValueAccess kva = writer.getKeyValueAccess();
+//
+// final int[] shardBlockGridSize = new int[]{6, 5};
+// final IndexLocation indexLocation = IndexLocation.END;
+// final IndexCodecAdapter indexCodecAdapter = new IndexCodecAdapter(
+// new RawBlockCodecInfo(),
+// new Crc32cChecksumCodec()
+// );
+//
+// final ShardIndex index = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecAdapter);
+// index.set(0, 6, new int[]{0, 0});
+// index.set(19, 32, new int[]{1, 0});
+// index.set(93, 111, new int[]{3, 0});
+// index.set(143, 1, new int[]{1, 2});
+//
+// final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "indexTest").toString();
+// try (
+// final LockedChannel channel = kva.lockForWriting(path);
+// final OutputStream out = channel.newOutputStream()
+// ) {
+//
+// ShardIndex.write(out, index);
+// }
+//
+// final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecAdapter);
+// try (
+// final LockedChannel channel = kva.lockForReading(path);
+// final InputStream in = channel.newInputStream()
+// ) {
+// ShardIndex.read(in, indexRead);
+// }
+// assertEquals(index, indexRead);
+ }
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java
new file mode 100644
index 000000000..18c0a86cf
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java
@@ -0,0 +1,587 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import org.janelia.saalfeldlab.n5.ByteArrayDataBlock;
+import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess;
+import org.janelia.saalfeldlab.n5.KeyValueAccess;
+import org.janelia.saalfeldlab.n5.KeyValueAccessReadData;
+import org.janelia.saalfeldlab.n5.N5Exception;
+import org.janelia.saalfeldlab.n5.N5FSTest;
+import org.janelia.saalfeldlab.n5.N5KeyValueWriter;
+import org.janelia.saalfeldlab.n5.N5Writer;
+import org.janelia.saalfeldlab.n5.NameConfigAdapter;
+import org.janelia.saalfeldlab.n5.RawCompression;
+import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException;
+import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer;
+import org.janelia.saalfeldlab.n5.codec.CodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@RunWith(Parameterized.class)
+public class ShardTest {
+
+ private static final boolean LOCAL_DEBUG = false;
+
+ private static final N5FSTest tempN5Factory = new N5FSTest() {
+
+ @Override public N5Writer createTempN5Writer() {
+
+ if (LOCAL_DEBUG) {
+ final N5Writer writer = new ShardedN5Writer("src/test/resources/test.n5");
+ writer.remove(""); // Clear old when starting new test
+ return writer;
+ }
+
+ final String basePath = new File(tempN5PathName()).toURI().normalize().getPath();
+ try {
+ String uri = new URI("file", null, basePath, null).toString();
+ return new ShardedN5Writer(uri);
+ } catch (URISyntaxException e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ private String tempN5PathName() {
+
+ try {
+ final File tmpFile = Files.createTempDirectory("n5-shard-test-").toFile();
+ tmpFile.delete();
+ tmpFile.mkdir();
+ tmpFile.deleteOnExit();
+ return tmpFile.getCanonicalPath();
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
+ public static GsonBuilder gsonBuilder() {
+ return new GsonBuilder();
+ }
+
+ @Parameterized.Parameters(name = "IndexLocation({0}), Index ByteOrder({1})")
+ public static Collection data() {
+
+ final ArrayList params = new ArrayList<>();
+ for (IndexLocation indexLoc : IndexLocation.values()) {
+ for (ByteOrder indexByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) {
+ params.add(new Object[]{indexLoc, indexByteOrder});
+ }
+ }
+ final int numParams = params.size();
+ final Object[][] paramArray = new Object[numParams][];
+ Arrays.setAll(paramArray, params::get);
+ return Arrays.asList(paramArray);
+ }
+
+ @Parameterized.Parameter()
+ public IndexLocation indexLocation;
+
+ @Parameterized.Parameter(1)
+ public ByteOrder indexByteOrder;
+
+ @After
+ public void removeTempWriters() {
+
+ tempN5Factory.removeTempWriters();
+ }
+
+ private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] blockSize) {
+
+ DefaultShardCodecInfo blockCodec = new DefaultShardCodecInfo(
+ blockSize,
+ new N5BlockCodecInfo(),
+ new DataCodecInfo[]{new RawCompression()},
+ new RawBlockCodecInfo(),
+ new DataCodecInfo[]{new RawCompression()},
+ IndexLocation.END);
+
+ return new DatasetAttributes(
+ dimensions,
+ shardSize,
+ DataType.UINT8,
+ blockCodec);
+ }
+
+ protected DatasetAttributes getTestAttributes() {
+
+ return getTestAttributes(new long[]{8, 8}, new int[]{4, 4}, new int[]{2, 2});
+ }
+
+ @Test
+ public void writeReadBlocksTest() {
+
+ final N5Writer writer = tempN5Factory.createTempN5Writer();
+
+ final DatasetAttributes datasetAttributes = getTestAttributes(
+ new long[]{24, 24},
+ new int[]{8, 8},
+ new int[]{2, 2}
+ );
+
+ final String dataset = "writeReadBlocks";
+ writer.remove(dataset);
+ writer.createDataset(dataset, datasetAttributes);
+
+ final int[] blockSize = datasetAttributes.getBlockSize();
+ final int numElements = blockSize[0] * blockSize[1];
+
+ final byte[] data = new byte[numElements];
+ for (int i = 0; i < data.length; i++) {
+ data[i] = (byte)((100) + (10) + i);
+ }
+
+ writer.writeBlocks(
+ dataset,
+ datasetAttributes,
+ /* shard (0, 0) */
+ new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data),
+ new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data),
+ new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data),
+ new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data),
+
+ /* shard (1, 0) */
+ new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data),
+ new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data),
+
+ /* shard (2, 2) */
+ new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data)
+ );
+
+ final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess();
+
+ long[][] keys = new long[][]{
+ {0, 0},
+ {1, 0},
+ {2, 2}
+ };
+
+ final long[][] someUnusedKeys = new long[][]{
+ {0, 1},
+ {1, 1},
+ {1, 2},
+ {2, 1}
+ };
+
+ ensureKeysExist(kva, writer.getURI(), dataset, datasetAttributes, keys);
+ ensureKeysDoNotExist(kva, writer.getURI(), dataset, datasetAttributes, someUnusedKeys);
+
+ final long[][] blockIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}};
+ for (long[] blockIndex : blockIndices) {
+ final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex);
+ Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData());
+ }
+
+ final byte[] data2 = new byte[numElements];
+ for (int i = 0; i < data2.length; i++) {
+ data2[i] = (byte)(10 + i);
+ }
+
+ writer.writeBlocks(
+ dataset,
+ datasetAttributes,
+ /* shard (0, 0) */
+ new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data2),
+ new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data2),
+
+ /* shard (0, 1) */
+ new ByteArrayDataBlock(blockSize, new long[]{0, 4}, data2),
+ new ByteArrayDataBlock(blockSize, new long[]{0, 5}, data2),
+
+ /* shard (2, 2) */
+ new ByteArrayDataBlock(blockSize, new long[]{10, 10}, data2));
+
+ long[][] keys2 = new long[][]{
+ {0, 0},
+ {1, 0},
+ {0, 1},
+ {2, 2}
+ };
+
+ long[][] someUnusedKeys2 = new long[][]{
+ {1, 1},
+ {1, 2},
+ {2, 1}
+ };
+
+ ensureKeysExist(kva, writer.getURI(), dataset, datasetAttributes, keys2);
+ ensureKeysDoNotExist(kva, writer.getURI(), dataset, datasetAttributes, someUnusedKeys2);
+
+ final long[][] oldBlockIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}};
+ for (long[] blockIndex : oldBlockIndices) {
+ final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex);
+ Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData());
+ }
+
+ final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}};
+ final List newBlockIndexList = Arrays.asList(newBlockIndices);
+ final List