void writeBlock(
"Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path,
e);
}
+
}
@Override
@@ -300,4 +301,4 @@ default boolean deleteBlock(
final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes);
return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition);
}
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java
index 3f15be8f9..cc9bc7646 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java
@@ -28,6 +28,7 @@
*/
package org.janelia.saalfeldlab.n5;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
@@ -196,39 +197,46 @@ default boolean remove() throws N5Exception {
/**
* Creates a dataset. This does not create any data but the path and
- * mandatory attributes only.
+ * mandatory attributes only. The returned DatasetAttributes should be used
+ * for future read/write operations on this dataset. It may not be the same
+ * DatasetAttributes object that was provided, depending on the implementation.
*
* @param datasetPath dataset path
* @param datasetAttributes the dataset attributes
- * @throws N5Exception the exception
+ * @return DatasetAttributes optimal attributes object to be used for read/write operations
+ * @throws N5Exception
*/
- default void createDataset(
+ default DatasetAttributes createDataset(
final String datasetPath,
final DatasetAttributes datasetAttributes) throws N5Exception {
final String normalPath = N5URI.normalizeGroupPath(datasetPath);
createGroup(normalPath);
setDatasetAttributes(normalPath, datasetAttributes);
+ return datasetAttributes;
}
/**
* Creates a dataset. This does not create any data but the path and
- * mandatory attributes only.
+ * mandatory attributes only. Returns the DatasetAttributes object to be
+ * used for future read/write operations on this dataset.
*
* @param datasetPath dataset path
* @param dimensions the dataset dimensions
* @param blockSize the block size
* @param dataType the data type
* @param compression the compression
+ * @return DatasetAttributes optimal attributes object to be used for read/write operations
+ * @throws N5Exception
*/
- default void createDataset(
+ default DatasetAttributes createDataset(
final String datasetPath,
final long[] dimensions,
final int[] blockSize,
final DataType dataType,
final Compression compression) throws N5Exception {
- createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression));
+ return createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression));
}
/**
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java
index 77fd10eb4..fe846ebd6 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java
@@ -1,31 +1,3 @@
-/*-
- * #%L
- * Not HDF5
- * %%
- * Copyright (C) 2017 - 2025 Stephan Saalfeld
- * %%
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- * #L%
- */
/**
* Copyright (c) 2017, Stephan Saalfeld
* All rights reserved.
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodec.java
new file mode 100644
index 000000000..dee1ba8ba
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodec.java
@@ -0,0 +1,21 @@
+package org.janelia.saalfeldlab.n5.codec;
+
+import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+
+/**
+ * A Codec that transforms the contents of a {@link DataBlock}.
+ *
+ * This class is N5's analogue to Zarr's array -> array codec.
+ */
+public interface DatasetCodec {
+
+ // TODO Name ideas:
+ // "ImageCodec"?
+ // BlockTransformationCodec
+
+ DataBlock encode(DataBlock block) throws N5IOException;
+
+ DataBlock decode(DataBlock dataBlock) throws N5IOException;
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodecInfo.java
new file mode 100644
index 000000000..8ca2f40ad
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DatasetCodecInfo.java
@@ -0,0 +1,38 @@
+/*-
+ * #%L
+ * Not HDF5
+ * %%
+ * Copyright (C) 2017 - 2025 Stephan Saalfeld
+ * %%
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * #L%
+ */
+package org.janelia.saalfeldlab.n5.codec;
+
+import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.serialization.NameConfig;
+
+@NameConfig.Prefix("data-codec")
+public interface DatasetCodecInfo extends CodecInfo {
+
+ DatasetCodec create(final DatasetAttributes attributes);
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java
index 1b0e848ef..4d4b981ff 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java
@@ -40,13 +40,12 @@
import org.janelia.saalfeldlab.n5.DataType;
import org.janelia.saalfeldlab.n5.serialization.NameConfig;
-
@NameConfig.Name(value = RawBlockCodecInfo.TYPE)
public class RawBlockCodecInfo implements BlockCodecInfo {
private static final long serialVersionUID = 3282569607795127005L;
- public static final String TYPE = "bytes";
+ public static final String TYPE = "raw-bytes";
@NameConfig.Parameter(value = "endian", optional = true)
private final ByteOrder byteOrder;
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java
index 2e5aa9817..f82d27054 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java
@@ -29,40 +29,53 @@
package org.janelia.saalfeldlab.n5.codec.checksum;
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
-import java.util.zip.CheckedInputStream;
+import java.nio.ByteOrder;
+import java.util.function.Supplier;
import java.util.zip.CheckedOutputStream;
import java.util.zip.Checksum;
+import org.janelia.saalfeldlab.n5.N5Exception;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import org.janelia.saalfeldlab.n5.codec.CodecInfo;
import org.janelia.saalfeldlab.n5.codec.DataCodec;
import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
-import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DeterministicSizeDataCodec;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
/**
- * A {@link CodecInfo} that appends a checksum to data when encoding and can validate against that checksum when decoding.
+ * A {@link CodecInfo} that appends a checksum to data when encoding and can
+ * validate against that checksum when decoding.
+ *
+ * Checksum codec instances are expected to be thread safe, but {@link Checksum}
+ * implementations may not be. As a result, subclasses of this implementation
+ * provide a {@link Supplier} for an appropriate Checksum type, a new instance
+ * of which is created by {@link #getChecksum()} for each
+ * {@link #encode(ReadData)} and {@link #decode(ReadData)} call.
*/
-public abstract class ChecksumCodec implements DataCodec, DataCodecInfo, DeterministicSizeCodecInfo {
+public abstract class ChecksumCodec implements DataCodec, DataCodecInfo, DeterministicSizeDataCodec {
private static final long serialVersionUID = 3141427377277375077L;
private int numChecksumBytes;
- private Checksum checksum;
+ private Supplier checksumSupplier;
- public ChecksumCodec(Checksum checksum, int numChecksumBytes) {
+ public ChecksumCodec(Supplier checksumSupplier, int numChecksumBytes) {
- this.checksum = checksum;
+ this.checksumSupplier = checksumSupplier;
this.numChecksumBytes = numChecksumBytes;
}
+ /**
+ * Returns a new {@link Checksum} instance.
+ *
+ * @return the checksum
+ */
public Checksum getChecksum() {
- return checksum;
+ return checksumSupplier.get();
}
public int numChecksumBytes() {
@@ -71,13 +84,14 @@ public int numChecksumBytes() {
}
private CheckedOutputStream createStream(OutputStream out) {
- return new CheckedOutputStream(out, getChecksum()) {
+ final Checksum checksum = getChecksum();
+ return new CheckedOutputStream(out, checksum) {
private boolean closed = false;
- @Override public void close() throws IOException {
-
+ @Override
+ public void close() throws IOException {
if (!closed) {
- writeChecksum(out);
+ writeChecksum(checksum, out);
closed = true;
out.close();
}
@@ -88,37 +102,48 @@ private CheckedOutputStream createStream(OutputStream out) {
@Override public ReadData encode(ReadData readData) {
return readData.encode(this::createStream);
-
}
@Override public ReadData decode(ReadData readData) throws N5IOException {
+ final ReadData rdm = readData.materialize();
+ final long N = rdm.requireLength();
- return ReadData.from(new CheckedInputStream(readData.inputStream(), getChecksum()));
- }
+ final ReadData data = rdm.slice(0, N - numChecksumBytes);
+ final long calculatedChecksum = computeChecksum(data);
- @Override
- public long encodedSize(final long size) {
+ final ReadData checksumRd = rdm.slice(N - numChecksumBytes, numChecksumBytes);
+ final long storedChecksum = readChecksum(checksumRd);
- return size + numChecksumBytes();
+ if( calculatedChecksum != storedChecksum)
+ throw new N5Exception(String.format("Calculated checksum (%d) does not match stored checksum (%d).",
+ calculatedChecksum, storedChecksum));
+
+ return data;
}
@Override
- public long decodedSize(final long size) {
+ public long encodedSize(final long size) {
- return size - numChecksumBytes();
+ return size + numChecksumBytes();
}
- protected boolean valid(InputStream in) throws IOException {
+ protected long readChecksum(ReadData checksumData) {
- return readChecksum(in) == getChecksum().getValue();
+ // the computed checksum is a long that can take values in [0, 2^32 - 1]
+ // so convert the four bytes to an appropriate long
+ ByteBuffer buf = ByteBuffer.allocate(8);
+ buf.order(ByteOrder.LITTLE_ENDIAN);
+ buf.put(checksumData.allBytes());
+ buf.putInt(0);
+ buf.rewind();
+ return buf.getLong();
}
- protected long readChecksum(InputStream in) throws IOException {
-
- final byte[] checksum = new byte[numChecksumBytes()];
- in.read(checksum);
- return ByteBuffer.wrap(checksum).getLong();
+ protected long computeChecksum(ReadData data) {
+ final Checksum checksum = getChecksum();
+ checksum.update(data.allBytes(), 0, (int)data.requireLength());
+ return checksum.getValue();
}
/**
@@ -126,12 +151,11 @@ protected long readChecksum(InputStream in) throws IOException {
*
* @return a ByteBuffer representing the checksum value
*/
- public abstract ByteBuffer getChecksumValue();
+ public abstract ByteBuffer getChecksumValue(Checksum checksum);
- public void writeChecksum(OutputStream out) throws IOException {
+ protected void writeChecksum(Checksum checksum, OutputStream out) throws IOException {
- out.write(getChecksumValue().array());
+ out.write(getChecksumValue(checksum).array());
}
-
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java
index 678e14e8c..43c1a0203 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java
@@ -28,12 +28,13 @@
*/
package org.janelia.saalfeldlab.n5.codec.checksum;
-import org.apache.commons.lang3.NotImplementedException;
+import org.apache.commons.codec.digest.PureJavaCrc32C;
import org.janelia.saalfeldlab.n5.codec.DataCodec;
import org.janelia.saalfeldlab.n5.serialization.NameConfig;
import java.nio.ByteBuffer;
-import java.util.zip.CRC32;
+import java.nio.ByteOrder;
+import java.util.zip.Checksum;
@NameConfig.Name(Crc32cChecksumCodec.TYPE)
public class Crc32cChecksumCodec extends ChecksumCodec {
@@ -44,26 +45,14 @@ public class Crc32cChecksumCodec extends ChecksumCodec {
public Crc32cChecksumCodec() {
- super(new CRC32(), 4);
+ super(() -> new PureJavaCrc32C(), 4);
}
@Override
- public long encodedSize(final long size) {
-
- return size + numChecksumBytes();
- }
-
- @Override
- public long decodedSize(final long size) {
-
- return size - numChecksumBytes();
- }
-
- @Override
- public ByteBuffer getChecksumValue() {
+ public ByteBuffer getChecksumValue(Checksum checksum) {
final ByteBuffer buf = ByteBuffer.allocate(numChecksumBytes());
- buf.putInt((int)getChecksum().getValue());
+ buf.order(ByteOrder.LITTLE_ENDIAN).putInt((int)checksum.getValue());
return buf;
}
@@ -77,4 +66,5 @@ public String getType() {
return this;
}
+
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/Transpose.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/Transpose.java
new file mode 100644
index 000000000..90ccc0ff9
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/Transpose.java
@@ -0,0 +1,240 @@
+package org.janelia.saalfeldlab.n5.codec.transpose;
+
+import org.janelia.saalfeldlab.n5.DataType;
+
+class Transpose {
+
+ // TODO: detect when 1-sized dimensions are permuted. This should allow to
+ // simplify (or completely avoid) copying under certain conditions.
+
+ public static int[] encode(final int[] decodedPos, final int[] order) {
+ final int[] encodedPos = new int[decodedPos.length];
+ encode(decodedPos, order, encodedPos);
+ return encodedPos;
+ }
+
+ public static int[] decode(final int[] encodedPos, final int[] order) {
+ final int[] decodedPos = new int[encodedPos.length];
+ decode(encodedPos, order, decodedPos);
+ return decodedPos;
+ }
+
+ public static void encode(int[] decodedPos, int[] order, int[] encodedPos) {
+ for (int d = 0; d < order.length; d++)
+ encodedPos[d] = decodedPos[order[d]];
+ }
+
+ public static void decode(int[] encodedPos, int[] order, int[] decodedPos) {
+ for (int d = 0; d < order.length; d++)
+ decodedPos[order[d]] = encodedPos[d];
+ }
+
+ @SuppressWarnings("unchecked")
+ public static Transpose of(final DataType dataType, final int numDimensions) {
+ return (Transpose) new Transpose<>(MemCopy.forDataType(dataType), numDimensions);
+ }
+
+ private final MemCopy memCopy;
+
+ private final int[] ssize;
+ private final int[] tsize;
+ private final int[] ssteps;
+ private final int[] tsteps;
+ private final int[] csteps;
+
+ Transpose(final MemCopy memCopy, final int n) {
+ this.memCopy = memCopy;
+ ssize = new int[n];
+ tsize = new int[n];
+ ssteps = new int[n];
+ tsteps = new int[n];
+ csteps = new int[n];
+ }
+
+ public void encode(final T decoded, final T encoded, final int[] decodedSize, final int[] order) {
+ final int n = ssize.length;
+
+ for (int d = 0; d < n; ++d)
+ ssize[d] = decodedSize[d];
+
+ for (int d = 0; d < n; ++d)
+ tsize[d] = decodedSize[order[d]];
+
+ ssteps[0] = 1;
+ for (int d = 0; d < n - 1; ++d)
+ ssteps[d + 1] = ssteps[d] * ssize[d];
+
+ tsteps[0] = 1;
+ for (int d = 0; d < n - 1; ++d)
+ tsteps[d + 1] = tsteps[d] * tsize[d];
+
+ for (int d = 0; d < n; ++d)
+ csteps[order[d]] = tsteps[d];
+
+ copyRecursively(decoded, 0, encoded, 0, n - 1);
+ }
+
+ public void decode(final T encoded, final T decoded, final int[] decodedSize, final int[] order) {
+ final int n = ssize.length;
+
+ for (int d = 0; d < n; ++d)
+ ssize[d] = decodedSize[order[d]];
+
+ ssteps[0] = 1;
+ for (int d = 0; d < n - 1; ++d)
+ ssteps[d + 1] = ssteps[d] * ssize[d];
+
+ tsteps[0] = 1;
+ for (int d = 0; d < n - 1; ++d)
+ tsteps[d + 1] = tsteps[d] * decodedSize[d];
+
+ for (int d = 0; d < n; ++d)
+ csteps[d] = tsteps[order[d]];
+
+ copyRecursively(encoded, 0, decoded, 0, n - 1);
+ }
+
+ private void copyRecursively(final T src, final int srcPos, final T dest, final int destPos, final int d) {
+ if (d == 0) {
+ final int length = ssize[d];
+ final int stride = csteps[d];
+ memCopy.copyStrided(src, srcPos, dest, destPos, stride, length);
+ } else {
+ final int length = ssize[d];
+ final int srcStride = ssteps[d];
+ final int destStride = csteps[d];
+ for (int i = 0; i < length; ++i)
+ copyRecursively(src, srcPos + i * srcStride, dest, destPos + i * destStride, d - 1);
+ }
+ }
+
+ Transpose newInstance() {
+ return new Transpose<>(memCopy, ssize.length);
+ }
+
+ /**
+ * Low-level range copying methods between source and target primitve array
+ * (type {@code T}, e.g., {@code double[]}).
+ *
+ * @param
+ * the source/target type. Must be a primitive array type (e.g., {@code double[]})
+ */
+ interface MemCopy {
+
+ MemCopy.MemCopyByte BYTE = new MemCopy.MemCopyByte();
+ MemCopy.MemCopyShort SHORT = new MemCopy.MemCopyShort();
+ MemCopy.MemCopyInt INT = new MemCopy.MemCopyInt();
+ MemCopy.MemCopyLong LONG = new MemCopy.MemCopyLong();
+ MemCopy.MemCopyFloat FLOAT = new MemCopy.MemCopyFloat();
+ MemCopy.MemCopyDouble DOUBLE = new MemCopy.MemCopyDouble();
+
+ static MemCopy forDataType(final DataType dataType) {
+ switch (dataType) {
+ case UINT8:
+ case INT8:
+ return BYTE;
+ case UINT16:
+ case INT16:
+ return SHORT;
+ case UINT32:
+ case INT32:
+ return INT;
+ case UINT64:
+ case INT64:
+ return LONG;
+ case FLOAT32:
+ return FLOAT;
+ case FLOAT64:
+ return DOUBLE;
+ case STRING:
+ case OBJECT:
+ throw new UnsupportedOperationException("TODO?");
+ default:
+ throw new IllegalArgumentException();
+ }
+ }
+
+ /**
+ * Copy {@code length} components from the {@code src} array to the {@code
+ * dest} array. The components at positions {@code srcPos} through {@code
+ * srcPos+length-1} in the source array are copied into positions {@code
+ * destPos}, {@code destPos+destStride}, {@code destPos + 2*destStride},
+ * etc., through {@code destPos+(length-1)*destStride} of the destination
+ * array.
+ */
+ void copyStrided(T src, int srcPos, T dest, int destPos, int destStride, int length);
+
+ class MemCopyByte implements MemCopy {
+
+ @Override
+ public void copyStrided(final byte[] src, final int srcPos, final byte[] dest, final int destPos, final int destStride, final int length) {
+ if (destStride == 1)
+ System.arraycopy(src, srcPos, dest, destPos, length);
+ else
+ for (int i = 0; i < length; ++i)
+ dest[destPos + i * destStride] = src[srcPos + i];
+ }
+ }
+
+ class MemCopyShort implements MemCopy {
+
+ @Override
+ public void copyStrided(final short[] src, final int srcPos, final short[] dest, final int destPos, final int destStride, final int length) {
+ if (destStride == 1)
+ System.arraycopy(src, srcPos, dest, destPos, length);
+ else
+ for (int i = 0; i < length; ++i)
+ dest[destPos + i * destStride] = src[srcPos + i];
+ }
+ }
+
+ class MemCopyInt implements MemCopy {
+
+ @Override
+ public void copyStrided(final int[] src, final int srcPos, final int[] dest, final int destPos, final int destStride, final int length) {
+ if (destStride == 1)
+ System.arraycopy(src, srcPos, dest, destPos, length);
+ else
+ for (int i = 0; i < length; ++i)
+ dest[destPos + i * destStride] = src[srcPos + i];
+ }
+ }
+
+ class MemCopyLong implements MemCopy {
+
+ @Override
+ public void copyStrided(final long[] src, final int srcPos, final long[] dest, final int destPos, final int destStride, final int length) {
+ if (destStride == 1)
+ System.arraycopy(src, srcPos, dest, destPos, length);
+ else
+ for (int i = 0; i < length; ++i)
+ dest[destPos + i * destStride] = src[srcPos + i];
+ }
+ }
+
+ class MemCopyFloat implements MemCopy {
+
+ @Override
+ public void copyStrided(final float[] src, final int srcPos, final float[] dest, final int destPos, final int destStride, final int length) {
+ if (destStride == 1)
+ System.arraycopy(src, srcPos, dest, destPos, length);
+ else
+ for (int i = 0; i < length; ++i)
+ dest[destPos + i * destStride] = src[srcPos + i];
+ }
+ }
+
+ class MemCopyDouble implements MemCopy {
+
+ @Override
+ public void copyStrided(final double[] src, final int srcPos, final double[] dest, final int destPos, final int destStride, final int length) {
+ if (destStride == 1)
+ System.arraycopy(src, srcPos, dest, destPos, length);
+ else
+ for (int i = 0; i < length; ++i)
+ dest[destPos + i * destStride] = src[srcPos + i];
+ }
+ }
+ }
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposeCodec.java
new file mode 100644
index 000000000..b6de0a041
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposeCodec.java
@@ -0,0 +1,107 @@
+package org.janelia.saalfeldlab.n5.codec.transpose;
+
+import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.codec.DatasetCodec;
+
+public class TransposeCodec implements DatasetCodec {
+
+ private DataType dataType;
+ private final int[] order;
+
+ private final Transpose transpose;
+
+ public TransposeCodec(DataType dataType, int[] order) {
+
+ this.order = order;
+ this.dataType = dataType;
+ transpose = Transpose.of(dataType, order.length);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public DataBlock encode(DataBlock dataBlock) {
+
+ DataBlock encodedBlock = (DataBlock)dataType.createDataBlock(dataBlock.getSize(), dataBlock.getGridPosition(), dataBlock.getNumElements());
+ transpose.encode(dataBlock.getData(), encodedBlock.getData(), dataBlock.getSize(), order);
+ return encodedBlock;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public DataBlock decode(DataBlock dataBlock) {
+
+ DataBlock decodedBlock = (DataBlock)dataType.createDataBlock(dataBlock.getSize(), dataBlock.getGridPosition(), dataBlock.getNumElements());
+ transpose.decode((T)dataBlock.getData(), decodedBlock.getData(), dataBlock.getSize(), order);
+ return decodedBlock;
+ }
+
+ public static boolean isIdentity(int[] permutation) {
+
+ for (int i = 0; i < permutation.length; i++)
+ if (permutation[i] != i)
+ return false;
+
+ return true;
+ }
+
+ public static boolean isReversal(int[] permutation) {
+
+ for (int i = 0; i < permutation.length; i++)
+ if (permutation[i] != i)
+ return false;
+
+ return true;
+ }
+
+ public static int[] invertPermutation(final int[] p) {
+
+ final int[] inv = new int[p.length];
+ for (int i = 0; i < p.length; i++)
+ inv[p[i]] = i;
+
+ return inv;
+ }
+
+ /**
+ * Composes two permutations: result[i] = first[second[i]].
+ *
+ * @param first
+ * the first permutation
+ * @param second
+ * the second permutation
+ * @return the composition of first and second
+ */
+ public static int[] concatenatePermutations(final int[] first, final int[] second) {
+
+ int n = first.length;
+ final int[] result = new int[n];
+ for (int i = 0; i < n; i++) {
+ result[i] = first[second[i]];
+ }
+ return result;
+ }
+
+ /**
+ * Conjugates a permutation with the reversal permutation: rev * p * rev^-1,
+ * where rev is the permutation that reverses the elements.
+ *
+ * @param p
+ * the permutation to conjugate
+ * @return the conjugated permutation
+ */
+ public static int[] conjugateWithReverse(final int[] p) {
+
+ final int n = p.length;
+ final int[] rev = new int[n];
+ for (int i = 0; i < n; i++)
+ rev[i] = n - i - 1;
+
+ // note that rev is its own inverse
+ int[] result = concatenatePermutations(rev, p); // result = rev * p
+ result = concatenatePermutations(result, rev); // result = rev * p *
+ // rev^-1
+ return result;
+ }
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposeCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposeCodecInfo.java
new file mode 100644
index 000000000..550432cef
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposeCodecInfo.java
@@ -0,0 +1,96 @@
+package org.janelia.saalfeldlab.n5.codec.transpose;
+
+import java.util.Arrays;
+import java.util.stream.IntStream;
+
+import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.N5Exception;
+import org.janelia.saalfeldlab.n5.codec.DatasetCodec;
+import org.janelia.saalfeldlab.n5.codec.DatasetCodecInfo;
+import org.janelia.saalfeldlab.n5.serialization.NameConfig;
+
+/**
+ * Describes a permutation of the dimensions of a block.
+ *
+ * The {@code order} parameter parameterizes the permutation.
+ * The ith element of the order array gives the destination index of the ith element of the input.
+ * Example:
+ * order = [1, 2, 0]
+ * input = [7, 8, 9] // interpret as a block size
+ * result = [9, 7, 8] // permuted block size
+ *
+ *
+ * See the specification of Zarr's Transpose codec.
+ */
+@NameConfig.Name(value = TransposeCodecInfo.TYPE)
+public class TransposeCodecInfo implements DatasetCodecInfo {
+
+ public static final String TYPE = "n5-transpose";
+
+ @NameConfig.Parameter
+ private int[] order;
+
+ public TransposeCodecInfo() {
+ // for serialization
+ }
+
+ public TransposeCodecInfo(int[] order) {
+
+ this.order = order;
+ }
+
+ @Override
+ public String getType() {
+
+ return TYPE;
+ }
+
+ public int[] getOrder() {
+
+ return order;
+ }
+
+ @Override
+ public DatasetCodec create(DatasetAttributes datasetAttributes) {
+
+ validate();
+ return new TransposeCodec(datasetAttributes.getDataType(), getOrder());
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+
+ if (obj instanceof TransposeCodecInfo)
+ return Arrays.equals(order, ((TransposeCodecInfo)obj).getOrder());
+
+ return false;
+ }
+
+ private void validate() {
+
+ final boolean[] indexFound = new boolean[order.length];
+ for( int i : order )
+ indexFound[i] = true;
+
+ final int[] missingIndexes = IntStream.range(0, order.length).filter(i -> !indexFound[i]).toArray();
+ if( missingIndexes.length > 0 )
+ throw new N5Exception("Invalid order for TransposeCodec. Missing indexes: " + Arrays.toString(missingIndexes));
+
+ }
+
+ public static TransposeCodecInfo concatenate(TransposeCodecInfo[] infos) {
+
+ if( infos == null || infos.length == 0)
+ return null;
+
+ // copy the initial order so we don't modify to the original
+ int[] order = new int[infos[0].order.length];
+ System.arraycopy(infos[0].order, 0, order, 0, order.length);
+
+ for( int i = 1; i < infos.length; i++ )
+ order = TransposeCodec.concatenatePermutations(order, infos[i].order);
+
+ return new TransposeCodecInfo(order);
+ }
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposePlayground.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposePlayground.java
new file mode 100644
index 000000000..5b9b29ee7
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/transpose/TransposePlayground.java
@@ -0,0 +1,84 @@
+package org.janelia.saalfeldlab.n5.codec.transpose;
+
+import java.util.Arrays;
+
+import org.janelia.saalfeldlab.n5.DataType;
+
+/**
+ * Exploration for implementing "transpose" codec in N5
+ */
+public class TransposePlayground {
+
+ public static void main(String[] args) {
+// transpose2D();
+// System.out.println("\n\n --------------------------------------- \n\n");
+// transpose3D();
+ }
+
+ public static void transpose2D() {
+
+ Transpose transpose = Transpose.of(DataType.UINT32, 2);
+
+ int[] values = {
+ 1, 2, 3,
+ 4, 5, 6
+ };
+
+ int[] decoded_size = {3, 2};
+ int[] order = {0, 1};
+// int[] order = {1, 0};
+ System.out.println("decoded_size = " + Arrays.toString(decoded_size));
+ System.out.println("order = " + Arrays.toString(order));
+
+ System.out.println("values = \n" + toString(values, decoded_size));
+
+ int[] encoded = new int[values.length];
+ transpose.encode(values, encoded, decoded_size, order);
+
+ int[] encoded_size = Transpose.encode(decoded_size, order);
+ System.out.println("encoded = \n" + toString(encoded, encoded_size));
+
+ int[] decoded = new int[values.length];
+ transpose.decode(encoded, decoded, decoded_size, order);
+
+ System.out.println("decoded = \n" + toString(decoded, decoded_size));
+ }
+
+ public static void transpose3D() {
+
+// Transpose transpose = new Transpose<>(Transpose.MemCopy.INT, 3);
+ Transpose transpose = Transpose.of(DataType.UINT32, 3);
+
+ int[] decoded_size = {4, 3, 2};
+ int[] order = {1, 2, 0};
+ System.out.println("decoded_size = " + Arrays.toString(decoded_size));
+ System.out.println("order = " + Arrays.toString(order));
+
+ int[] encoded_size = Transpose.encode(decoded_size, order);
+ System.out.println("encoded_size = " + Arrays.toString(encoded_size));
+
+ int[] values = new int[4 * 3 * 2];
+ Arrays.setAll(values, i -> i);
+ System.out.println("values = \n" + toString(values, decoded_size));
+
+ int[] encoded = new int[values.length];
+ transpose.encode(values, encoded, decoded_size, order);
+ System.out.println("encoded = \n" + toString(encoded, encoded_size));
+
+ int[] decoded = new int[values.length];
+ transpose.decode(encoded, decoded, decoded_size, order);
+ System.out.println("decoded = \n" + toString(decoded, decoded_size));
+ }
+
+ static String toString(int[] values, int[] size) {
+ StringBuilder str = new StringBuilder();
+ final int w = size[0];
+ final int h = size[1];
+ for (int i = 0; i < values.length; i += w) {
+ str.append(Arrays.toString(Arrays.copyOfRange(values, i, Math.min(values.length, i + w)))).append("\n");
+ if ((i + w) % (w * h) == 0)
+ str.append("\n");
+ }
+ return str.toString();
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java
index 74f64d6e6..574acbabd 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java
@@ -42,6 +42,7 @@
import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException;
import org.janelia.saalfeldlab.n5.N5Writer.DataBlockSupplier;
import org.janelia.saalfeldlab.n5.codec.BlockCodec;
+import org.janelia.saalfeldlab.n5.codec.DatasetCodec;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid;
import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition;
@@ -52,10 +53,16 @@ public class DefaultDatasetAccess implements DatasetAccess {
private final NestedGrid grid;
private final BlockCodec[] codecs;
+ private final DatasetCodec[] datasetCodecs;
- public DefaultDatasetAccess(final NestedGrid grid, final BlockCodec[] codecs) {
+ public DefaultDatasetAccess(final NestedGrid grid, final BlockCodec[] codecs, DatasetCodec[] datasetCodecs ) {
this.grid = grid;
this.codecs = codecs;
+ this.datasetCodecs = datasetCodecs;
+ }
+
+ public DefaultDatasetAccess(final NestedGrid grid, final BlockCodec[] codecs) {
+ this( grid, codecs, new DatasetCodec[0]);
}
public NestedGrid getGrid() {
@@ -65,7 +72,7 @@ public NestedGrid getGrid() {
@Override
public DataBlock readBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException {
final NestedPosition position = grid.nestedPosition(gridPosition);
- return readBlockRecursive(pva.get(position.key()), position, grid.numLevels() - 1);
+ return (DataBlock)decodeWithDatasetCodecs(readBlockRecursive(pva.get(position.key()), position, grid.numLevels() - 1));
}
private DataBlock readBlockRecursive(
@@ -145,6 +152,7 @@ private List> readShardRecursive(
final NestedPosition firstBlock = positions.get(0);
final long[] shardPosition = firstBlock.absolute(level);
+ @SuppressWarnings("unchecked")
final BlockCodec codec = (BlockCodec) codecs[level];
final RawShard shard = codec.decode(readData, shardPosition).getData();
@@ -171,7 +179,10 @@ private List> readShardRecursive(
}
@Override
- public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException {
+ public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlockArg) throws N5IOException {
+
+ @SuppressWarnings("unchecked")
+ final DataBlock dataBlock = (DataBlock)encodeWithDatasetCodecs(dataBlockArg);
final NestedPosition position = grid.nestedPosition(dataBlock.getGridPosition());
final long[] key = position.key();
@@ -467,4 +478,24 @@ private DataBlock getDataBlock() {
return dataBlock;
}
}
+
+ private DataBlock decodeWithDatasetCodecs(DataBlock block) {
+
+ DataBlock result = block;
+ for (DatasetCodec codec : datasetCodecs) {
+ result = codec.decode(result);
+ }
+ return result;
+ }
+
+ private DataBlock encodeWithDatasetCodecs(DataBlock block) {
+
+ DataBlock result = block;
+ for (DatasetCodec codec : datasetCodecs) {
+ result = codec.encode(result);
+ }
+ return result;
+ }
+
+
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java
index 251d5c6c6..da4799835 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java
@@ -30,6 +30,7 @@
import java.util.Arrays;
import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.N5Exception;
import org.janelia.saalfeldlab.n5.codec.BlockCodec;
import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo;
import org.janelia.saalfeldlab.n5.codec.CodecInfo;
@@ -62,13 +63,13 @@ public String getType() {
@NameConfig.Parameter(value = "index_codecs")
private CodecInfo[] indexCodecs;
- private transient final BlockCodecInfo innerBlockCodecInfo;
+ private transient BlockCodecInfo innerBlockCodecInfo;
- private transient final DataCodecInfo[] innerDataCodecInfos;
+ private transient DataCodecInfo[] innerDataCodecInfos;
- private transient final BlockCodecInfo indexBlockCodecInfo;
+ private transient BlockCodecInfo indexBlockCodecInfo;
- private transient final DataCodecInfo[] indexDataCodecInfos;
+ private transient DataCodecInfo[] indexDataCodecInfos;
DefaultShardCodecInfo() {
// for serialization
@@ -94,6 +95,36 @@ public DefaultShardCodecInfo(
indexCodecs = concatenateCodecs(indexBlockCodecInfo, indexDataCodecInfos);
}
+ private void build() {
+
+ if (innerBlockCodecInfo != null)
+ return;
+
+ // sets
+ // innerBlockCodecInfo, innerDataCodecInfos
+ // indexBlockCodecInfo, indexDataCodecInfos
+ // from
+ // codecs and indexCodecs
+
+ if (codecs[0] instanceof BlockCodecInfo)
+ innerBlockCodecInfo = (BlockCodecInfo)codecs[0];
+ else
+ throw new N5Exception("Codec at index " + 0 + " must be a BlockCodec.");
+
+ innerDataCodecInfos = new DataCodecInfo[codecs.length - 1];
+ for (int i = 1; i < codecs.length; i++)
+ innerDataCodecInfos[i - 1] = (DataCodecInfo)codecs[i];
+
+ if (indexCodecs[0] instanceof BlockCodecInfo)
+ indexBlockCodecInfo = (BlockCodecInfo)indexCodecs[0];
+ else
+ throw new N5Exception("Codec at index " + 0 + " must be a BlockCodec.");
+
+ indexDataCodecInfos = new DataCodecInfo[indexCodecs.length - 1];
+ for (int i = 1; i < indexCodecs.length; i++)
+ indexDataCodecInfos[i - 1] = (DataCodecInfo)indexCodecs[i];
+ }
+
@Override
public int[] getInnerBlockSize() {
return innerBlockSize;
@@ -135,6 +166,8 @@ public CodecInfo[] getIndexCodecs() {
@Override
public RawShardCodec create(final int[] blockSize, final DataCodecInfo... codecs) {
+ build();
+
// Number of elements (DataBlocks, nested shards) in each dimension per shard.
final int[] size = new int[blockSize.length];
// blockSize argument is number of pixels in the shard
@@ -161,4 +194,5 @@ private static CodecInfo[] concatenateCodecs(BlockCodecInfo blkInfo, DataCodecIn
return allCodecs;
}
+
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java b/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java
new file mode 100644
index 000000000..1b7076d54
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java
@@ -0,0 +1,38 @@
+package org.janelia.saalfeldlab.n5.util;
+
+/*
+ * An immutable {@Position}.
+ */
+public class FinalPosition implements Position {
+
+ public final long[] position;
+
+ public FinalPosition(long[] position) {
+ this.position = position;
+ }
+
+ public FinalPosition(Position p) {
+ this.position = p.get().clone();
+ }
+
+ @Override
+ public long[] get() {
+ return position;
+ }
+
+ @Override
+ public long get(int i) {
+ return position[i];
+ }
+
+ @Override
+ public String toString() {
+ return Position.toString(this);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return Position.equals(this, obj);
+ }
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/FloatValueParser.java b/src/main/java/org/janelia/saalfeldlab/n5/util/FloatValueParser.java
new file mode 100644
index 000000000..5a2860f8b
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/util/FloatValueParser.java
@@ -0,0 +1,110 @@
+package org.janelia.saalfeldlab.n5.util;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.janelia.saalfeldlab.n5.N5Exception;
+
+/**
+ * Parses {@link Float} and {@link Double} values from JSON hex strings.
+ *
+ * This does not directly cover the Strings "NaN", "Infinity", and "-Infinity"
+ * but they are parsable by the parseDouble and parseFloat methods. Rather, this
+ * class handles converting to and from the hex representations of NaN,
+ * -Infinity, Infinity, and all other allowable values.
+ */
+public class FloatValueParser {
+
+ /**
+ * Parses a hex string to a float value.
+ *
+ * @param hexString
+ * hex string in format "0x" followed by 8 hex digits
+ * @return the float value
+ * @throws N5Exception
+ * if the string format is invalid
+ */
+ public static float parseFloat(String hexString) throws N5Exception {
+
+ validateFloat(hexString);
+ final int intValue = Integer.parseUnsignedInt(hexString.substring(2), 16);
+ return Float.intBitsToFloat(intValue);
+ }
+
+ /**
+ * Encodes a float value to a hex string.
+ *
+ * @param value
+ * the float to encode
+ * @return hex string in format "0x" followed by 8 hex digits
+ */
+ public static String encodeFloat(float value) {
+
+ return String.format("0x%08x", Float.floatToIntBits(value));
+ }
+
+ private static void validateFloat(String hexString) {
+
+ if (!hexString.startsWith("0x") || hexString.length() != 10)
+ throw new N5Exception("Could not parse string " + hexString + " as float.");
+ }
+
+ /**
+ * Parses a hex string to a double value.
+ *
+ * @param hexString
+ * hex string in format "0x" followed by 16 hex digits
+ * @return the double value
+ * @throws N5Exception
+ * if the string format is invalid
+ */
+ public static double parseDouble(String hexString) throws N5Exception {
+
+ validateDouble(hexString);
+ final long longValue = Long.parseUnsignedLong(hexString.substring(2), 16);
+ return Double.longBitsToDouble(longValue);
+ }
+
+ /**
+ * Encodes a double value to a hex string.
+ *
+ * @param value
+ * the double to encode
+ * @return hex string in format "0x" followed by 16 hex digits
+ */
+ public static String encodeDouble(double value) {
+
+ return String.format("0x%016x", Double.doubleToLongBits(value));
+ }
+
+ private static void validateDouble(String hexString) {
+
+ if (!hexString.startsWith("0x") || hexString.length() != 18)
+ throw new N5Exception("Could not parse string " + hexString + " as double.");
+ }
+
+ /**
+ * Parses a hex string to a byte array.
+ *
+ * @param hexString
+ * hex string in format "0x" followed by hex digits
+ * @return the decoded byte array
+ * @throws N5Exception
+ * if the string format is invalid or decoding fails
+ */
+ public static byte[] parseBytes(String hexString) throws N5Exception {
+
+ validateBytes(hexString);
+ try {
+ return Hex.decodeHex(hexString.substring(2));
+ } catch (DecoderException e) {
+ throw new N5Exception(e);
+ }
+ }
+
+ private static void validateBytes(String hexString) {
+
+ if (!hexString.startsWith("0x"))
+ throw new N5Exception("Could not parse string " + hexString + " to bytes.");
+ }
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java
new file mode 100644
index 000000000..43258efaf
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java
@@ -0,0 +1,178 @@
+package org.janelia.saalfeldlab.n5.util;
+
+import java.util.Iterator;
+
+/**
+ * Essentially imglib2's IntervalIterator, but N5 does not depend on imglib2.
+ */
+public class GridIterator implements Iterator {
+
+ final protected long[] dimensions;
+
+ final protected long[] steps;
+
+ final protected long[] position;
+
+ final protected int[] intPosition;
+
+ final protected long[] min;
+
+ final protected int lastIndex;
+
+ protected int index = -1;
+
+ public GridIterator(final long[] dimensions, final long[] min) {
+
+ final int n = dimensions.length;
+ this.dimensions = new long[n];
+ this.position = new long[n];
+ this.intPosition = new int[n];
+ this.min = min;
+ steps = new long[n];
+
+ final int m = n - 1;
+ long k = steps[0] = 1;
+ for (int d = 0; d < m; ) {
+ final long dimd = dimensions[d];
+ this.dimensions[d] = dimd;
+ k *= dimd;
+ steps[++d] = k;
+ }
+ final long dimm = dimensions[m];
+ this.dimensions[m] = dimm;
+ lastIndex = (int)(k * dimm - 1);
+ }
+
+ public GridIterator(final long[] dimensions) {
+
+ this(dimensions, new long[dimensions.length]);
+ }
+
+ public GridIterator(final int[] dimensions) {
+
+ this(int2long(dimensions));
+ }
+
+ public void fwd() {
+
+ ++index;
+ }
+
+ public void reset() {
+
+ index = -1;
+ }
+
+ @Override
+ public boolean hasNext() {
+
+ return index < lastIndex;
+ }
+
+ @Override
+ public long[] next() {
+
+ fwd();
+ indexToPosition(index, dimensions, min, position);
+ return position;
+ }
+
+ public int[] nextInt() {
+
+ next();
+ long2int(position, intPosition);
+ return intPosition;
+ }
+
+ public int getIndex() {
+
+ return index;
+ }
+
+ public static void indexToPosition(long index, final long[] dimensions, final long[] offset,
+ final long[] position) {
+
+ for (int dim = 0; dim < dimensions.length; dim++) {
+ position[dim] = (index % dimensions[dim]) + offset[dim];
+ index /= dimensions[dim];
+ }
+ }
+
+ public static void indexToPosition(long index, final int[] dimensions, final long[] offset,
+ final long[] position) {
+
+ for (int dim = 0; dim < dimensions.length; dim++) {
+ position[dim] = (index % dimensions[dim]) + offset[dim];
+ index /= dimensions[dim];
+ }
+ }
+
+ final static public long positionToIndex(final long[] dimensions, final long[] position) {
+ long idx = 0;
+ int cumulativeSize = 1;
+ for (int i = 0; i < position.length; i++) {
+ idx += position[i] * cumulativeSize;
+ cumulativeSize *= dimensions[i];
+ }
+ return idx;
+ }
+
+ final static public long positionToIndex(final long[] dimensions, final int[] position) {
+ long idx = 0;
+ int cumulativeSize = 1;
+ for (int i = 0; i < position.length; i++) {
+ idx += position[i] * cumulativeSize;
+ cumulativeSize *= dimensions[i];
+ }
+ return idx;
+ }
+
+ final static public long positionToIndex(final int[] dimensions, final long[] position) {
+ long idx = 0;
+ int cumulativeSize = 1;
+ for (int i = 0; i < position.length; i++) {
+ idx += position[i] * cumulativeSize;
+ cumulativeSize *= dimensions[i];
+ }
+ return idx;
+ }
+
+ final static public long positionToIndex(final int[] dimensions, final int[] position) {
+ long idx = 0;
+ int cumulativeSize = 1;
+ for (int i = 0; i < position.length; i++) {
+ idx += position[i] * cumulativeSize;
+ cumulativeSize *= dimensions[i];
+ }
+ return idx;
+ }
+
+ public static int[] long2int(final long[] src, final int[] tgt) {
+
+ for (int d = 0; d < tgt.length; ++d)
+ tgt[d] = (int)src[d];
+
+ return tgt;
+ }
+
+ public static int[] long2int(final long[] a) {
+
+ final int[] i = new int[a.length];
+
+ for (int d = 0; d < a.length; ++d)
+ i[d] = (int)a[d];
+
+ return i;
+ }
+
+ public static long[] int2long(final int[] i) {
+
+ final long[] l = new long[i.length];
+
+ for (int d = 0; d < l.length; ++d)
+ l[d] = i[d];
+
+ return l;
+ }
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java
new file mode 100644
index 000000000..2403835b9
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java
@@ -0,0 +1,66 @@
+package org.janelia.saalfeldlab.n5.util;
+
+import java.util.Arrays;
+
+/*
+ * A wrapper around a primitive long array that is lexicographically {@link Comparable}
+ * and for which we can test equality.
+ */
+public interface Position extends Comparable {
+
+ long[] get();
+
+ long get(int i);
+
+ default int numDimensions() {
+ return get().length;
+ }
+
+ @Override
+ default int compareTo(Position other) {
+
+ // use Arrays.compare when we update to Java 9+
+ final int N = numDimensions() > other.numDimensions() ? numDimensions() : other.numDimensions();
+ for (int i = 0; i < N; i++) {
+ final long diff = get(i) - other.get(i);
+ if (diff != 0)
+ return (int) diff;
+ }
+ return 0;
+ }
+
+ static boolean equals(final Position a, final Object b) {
+
+ if (a == null && b == null)
+ return true;
+
+ if (b == null)
+ return false;
+
+ if (!(b instanceof Position))
+ return false;
+
+ final Position other = (Position) b;
+ if (other.numDimensions() != a.numDimensions())
+ return false;
+
+ for (int i = 0; i < a.numDimensions(); i++)
+ if (other.get(i) != a.get(i))
+ return false;
+
+ return true;
+ }
+
+ static String toString(Position p) {
+ return "Position: " + Arrays.toString(p.get());
+ }
+
+ static Position wrap(final long[] p) {
+ return new FinalPosition(p);
+ }
+
+ static Position wrap(final int[] p) {
+ return new FinalPosition(GridIterator.int2long(p));
+ }
+
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/ChecksumCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/ChecksumCodecTests.java
new file mode 100644
index 000000000..da2e73a59
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/ChecksumCodecTests.java
@@ -0,0 +1,36 @@
+package org.janelia.saalfeldlab.n5.codec;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+
+import org.janelia.saalfeldlab.n5.N5Exception;
+import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.junit.Test;
+
+public class ChecksumCodecTests {
+
+ @Test
+ public void testCrc32cChecksumCodec() {
+
+ final ReadData rd = ReadData.from(new byte[] {0,1,2,3,4,5,6,7,8,9});
+ final long N = rd.requireLength();
+
+ final Crc32cChecksumCodec codec = new Crc32cChecksumCodec();
+ final ReadData encoded = codec.encode(rd);
+
+ // Crc32 adds 4 bytes to the data
+ assertEquals(N+codec.numChecksumBytes(), encoded.requireLength());
+
+ final ReadData decoded = codec.decode(encoded);
+ assertArrayEquals(rd.allBytes(), decoded.allBytes());
+
+ // attempting to decode perturbed data throws exception
+ final byte[] encodedBytes = encoded.allBytes();
+ encodedBytes[1]++;
+ final ReadData perturbed = ReadData.from(encodedBytes);
+ assertThrows(N5Exception.class, () -> codec.decode(perturbed));
+ }
+
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/DatasetCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/DatasetCodecTests.java
new file mode 100644
index 000000000..2a94bd8d2
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/DatasetCodecTests.java
@@ -0,0 +1,48 @@
+package org.janelia.saalfeldlab.n5.codec;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import org.janelia.saalfeldlab.n5.codec.transpose.TransposeCodecInfo;
+import org.junit.Test;
+
+public class DatasetCodecTests {
+
+ @Test
+ public void testTransposeCodecSimplification() throws Exception {
+
+ // 2d
+ final TransposeCodecInfo id2 = new TransposeCodecInfo(new int[]{0, 1});
+ final TransposeCodecInfo rev2 = new TransposeCodecInfo(new int[]{1, 0});
+
+ assertNull(TransposeCodecInfo.concatenate(null));
+ assertEquals(id2, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{id2}));
+ assertEquals(rev2, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{rev2}));
+
+ assertEquals(rev2, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{rev2, id2}));
+ assertEquals(rev2, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{id2, rev2, id2}));
+
+ assertEquals(id2, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{rev2, rev2}));
+ assertEquals(id2, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{rev2, rev2, rev2, rev2}));
+ assertEquals(id2, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{id2, rev2, id2, rev2, rev2, rev2}));
+
+ // 3d
+ final TransposeCodecInfo id3 = new TransposeCodecInfo(new int[]{0, 1, 2});
+ final TransposeCodecInfo rev3 = new TransposeCodecInfo(new int[]{2, 1, 0});
+
+ final TransposeCodecInfo t021 = new TransposeCodecInfo(new int[]{0, 2, 1});
+ final TransposeCodecInfo t102 = new TransposeCodecInfo(new int[]{1, 0, 2});
+ final TransposeCodecInfo t120 = new TransposeCodecInfo(new int[]{1, 2, 0});
+ final TransposeCodecInfo t201 = new TransposeCodecInfo(new int[]{2, 0, 1});
+
+ assertEquals(id3, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{id3}));
+ assertEquals(rev3, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{rev3}));
+
+ assertEquals(rev3, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{rev3, id3}));
+ assertEquals(rev3, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{id3, rev3, id3}));
+
+ assertEquals(t102, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{rev3, t102, t021}));
+ assertEquals(t201, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{t021, t102}));
+ assertEquals(t120, TransposeCodecInfo.concatenate(new TransposeCodecInfo[]{t102, t021}));
+ }
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java
new file mode 100644
index 000000000..20b69082a
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java
@@ -0,0 +1,93 @@
+package org.janelia.saalfeldlab.n5.demo;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.DatasetAttributes;
+import org.janelia.saalfeldlab.n5.RawCompression;
+import org.janelia.saalfeldlab.n5.codec.CodecInfo;
+import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo;
+import org.janelia.saalfeldlab.n5.util.GridIterator;
+
+public class BlockIterators {
+
+ public static void main(String[] args) {
+
+// blockIterator();
+// shardBlockIterator();
+ }
+
+ public static void shardBlockIterator() {
+
+// final DatasetAttributes attrs = new DatasetAttributes(
+// new long[] {12, 8}, // image size
+// new int[] {6, 4}, // shard size
+// new int[] {2, 2}, // block size
+// DataType.UINT8,
+// new ShardingCodec(
+// new int[] {2, 2},
+// new CodecInfo[] { new N5BlockCodecInfo() },
+// new DeterministicSizeCodecInfo[] { new RawBlockCodecInfo() },
+// IndexLocation.END
+// ));
+//
+// shardPositions(attrs)
+// .forEach(x -> System.out.println(Arrays.toString(x)));
+ }
+
+// public static void blockIterator() {
+//
+// final DatasetAttributes attrs = new DatasetAttributes(
+// new long[] {12, 8},
+// new int[] {2, 2},
+// DataType.UINT8,
+// new RawCompression());
+//
+// blockPositions(attrs).forEach(x -> System.out.println(Arrays.toString(x)));
+// }
+//
+// public static long[] blockGridSize(final DatasetAttributes attrs ) {
+// // this could be a nice method for DatasetAttributes
+//
+// return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getBlockSize()[i])).toArray();
+//
+// }
+//
+// public static long[] shardGridSize(final DatasetAttributes attrs ) {
+// // this could be a nice method for DatasetAttributes
+//
+// return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getShardSize()[i])).toArray();
+//
+// }
+//
+// public static Stream blockPositions( DatasetAttributes attrs ) {
+// return toStream(new GridIterator(blockGridSize(attrs)));
+// }
+//
+// public static Stream shardPositions( DatasetAttributes attrs ) {
+//
+// final int[] blocksPerShard = attrs.getBlocksPerShard();
+// return toStream( new GridIterator(shardGridSize(attrs)))
+// .flatMap( shardPosition -> {
+//
+// final int nd = attrs.getNumDimensions();
+// final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]);
+// return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min));
+// });
+// }
+//
+// public static Stream toStream( final Iterator it ) {
+// return StreamSupport.stream( Spliterators.spliteratorUnknownSize(
+// it, Spliterator.ORDERED),
+// false);
+// }
+
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java
index aba51a4d0..28139f4da 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java
@@ -33,7 +33,6 @@
import org.janelia.saalfeldlab.n5.CachedGsonKeyValueN5Reader;
import org.janelia.saalfeldlab.n5.CachedGsonKeyValueN5Writer;
import org.janelia.saalfeldlab.n5.DataBlock;
-import org.janelia.saalfeldlab.n5.DataType;
import org.janelia.saalfeldlab.n5.DatasetAttributes;
import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader;
import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer;
@@ -51,8 +50,6 @@
import java.util.concurrent.ExecutorService;
import java.util.function.Predicate;
-import static org.janelia.saalfeldlab.n5.N5KeyValueReader.ATTRIBUTES_JSON;
-
public class HttpReaderFsWriter implements GsonKeyValueN5Writer {
private final GsonKeyValueN5Writer writer;
@@ -263,9 +260,10 @@ public HttpRead
return writer.remove();
}
- @Override public void createDataset(String datasetPath, DatasetAttributes datasetAttributes) throws N5Exception {
+ @Override public DatasetAttributes createDataset(String datasetPath, DatasetAttributes datasetAttributes) throws N5Exception {
writer.createDataset(datasetPath, datasetAttributes);
+ return datasetAttributes;
}
@Override public void writeBlock(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception {
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java
new file mode 100644
index 000000000..ea1406d83
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java
@@ -0,0 +1,105 @@
+package org.janelia.saalfeldlab.n5.shard;
+
+import org.janelia.saalfeldlab.n5.KeyValueAccess;
+import org.janelia.saalfeldlab.n5.LockedChannel;
+import org.janelia.saalfeldlab.n5.N5FSTest;
+import org.janelia.saalfeldlab.n5.N5KeyValueWriter;
+import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter;
+import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo;
+import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec;
+import org.janelia.saalfeldlab.n5.util.GridIterator;
+import org.junit.After;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Paths;
+
+import static org.junit.Assert.assertEquals;
+
+public class ShardIndexTest {
+
+ private static final N5FSTest tempN5Factory = new N5FSTest();
+
+ @After
+ public void removeTempWriters() {
+
+ tempN5Factory.removeTempWriters();
+ }
+
+ @Test
+ @Ignore
+ public void testOffsetIndex() {
+
+ // TODO
+// int[] shardBlockGridSize = new int[]{5, 4, 3};
+// ShardIndex index = new ShardIndex(
+// shardBlockGridSize,
+// IndexLocation.END, new RawBlockCodecInfo());
+//
+// GridIterator it = new GridIterator(shardBlockGridSize);
+// int i = 0;
+// while (it.hasNext()) {
+// int j = index.getOffsetIndex(GridIterator.long2int(it.next()));
+// assertEquals(i, j);
+// i += 2;
+// }
+//
+// shardBlockGridSize = new int[]{5, 4, 3, 13};
+// index = new ShardIndex(
+// shardBlockGridSize,
+// IndexLocation.END, new RawBlockCodecInfo());
+//
+// it = new GridIterator(shardBlockGridSize);
+// i = 0;
+// while (it.hasNext()) {
+// int j = index.getOffsetIndex(GridIterator.long2int(it.next()));
+// assertEquals(i, j);
+// i += 2;
+// }
+
+ }
+
+ @Test
+ @Ignore
+ public void writeReadTest() throws IOException {
+
+ // TODO
+
+// final N5KeyValueWriter writer = (N5KeyValueWriter)tempN5Factory.createTempN5Writer();
+// final KeyValueAccess kva = writer.getKeyValueAccess();
+//
+// final int[] shardBlockGridSize = new int[]{6, 5};
+// final IndexLocation indexLocation = IndexLocation.END;
+// final IndexCodecAdapter indexCodecAdapter = new IndexCodecAdapter(
+// new RawBlockCodecInfo(),
+// new Crc32cChecksumCodec()
+// );
+//
+// final ShardIndex index = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecAdapter);
+// index.set(0, 6, new int[]{0, 0});
+// index.set(19, 32, new int[]{1, 0});
+// index.set(93, 111, new int[]{3, 0});
+// index.set(143, 1, new int[]{1, 2});
+//
+// final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "indexTest").toString();
+// try (
+// final LockedChannel channel = kva.lockForWriting(path);
+// final OutputStream out = channel.newOutputStream()
+// ) {
+//
+// ShardIndex.write(out, index);
+// }
+//
+// final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecAdapter);
+// try (
+// final LockedChannel channel = kva.lockForReading(path);
+// final InputStream in = channel.newInputStream()
+// ) {
+// ShardIndex.read(in, indexRead);
+// }
+// assertEquals(index, indexRead);
+ }
+}