From c4f96f24aaa14aac8881ae62b7acafffba6b1fb8 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 21 May 2024 17:12:39 -0400 Subject: [PATCH 001/423] feat: wip toward codecs --- .../saalfeldlab/n5/DatasetAttributes.java | 41 +++- .../org/janelia/saalfeldlab/n5/N5Writer.java | 30 ++- .../saalfeldlab/n5/codec/AsTypeCodec.java | 178 ++++++++++++++++++ .../janelia/saalfeldlab/n5/codec/Codec.java | 35 ++++ .../saalfeldlab/n5/codec/ComposedCodec.java | 41 ++++ .../FixedLengthConvertedInputStream.java | 71 +++++++ .../FixedLengthConvertedOutputStream.java | 64 +++++++ .../n5/codec/FixedScaleOffsetCodec.java | 55 ++++++ .../saalfeldlab/n5/codec/IdentityCodec.java | 23 +++ .../saalfeldlab/n5/codec/AsTypeTests.java | 80 ++++++++ .../codec/FixedConvertedInputStreamTest.java | 88 +++++++++ .../codec/FixedConvertedOutputStreamTest.java | 111 +++++++++++ .../n5/codec/FixedScaleOffsetTests.java | 62 ++++++ 13 files changed, 874 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedOutputStream.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedInputStreamTest.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index f4aea9fe5..5fa19fc5a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -29,6 +29,8 @@ import java.util.Arrays; import java.util.HashMap; +import org.janelia.saalfeldlab.n5.codec.Codec; + /** * Mandatory dataset attributes: * @@ -38,6 +40,11 @@ *
  • {@link DataType} : dataType
    • *
  • {@link Compression} : compression
    • * + * + * Optional dataset attributes: + *
      + *
    1. {@link Codec}[] : codecs
      2. + *
    * * @author Stephan Saalfeld * @@ -50,6 +57,7 @@ public class DatasetAttributes implements Serializable { public static final String BLOCK_SIZE_KEY = "blockSize"; public static final String DATA_TYPE_KEY = "dataType"; public static final String COMPRESSION_KEY = "compression"; + public static final String CODEC_KEY = "codecs"; /* version 0 */ protected static final String compressionTypeKey = "compressionType"; @@ -58,17 +66,29 @@ public class DatasetAttributes implements Serializable { private final int[] blockSize; private final DataType dataType; private final Compression compression; + private final Codec[] codecs; public DatasetAttributes( final long[] dimensions, final int[] blockSize, final DataType dataType, - final Compression compression) { + final Compression compression, + final Codec[] codecs ) { this.dimensions = dimensions; this.blockSize = blockSize; this.dataType = dataType; this.compression = compression; + this.codecs = codecs; + } + + public DatasetAttributes( + final long[] dimensions, + final int[] blockSize, + final DataType dataType, + final Compression compression) { + + this(dimensions, blockSize, dataType, compression, null); } public long[] getDimensions() { @@ -96,6 +116,11 @@ public DataType getDataType() { return dataType; } + public Codec[] getCodecs() { + + return codecs; + } + public HashMap asMap() { final HashMap map = new HashMap<>(); @@ -103,6 +128,7 @@ public HashMap asMap() { map.put(BLOCK_SIZE_KEY, blockSize); map.put(DATA_TYPE_KEY, dataType); map.put(COMPRESSION_KEY, compression); + map.put(CODEC_KEY, codecs); // TODO : consider not adding to map when null? return map; } @@ -113,6 +139,17 @@ static DatasetAttributes from( Compression compression, final String compressionVersion0Name) { + return from(dimensions, dataType, blockSize, compression, compressionVersion0Name, null); + } + + static DatasetAttributes from( + final long[] dimensions, + final DataType dataType, + int[] blockSize, + Compression compression, + final String compressionVersion0Name, + Codec[] codecs) { + if (blockSize == null) blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); @@ -137,6 +174,6 @@ static DatasetAttributes from( } } - return new DatasetAttributes(dimensions, blockSize, dataType, compression); + return new DatasetAttributes(dimensions, blockSize, dataType, compression, codecs); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 4cfd52bef..867044dba 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -34,6 +34,8 @@ import java.util.List; import java.util.Map; +import org.janelia.saalfeldlab.n5.codec.Codec; + /** * A simple structured container API for hierarchies of chunked * n-dimensional datasets and attributes. @@ -208,8 +210,30 @@ default void createDataset( /** * Creates a dataset. This does not create any data but the path and - * mandatory - * attributes only. + * mandatory attributes only. + * + * @param datasetPath dataset path + * @param dimensions the dataset dimensions + * @param blockSize the block size + * @param dataType the data type + * @param compression the compression + * @param codecs optional codecs (may be null) + * @throws N5Exception the exception + */ + default void createDataset( + final String datasetPath, + final long[] dimensions, + final int[] blockSize, + final DataType dataType, + final Compression compression, + final Codec[] codecs) throws N5Exception { + + createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression, codecs)); + } + + /** + * Creates a dataset. This does not create any data but the path and + * mandatory attributes only. * * @param datasetPath dataset path * @param dimensions the dataset dimensions @@ -225,7 +249,7 @@ default void createDataset( final DataType dataType, final Compression compression) throws N5Exception { - createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression)); + createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression, null)); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java new file mode 100644 index 000000000..787308493 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java @@ -0,0 +1,178 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.function.BiConsumer; + +import org.janelia.saalfeldlab.n5.DataType; + + +public class AsTypeCodec implements Codec { + + private static final long serialVersionUID = 1031322606191894484L; + + protected transient final int numBytes; + protected transient final int numEncodedBytes; + + protected transient final BiConsumer encoder; + protected transient final BiConsumer decoder; + + protected final DataType type; + protected final DataType encodedType; + + public AsTypeCodec( DataType type, DataType encodedType ) + { + this.type = type; + this.encodedType = encodedType; + + numBytes = bytes(type); + numEncodedBytes = bytes(encodedType); + + // TODO fill this out + if (type == DataType.UINT8 && encodedType == DataType.UINT32) { + encoder = BYTE_TO_INT; + decoder = INT_TO_BYTE; + } else if (type == DataType.UINT32 && encodedType == DataType.UINT8) { + encoder = INT_TO_BYTE; + decoder = BYTE_TO_INT; + } else if (type == DataType.FLOAT64 && encodedType == DataType.INT8) { + encoder = DOUBLE_TO_BYTE; + decoder = BYTE_TO_DOUBLE; + } else if (type == DataType.FLOAT32 && encodedType == DataType.INT8) { + encoder = FLOAT_TO_BYTE; + decoder = BYTE_TO_FLOAT; + } else { + encoder = IDENTITY; + decoder = IDENTITY; + } + } + + @Override + public InputStream decode(InputStream in) throws IOException { + + return new FixedLengthConvertedInputStream(numEncodedBytes, numBytes, decoder, in); + } + + @Override + public OutputStream encode(OutputStream out) throws IOException { + + return new FixedLengthConvertedOutputStream(numBytes, numEncodedBytes, encoder, out); + } + + public static int bytes(DataType type) { + + switch (type) { + case UINT8: + case INT8: + return 1; + case UINT16: + case INT16: + return 2; + case UINT32: + case INT32: + case FLOAT32: + return 4; + case UINT64: + case INT64: + case FLOAT64: + return 8; + default: + return -1; + } + } + + public static final BiConsumer IDENTITY_ARR = (x, y) -> { + System.arraycopy(x, 0, y, 0, y.length); + }; + + public static final BiConsumer IDENTITY_ONE_ARR = (x, y) -> { + y[0] = x[0]; + }; + + public static final BiConsumer BYTE_TO_INT_ARR = (b, i) -> { + i[0] = 0; + i[1] = 0; + i[2] = 0; + i[3] = b[0]; + }; + + public static final BiConsumer INT_TO_BYTE_ARR = (i, b) -> { + b[0] = i[3]; + }; + + public static final BiConsumer INT_TO_FLOAT_ARR = (i, f) -> { + ByteBuffer.wrap(f).putFloat( + (float)ByteBuffer.wrap(i).getInt()); + }; + + public static final BiConsumer FLOAT_TO_INT_ARR = (f, i) -> { + ByteBuffer.wrap(i).putInt( + (int)ByteBuffer.wrap(f).getFloat()); + }; + + public static final BiConsumer INT_TO_DOUBLE_ARR = (i, f) -> { + ByteBuffer.wrap(f).putDouble( + (float)ByteBuffer.wrap(i).getInt()); + }; + + public static final BiConsumer DOUBLE_TO_INT_ARR = (f, i) -> { + ByteBuffer.wrap(i).putInt( + (int)ByteBuffer.wrap(f).getDouble()); + }; + + public static final BiConsumer IDENTITY = (x, y) -> { + for (int i = 0; i < y.capacity(); i++) + y.put(x.get()); + }; + + public static final BiConsumer IDENTITY_ONE = (x, y) -> { + y.put(x.get()); + }; + + public static final BiConsumer BYTE_TO_INT = (b, i) -> { + final byte zero = 0; + i.put(zero); + i.put(zero); + i.put(zero); + i.put(b.get()); + }; + + public static final BiConsumer INT_TO_BYTE = (i, b) -> { + b.put(i.get(3)); + }; + + public static final BiConsumer INT_TO_FLOAT = (i, f) -> { + f.putFloat((float)i.getInt()); + }; + + public static final BiConsumer FLOAT_TO_INT = (f, i) -> { + i.putInt((int)f.getFloat()); + }; + + public static final BiConsumer INT_TO_DOUBLE = (i, f) -> { + f.putDouble((float)i.getInt()); + }; + + public static final BiConsumer DOUBLE_TO_INT = (f, i) -> { + i.putInt((int)f.getDouble()); + }; + + public static final BiConsumer BYTE_TO_FLOAT = (b, f) -> { + f.putFloat((float)b.get()); + }; + + public static final BiConsumer FLOAT_TO_BYTE = (f, b) -> { + b.put((byte)f.getFloat()); + }; + + public static final BiConsumer BYTE_TO_DOUBLE = (b, d) -> { + d.putDouble((double)b.get()); + }; + + public static final BiConsumer DOUBLE_TO_BYTE = (d, b) -> { + b.put((byte)d.getDouble()); + }; + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java new file mode 100644 index 000000000..95865dd32 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -0,0 +1,35 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Serializable; + +/** + * Interface representing a filter can encode a {@link OutputStream}s when writing data, and decode + * the {@link InputStream}s when reading data. + * + * Modeled after Filters in + * Zarr. + */ +public interface Codec extends Serializable { + + /** + * Decode an {@link InputStream}. + * + * @param in + * input stream + * @return the decoded input stream + */ + public InputStream decode(InputStream in) throws IOException; + + /** + * Encode an {@link OutputStream}. + * + * @param out + * the output stream + * @return the encoded output stream + */ + public OutputStream encode(OutputStream out) throws IOException; + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java new file mode 100644 index 000000000..ecf388379 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java @@ -0,0 +1,41 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * A {@link Codec} that is composition of a collection of codecs. + */ +public class ComposedCodec implements Codec { + + private static final long serialVersionUID = 5068349140842235924L; + private final Codec[] filters; + + public ComposedCodec(final Codec... filters) { + + this.filters = filters; + } + + @Override + public InputStream decode(InputStream in) throws IOException { + + // DOCME : note that decoding is in reverse order + InputStream decoded = in; + for( int i = filters.length - 1; i >= 0; i-- ) + decoded = filters[i].decode(decoded); + + return decoded; + } + + @Override + public OutputStream encode(OutputStream out) throws IOException { + + OutputStream encoded = out; + for( int i = 0; i < filters.length; i++ ) + encoded = filters[i].encode(encoded); + + return encoded; + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java new file mode 100644 index 000000000..f9d65a87c --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java @@ -0,0 +1,71 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.function.BiConsumer; + +/* + * An {@link InputStream} that converts between two fixed-length types. + */ +public class FixedLengthConvertedInputStream extends InputStream { + + private final int numBytes; + private final int numBytesAfterDecoding; + + private final byte[] raw; + private final byte[] decoded; + + private final ByteBuffer rawBuffer; + private final ByteBuffer decodedBuffer; + + private final InputStream src; + + private BiConsumer converter; + + private int incrememntalBytesRead; + + public FixedLengthConvertedInputStream( + final int numBytes, + final int numBytesAfterDecoding, + BiConsumer converter, + final InputStream src ) { + + this.numBytes = numBytes; + this.numBytesAfterDecoding = numBytesAfterDecoding; + this.converter = converter; + + raw = new byte[numBytes]; + decoded = new byte[numBytesAfterDecoding]; + incrememntalBytesRead = 0; + + rawBuffer = ByteBuffer.wrap(raw); + decodedBuffer = ByteBuffer.wrap(decoded); + + this.src = src; + } + + @Override + public int read() throws IOException { + + // TODO not sure if this always reads enough bytes + // int n = src.read(toEncode); + if (incrememntalBytesRead == 0) { + + rawBuffer.rewind(); + decodedBuffer.rewind(); + + for (int i = 0; i < numBytes; i++) + raw[i] = (byte)src.read(); + + converter.accept(rawBuffer, decodedBuffer); + } + + final int out = decoded[incrememntalBytesRead++]; + if (incrememntalBytesRead == numBytesAfterDecoding) + incrememntalBytesRead = 0; + + return out; + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedOutputStream.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedOutputStream.java new file mode 100644 index 000000000..87544fc79 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedOutputStream.java @@ -0,0 +1,64 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.function.BiConsumer; + +/* + * An {@link OutputStream} that converts between two fixed-length types. + */ +public class FixedLengthConvertedOutputStream extends OutputStream { + + private final int numBytes; + + private final byte[] raw; + private final byte[] encoded; + + private final ByteBuffer rawBuffer; + private final ByteBuffer encodedBuffer; + + private final OutputStream src; + + private BiConsumer converter; + + private int incrememntalBytesWritten; + + public FixedLengthConvertedOutputStream( + final int numBytes, + final int numBytesAfterEncoding, + final BiConsumer converter, + final OutputStream src ) { + + this.numBytes = numBytes; + this.converter = converter; + + raw = new byte[numBytes]; + encoded = new byte[numBytesAfterEncoding]; + + rawBuffer = ByteBuffer.wrap(raw); + encodedBuffer = ByteBuffer.wrap(encoded); + + incrememntalBytesWritten = 0; + + this.src = src; + } + + @Override + public void write(int b) throws IOException { + + raw[incrememntalBytesWritten++] = (byte)b; + + // write out the encoded bytes after writing numBytes bytes + if (incrememntalBytesWritten == numBytes) { + + rawBuffer.rewind(); + encodedBuffer.rewind(); + + converter.accept(rawBuffer, encodedBuffer); + src.write(encoded); + incrememntalBytesWritten = 0; + } + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java new file mode 100644 index 000000000..955be2356 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java @@ -0,0 +1,55 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.function.BiConsumer; + +import org.janelia.saalfeldlab.n5.DataType; + +public class FixedScaleOffsetCodec extends AsTypeCodec { + + private static final long serialVersionUID = 8024945290803548528L; + + @SuppressWarnings("unused") + private final double scale; + + @SuppressWarnings("unused") + private final double offset; + + public transient final BiConsumer encoder; + public transient final BiConsumer decoder; + + public FixedScaleOffsetCodec(final double scale, final double offset, DataType type, DataType encodedType) { + + super(type, encodedType); + this.scale = scale; + this.offset = offset; + + encoder = (f, i) -> { + final double in = f.getDouble(); + final byte res = (byte)(scale * in + offset); + i.put((byte)(scale * in + offset)); + }; + + decoder = (i, f) -> { + final byte in = i.get(); + final double conv = (((double)in) - offset) / scale; + f.putDouble(conv); + }; + } + + @Override + public InputStream decode(InputStream in) throws IOException { + + return new FixedLengthConvertedInputStream(numEncodedBytes, numBytes, this.decoder, in); + } + + @Override + public OutputStream encode(OutputStream out) throws IOException { + + return new FixedLengthConvertedOutputStream(numBytes, numEncodedBytes, this.encoder, out); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java new file mode 100644 index 000000000..83e3925c3 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -0,0 +1,23 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public class IdentityCodec implements Codec { + + private static final long serialVersionUID = 8354269325800855621L; + + @Override + public InputStream decode(InputStream in) throws IOException { + + return in; + } + + @Override + public OutputStream encode(OutputStream out) throws IOException { + + return out; + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java new file mode 100644 index 000000000..8cd38d830 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java @@ -0,0 +1,80 @@ +package org.janelia.saalfeldlab.n5.codec; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.stream.IntStream; + +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.junit.Test; + +public class AsTypeTests { + + @Test + public void testInt2Byte() throws IOException { + + final int N = 16; + final int[] ints = IntStream.rangeClosed(0, N).toArray(); + final ByteBuffer encodedInts = ByteBuffer.allocate(Integer.BYTES * N); + final byte[] bytes = new byte[N]; + for (int i = 0; i < N; i++) { + + bytes[i] = (byte)ints[i]; + encodedInts.putInt(ints[i]); + } + + final AsTypeCodec int2Byte = new AsTypeCodec(DataType.UINT32, DataType.UINT8); + testEncoding( int2Byte, bytes, encodedInts.array()); + testDecoding( int2Byte, encodedInts.array(), bytes); + + final AsTypeCodec byte2Int = new AsTypeCodec(DataType.UINT8, DataType.UINT32); + testEncoding( byte2Int, encodedInts.array(), bytes); + testDecoding( byte2Int, bytes, encodedInts.array()); + } + + @Test + public void testDouble2Byte() throws IOException { + + final int N = 16; + final double[] doubles = new double[N]; + final byte[] bytes = new byte[N]; + final ByteBuffer encodedDoubles = ByteBuffer.allocate(Double.BYTES * N); + for (int i = 0; i < N; i++) { + doubles[i] = i; + encodedDoubles.putDouble(doubles[i]); + + bytes[i] = (byte)i; + } + + final AsTypeCodec double2Byte = new AsTypeCodec(DataType.FLOAT64, DataType.INT8); + testEncoding(double2Byte, bytes, encodedDoubles.array()); + testDecoding(double2Byte, encodedDoubles.array(), bytes); + } + + protected static void testDecoding( final Codec codec, final byte[] expected, final byte[] input ) throws IOException + { + final InputStream result = codec.decode(new ByteArrayInputStream(input)); + for (int i = 0; i < expected.length; i++) + assertEquals(expected[i], (byte)result.read()); + } + + protected static void testEncoding( final Codec codec, final byte[] expected, final byte[] data ) throws IOException + { + + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(expected.length); + final OutputStream encodedStream = codec.encode(outputStream); + encodedStream.write(data); + encodedStream.flush(); + assertArrayEquals(expected, outputStream.toByteArray()); + encodedStream.close(); + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedInputStreamTest.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedInputStreamTest.java new file mode 100644 index 000000000..e23ab07d8 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedInputStreamTest.java @@ -0,0 +1,88 @@ +package org.janelia.saalfeldlab.n5.codec; + +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.stream.IntStream; + +import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; +import org.janelia.saalfeldlab.n5.codec.FixedLengthConvertedInputStream; +import org.junit.Test; + +public class FixedConvertedInputStreamTest { + + @Test + public void testLengthOne() throws IOException + { + + final byte expected = 5; + final byte[] data = new byte[32]; + Arrays.fill(data, expected); + + final FixedLengthConvertedInputStream convertedId = new FixedLengthConvertedInputStream(1, 1, + AsTypeCodec.IDENTITY_ONE, + new ByteArrayInputStream(data)); + + final FixedLengthConvertedInputStream convertedPlusOne = new FixedLengthConvertedInputStream(1, 1, + (x, y) -> { + y.put((byte)(x.get() + 1)); + }, + new ByteArrayInputStream(data)); + + for (int i = 0; i < 32; i++) { + assertEquals(expected, convertedId.read()); + assertEquals(expected + 1, convertedPlusOne.read()); + } + + convertedId.close(); + convertedPlusOne.close(); + } + + @Test + public void testIntToByte() throws IOException + { + + final int N = 16; + final ByteBuffer buf = ByteBuffer.allocate(Integer.BYTES * N); + IntStream.range(0, N).forEach( x -> { + buf.putInt(x); + }); + + final byte[] data = buf.array(); + final FixedLengthConvertedInputStream intToByte = new FixedLengthConvertedInputStream( + 4, 1, + AsTypeCodec.INT_TO_BYTE, + new ByteArrayInputStream(data)); + + for( int i = 0; i < N; i++ ) + assertEquals((byte)i, intToByte.read()); + + intToByte.close(); + } + + @Test + public void testByteToInt() throws IOException + { + + final int N = 16; + final byte[] data = new byte[16]; + for( int i = 0; i < N; i++ ) + data[i] = (byte)i; + + final FixedLengthConvertedInputStream byteToInt = new FixedLengthConvertedInputStream( + 1, 4, AsTypeCodec.BYTE_TO_INT, + new ByteArrayInputStream(data)); + + final DataInputStream dataStream = new DataInputStream(byteToInt); + for( int i = 0; i < N; i++ ) + assertEquals(i, dataStream.readInt()); + + dataStream.close(); + byteToInt.close(); + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java new file mode 100644 index 000000000..a457dfb1b --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java @@ -0,0 +1,111 @@ +package org.janelia.saalfeldlab.n5.codec; + +import static org.junit.Assert.assertArrayEquals; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.stream.IntStream; + +import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; +import org.janelia.saalfeldlab.n5.codec.FixedLengthConvertedOutputStream; +import org.junit.Test; + +public class FixedConvertedOutputStreamTest { + + @Test + public void testLengthOne() throws IOException + { + final int N = 2; + final byte expected = 5; + final byte expectedPlusOne = 6; + final byte[] expectedData = new byte[N]; + Arrays.fill(expectedData, expected); + + final byte[] expectedPlusOneData = new byte[N]; + Arrays.fill(expectedPlusOneData, expectedPlusOne); + + final ByteArrayOutputStream outId = new ByteArrayOutputStream(N); + final FixedLengthConvertedOutputStream convertedId = new FixedLengthConvertedOutputStream(1, 1, + AsTypeCodec.IDENTITY_ONE, + outId); + + convertedId.write(expectedData); + convertedId.flush(); + convertedId.close(); + + assertArrayEquals(expectedData, outId.toByteArray()); + + + final ByteArrayOutputStream outPlusOne = new ByteArrayOutputStream(N); + final FixedLengthConvertedOutputStream convertedPlusOne = new FixedLengthConvertedOutputStream(1, 1, + (x, y) -> { + y.put((byte)(x.get() + 1)); + }, + outPlusOne); + + convertedPlusOne.write(expectedData); + convertedPlusOne.close(); + assertArrayEquals(expectedPlusOneData, outPlusOne.toByteArray()); + } + + @Test + public void testIntToByte() throws IOException + { + + final int N = 16; + final ByteBuffer buf = ByteBuffer.allocate(Integer.BYTES * N); + IntStream.range(0, N).forEach( x -> { + buf.putInt(x); + }); + + final ByteBuffer expected = ByteBuffer.allocate(N); + IntStream.range(0, N).forEach( x -> { + expected.put((byte)x); + }); + + final ByteArrayOutputStream outStream = new ByteArrayOutputStream(N); + final FixedLengthConvertedOutputStream intToByte = new FixedLengthConvertedOutputStream( + 4, 1, + AsTypeCodec.INT_TO_BYTE, + outStream); + + intToByte.write(buf.array()); + intToByte.close(); + + System.out.println(Arrays.toString(buf.array())); + System.out.println(Arrays.toString(expected.array())); + System.out.println(Arrays.toString(outStream.toByteArray())); +// +// assertArrayEquals(expected.array(), outStream.toByteArray()); + } +// +// @Test +// public void testByteToInt() throws IOException +// { +// +// final int N = 16; +// final byte[] data = new byte[16]; +// for( int i = 0; i < N; i++ ) +// data[i] = (byte)i; +// +// FixedLengthConvertedInputStream byteToInt = new FixedLengthConvertedInputStream( +// 1, 4, +// (x, y) -> { +// y[0] = 0; // the setting to zero is not strictly necessary in this case +// y[1] = 0; +// y[2] = 0; +// y[3] = x[0]; +// }, +// new ByteArrayInputStream(data)); +// +// final DataInputStream dataStream = new DataInputStream(byteToInt); +// for( int i = 0; i < N; i++ ) +// assertEquals(i, dataStream.readInt()); +// +// dataStream.close(); +// byteToInt.close(); +// } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java new file mode 100644 index 000000000..098d8ecee --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java @@ -0,0 +1,62 @@ +package org.janelia.saalfeldlab.n5.codec; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.stream.DoubleStream; + +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.FixedScaleOffsetCodec; +import org.junit.Test; + +public class FixedScaleOffsetTests { + + @Test + public void testDouble2Byte() throws IOException { + + final int N = 16; + final double[] doubles = DoubleStream.iterate(0.0, x -> x + 1).limit(N).toArray(); + final ByteBuffer encodedDoubles = ByteBuffer.allocate(Double.BYTES * N); + final byte[] bytes = new byte[N]; + + final double scale = 2; + final double offset = 1; + + for (int i = 0; i < N; i++) { + final double val = (scale * doubles[i] + offset); + bytes[i] = (byte)val; + encodedDoubles.putDouble(i); + } + + final FixedScaleOffsetCodec double2Byte = new FixedScaleOffsetCodec(scale, offset, DataType.FLOAT64, DataType.UINT8); + testEncoding(double2Byte, bytes, encodedDoubles.array()); + testDecoding(double2Byte, encodedDoubles.array(), bytes); + } + + protected static void testDecoding( final Codec codec, final byte[] expected, final byte[] input ) throws IOException + { + final InputStream result = codec.decode(new ByteArrayInputStream(input)); + for (int i = 0; i < expected.length; i++) + assertEquals(expected[i], (byte)result.read()); + } + + protected static void testEncoding( final Codec codec, final byte[] expected, final byte[] data ) throws IOException + { + + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(expected.length); + final OutputStream encodedStream = codec.encode(outputStream); + encodedStream.write(data); + encodedStream.flush(); + final byte[] convertedArr = outputStream.toByteArray(); + assertArrayEquals( expected, outputStream.toByteArray()); + encodedStream.close(); + } + +} From 14d3b696ae9c399f4ff0cab45c1dcd77ab55a1e7 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 28 May 2024 15:23:22 -0400 Subject: [PATCH 002/423] feat(wip): reading and writing blocks uses codecs --- .../saalfeldlab/n5/Bzip2Compression.java | 16 +- .../janelia/saalfeldlab/n5/CodecAdapter.java | 92 +++++ .../janelia/saalfeldlab/n5/Compression.java | 64 ++++ .../saalfeldlab/n5/DatasetAttributes.java | 48 +-- .../saalfeldlab/n5/DefaultBlockReader.java | 52 +++ .../saalfeldlab/n5/DefaultBlockWriter.java | 51 +++ .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 2 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 5 +- .../janelia/saalfeldlab/n5/GsonN5Reader.java | 5 +- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 3 + .../saalfeldlab/n5/GzipCompression.java | 17 +- .../saalfeldlab/n5/Lz4Compression.java | 16 +- .../saalfeldlab/n5/RawCompression.java | 17 +- .../janelia/saalfeldlab/n5/XzCompression.java | 16 +- .../saalfeldlab/n5/codec/AsTypeCodec.java | 358 ++++++++++++++---- .../janelia/saalfeldlab/n5/codec/Codec.java | 2 + .../saalfeldlab/n5/codec/ComposedCodec.java | 14 +- .../n5/codec/FixedScaleOffsetCodec.java | 82 +++- .../saalfeldlab/n5/codec/IdentityCodec.java | 8 + .../saalfeldlab/n5/codec/AsTypeTests.java | 53 ++- .../codec/FixedConvertedInputStreamTest.java | 8 +- .../codec/FixedConvertedOutputStreamTest.java | 6 +- .../n5/codec/FixedScaleOffsetTests.java | 52 ++- 23 files changed, 789 insertions(+), 198 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 5d3d61614..0c40d01a8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -52,17 +52,29 @@ public Bzip2Compression() { } @Override - public InputStream getInputStream(final InputStream in) throws IOException { + public InputStream decode(final InputStream in) throws IOException { return new BZip2CompressorInputStream(in); } @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { + public InputStream getInputStream(final InputStream in) throws IOException { + + return decode(in); + } + + @Override + public OutputStream encode(final OutputStream out) throws IOException { return new BZip2CompressorOutputStream(out, blockSize); } + @Override + public OutputStream getOutputStream(final OutputStream out) throws IOException { + + return encode(out); + } + @Override public Bzip2Compression getReader() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java new file mode 100644 index 000000000..0933433e4 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java @@ -0,0 +1,92 @@ +/** + * Copyright (c) 2017, Stephan Saalfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.lang.reflect.Type; + +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.FixedScaleOffsetCodec; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +public class CodecAdapter implements JsonDeserializer, JsonSerializer { + + @Override + public JsonElement serialize( + final Codec codec, + final Type typeOfSrc, + final JsonSerializationContext context) { + + if (codec.getId().equals(FixedScaleOffsetCodec.FIXED_SCALE_OFFSET_CODEC_ID)) { + final FixedScaleOffsetCodec c = (FixedScaleOffsetCodec)codec; + final JsonObject obj = new JsonObject(); + obj.addProperty("id", c.getId()); + obj.addProperty("scale", c.getScale()); + obj.addProperty("offset", c.getOffset()); + obj.addProperty("type", c.getType().toString().toLowerCase()); + obj.addProperty("encodedType", c.getEncodedType().toString().toLowerCase()); + return obj; + } + + return JsonNull.INSTANCE; + } + + @Override + public Codec deserialize( + final JsonElement json, + final Type typeOfT, + final JsonDeserializationContext context) throws JsonParseException { + + if (json == null) + return null; + else if (!json.isJsonObject()) + return null; + + final JsonObject jsonObject = json.getAsJsonObject(); + if (jsonObject.has("id")) { + + final String id = jsonObject.get("id").getAsString(); + if (id.equals(FixedScaleOffsetCodec.FIXED_SCALE_OFFSET_CODEC_ID)) { + + return new FixedScaleOffsetCodec( + jsonObject.get("scale").getAsDouble(), + jsonObject.get("offset").getAsDouble(), + DataType.valueOf(jsonObject.get("type").getAsString().toUpperCase()), + DataType.valueOf(jsonObject.get("encodedType").getAsString().toUpperCase())); + } + } + + return null; + } + +} \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index df0ca49e1..3092bbe45 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -25,6 +25,9 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.io.Serializable; import java.lang.annotation.ElementType; import java.lang.annotation.Inherited; @@ -32,6 +35,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import org.janelia.saalfeldlab.n5.codec.Codec; import org.scijava.annotations.Indexable; /** @@ -41,6 +45,12 @@ */ public interface Compression extends Serializable { + // @Override + // public default String getId() { + // + // return getType(); + // } + /** * Annotation for runtime discovery of compression schemes. * @@ -72,7 +82,61 @@ public default String getType() { return compressionType.value(); } + public BlockReader getReader(); public BlockWriter getWriter(); + + /** + * Decode an {@link InputStream}. + * + * @param in + * input stream + * @return the decoded input stream + */ + public InputStream decode(InputStream in) throws IOException; + + /** + * Encode an {@link OutputStream}. + * + * @param out + * the output stream + * @return the encoded output stream + */ + public OutputStream encode(OutputStream out) throws IOException; + + public static Codec getCompressionAsCodec(Compression compression) { + + return new CompressionCodec(compression); + } + + public static class CompressionCodec implements Codec { + + private static final long serialVersionUID = -7931131454184340637L; + private Compression compression; + + public CompressionCodec(Compression compression) { + + this.compression = compression; + } + + @Override + public InputStream decode(InputStream in) throws IOException { + + return compression.decode(in); + } + + @Override + public OutputStream encode(OutputStream out) throws IOException { + + return compression.encode(out); + } + + @Override + public String getId() { + + return compression.getType(); + } + + } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 5fa19fc5a..9f67d0d06 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -1,28 +1,3 @@ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.Serializable; @@ -30,6 +5,7 @@ import java.util.HashMap; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.ComposedCodec; /** * Mandatory dataset attributes: @@ -40,7 +16,7 @@ *
  • {@link DataType} : dataType
    • *
  • {@link Compression} : compression
    • * - * + * * Optional dataset attributes: *
      *
    1. {@link Codec}[] : codecs
      2. @@ -121,6 +97,24 @@ public Codec[] getCodecs() { return codecs; } + public Codec collectCodecs() { + + final Codec compressionCodec = Compression.getCompressionAsCodec(compression); + + if (codecs == null || codecs.length == 0) + return compressionCodec; + else if (codecs.length == 1) + return new ComposedCodec(codecs[0], compressionCodec); + else { + final Codec[] codecsAndCompresor = new Codec[codecs.length + 1]; + for (int i = 0; i < codecs.length; i++) + codecsAndCompresor[i] = codecs[i]; + + codecsAndCompresor[codecs.length] = compressionCodec; + return new ComposedCodec(codecsAndCompresor); + } + } + public HashMap asMap() { final HashMap map = new HashMap<>(); @@ -128,7 +122,7 @@ public HashMap asMap() { map.put(BLOCK_SIZE_KEY, blockSize); map.put(DATA_TYPE_KEY, dataType); map.put(COMPRESSION_KEY, compression); - map.put(CODEC_KEY, codecs); // TODO : consider not adding to map when null? + map.put(CODEC_KEY, codecs); // TODO : consider not adding to map when null return map; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 58c59780c..4e5858193 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -95,4 +95,56 @@ public static DataBlock readBlock( reader.read(dataBlock, in); return dataBlock; } + + /** + * Reads a {@link DataBlock} from an {@link InputStream}. + * + * @param in + * the input stream + * @param datasetAttributes + * the dataset attributes + * @param gridPosition + * the grid position + * @return the block + * @throws IOException + * the exception + */ + public static DataBlock readBlockWithCodecs( + final InputStream in, + final DatasetAttributes datasetAttributes, + final long[] gridPosition) throws IOException { + + final DataInputStream dis = new DataInputStream(in); + final short mode = dis.readShort(); + final int numElements; + final DataBlock dataBlock; + if (mode != 2) { + final int nDim = dis.readShort(); + final int[] blockSize = new int[nDim]; + for (int d = 0; d < nDim; ++d) + blockSize[d] = dis.readInt(); + if (mode == 0) { + numElements = DataBlock.getNumElements(blockSize); + } else { + numElements = dis.readInt(); + } + dataBlock = datasetAttributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); + } else { + numElements = dis.readInt(); + dataBlock = datasetAttributes.getDataType().createDataBlock(null, gridPosition, numElements); + } + + readFromStream(dataBlock, datasetAttributes.collectCodecs().decode(in)); + return dataBlock; + } + + public static > void readFromStream(final B dataBlock, final InputStream in) throws IOException { + + final ByteBuffer buffer = dataBlock.toByteBuffer(); + final DataInputStream dis = new DataInputStream(in); + dis.readFully(buffer.array()); + dataBlock.readData(buffer); + } + + } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index c53aae2df..c5d985f92 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -95,4 +95,55 @@ else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSiz final BlockWriter writer = datasetAttributes.getCompression().getWriter(); writer.write(dataBlock, out); } + + /** + * Writes a {@link DataBlock} into an {@link OutputStream}. + * + * @param + * the type of data + * @param out + * the output stream + * @param datasetAttributes + * the dataset attributes + * @param dataBlock + * the data block the block data type + * @throws IOException + * the exception + */ + public static void writeBlockWithCodecs( + final OutputStream out, + final DatasetAttributes datasetAttributes, + final DataBlock dataBlock) throws IOException { + + final DataOutputStream dos = new DataOutputStream(out); + + final int mode; + if (datasetAttributes.getDataType() == DataType.OBJECT || dataBlock.getSize() == null) + mode = 2; + else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSize())) + mode = 0; + else + mode = 1; + dos.writeShort(mode); + + if (mode != 2) { + dos.writeShort(datasetAttributes.getNumDimensions()); + for (final int size : dataBlock.getSize()) + dos.writeInt(size); + } + + if (mode != 0) + dos.writeInt(dataBlock.getNumElements()); + + try (final OutputStream encodedStream = datasetAttributes.collectCodecs().encode(out)) { + writeFromStream(dataBlock, encodedStream); + out.flush(); + } + } + + public static void writeFromStream(final DataBlock dataBlock, final OutputStream out) throws IOException { + + final ByteBuffer buffer = dataBlock.toByteBuffer(); + out.write(buffer.array()); + } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 644aca51a..76aea228c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -98,7 +98,7 @@ default DataBlock readBlock( return null; try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(path)) { - return DefaultBlockReader.readBlock(lockedChannel.newInputStream(), datasetAttributes, gridPosition); + return DefaultBlockReader.readBlockWithCodecs(lockedChannel.newInputStream(), datasetAttributes, gridPosition); } catch (final IOException | UncheckedIOException e) { throw new N5IOException( "Failed to read block " + Arrays.toString(gridPosition) + " from dataset " + path, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 38c754be9..a784dcc6f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -31,13 +31,13 @@ import java.util.List; import java.util.Map; -import com.google.gson.JsonSyntaxException; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonNull; import com.google.gson.JsonObject; +import com.google.gson.JsonSyntaxException; /** * Default implementation of {@link N5Writer} with JSON attributes parsed with @@ -218,7 +218,8 @@ default void writeBlock( final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition()); try (final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath)) { - DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); + DefaultBlockWriter.writeBlockWithCodecs(lock.newOutputStream(), datasetAttributes, dataBlock); + // DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); } catch (final IOException | UncheckedIOException e) { throw new N5IOException( "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index be16ed080..ea7ea878b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -28,6 +28,8 @@ import java.lang.reflect.Type; import java.util.Map; +import org.janelia.saalfeldlab.n5.codec.Codec; + import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonSyntaxException; @@ -69,13 +71,14 @@ default DatasetAttributes createDatasetAttributes(final JsonElement attributes) final int[] blockSize = GsonUtils.readAttribute(attributes, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, getGson()); final Compression compression = GsonUtils.readAttribute(attributes, DatasetAttributes.COMPRESSION_KEY, Compression.class, getGson()); + final Codec[] codecs = GsonUtils.readAttribute(attributes, DatasetAttributes.CODEC_KEY, Codec[].class, getGson()); /* version 0 */ final String compressionVersion0Name = compression == null ? GsonUtils.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, getGson()) : null; - return DatasetAttributes.from(dimensions, dataType, blockSize, compression, compressionVersion0Name); + return DatasetAttributes.from(dimensions, dataType, blockSize, compression, compressionVersion0Name, codecs); } catch (JsonSyntaxException | NumberFormatException | ClassCastException e) { /* We cannot create a dataset, so return null. */ return null; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index d72307c0d..f7b988def 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -34,6 +34,8 @@ import java.util.Map; import java.util.regex.Matcher; +import org.janelia.saalfeldlab.n5.codec.Codec; + import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; @@ -54,6 +56,7 @@ static Gson registerGson(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Codec.class, new CodecAdapter()); gsonBuilder.disableHtmlEscaping(); return gsonBuilder.create(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index b691a6d3e..0b6d734d8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -68,7 +68,7 @@ public GzipCompression(final int level, final boolean useZlib) { } @Override - public InputStream getInputStream(final InputStream in) throws IOException { + public InputStream decode(InputStream in) throws IOException { if (useZlib) { return new InflaterInputStream(in); @@ -78,7 +78,13 @@ public InputStream getInputStream(final InputStream in) throws IOException { } @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { + public InputStream getInputStream(final InputStream in) throws IOException { + + return decode(in); + } + + @Override + public OutputStream encode(OutputStream out) throws IOException { if (useZlib) { return new DeflaterOutputStream(out, new Deflater(level)); @@ -88,6 +94,12 @@ public OutputStream getOutputStream(final OutputStream out) throws IOException { } } + @Override + public OutputStream getOutputStream(final OutputStream out) throws IOException { + + return encode(out); + } + @Override public GzipCompression getReader() { @@ -116,4 +128,5 @@ public boolean equals(final Object other) { return useZlib == gz.useZlib && level == gz.level; } } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index d76e4fe53..0ba88e126 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -53,17 +53,29 @@ public Lz4Compression() { } @Override - public InputStream getInputStream(final InputStream in) throws IOException { + public InputStream decode(final InputStream in) throws IOException { return new LZ4BlockInputStream(in); } @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { + public InputStream getInputStream(final InputStream in) throws IOException { + + return decode(in); + } + + @Override + public OutputStream encode(final OutputStream out) throws IOException { return new LZ4BlockOutputStream(out, blockSize); } + @Override + public OutputStream getOutputStream(final OutputStream out) throws IOException { + + return encode(out); + } + @Override public Lz4Compression getReader() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index ffa674fc4..7d1327b0d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -37,17 +37,30 @@ public class RawCompression implements DefaultBlockReader, DefaultBlockWriter, C private static final long serialVersionUID = 7526445806847086477L; @Override - public InputStream getInputStream(final InputStream in) throws IOException { + public InputStream decode(final InputStream in) throws IOException { return in; } @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { + public InputStream getInputStream(final InputStream in) throws IOException { + + return decode(in); + } + + + @Override + public OutputStream encode(final OutputStream out) throws IOException { return out; } + @Override + public OutputStream getOutputStream(final OutputStream out) throws IOException { + + return encode(out); + } + @Override public RawCompression getReader() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index 5204e799f..ed73b7d53 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -52,17 +52,29 @@ public XzCompression() { } @Override - public InputStream getInputStream(final InputStream in) throws IOException { + public InputStream decode(final InputStream in) throws IOException { return new XZCompressorInputStream(in); } @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { + public InputStream getInputStream(final InputStream in) throws IOException { + + return decode(in); + } + + @Override + public OutputStream encode(final OutputStream out) throws IOException { return new XZCompressorOutputStream(out, preset); } + @Override + public OutputStream getOutputStream(final OutputStream out) throws IOException { + + return encode(out); + } + @Override public XzCompression getReader() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java index 787308493..d4852d4a0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java @@ -22,6 +22,8 @@ public class AsTypeCodec implements Codec { protected final DataType type; protected final DataType encodedType; + protected final String id = "astype"; + public AsTypeCodec( DataType type, DataType encodedType ) { this.type = type; @@ -30,23 +32,14 @@ public AsTypeCodec( DataType type, DataType encodedType ) numBytes = bytes(type); numEncodedBytes = bytes(encodedType); - // TODO fill this out - if (type == DataType.UINT8 && encodedType == DataType.UINT32) { - encoder = BYTE_TO_INT; - decoder = INT_TO_BYTE; - } else if (type == DataType.UINT32 && encodedType == DataType.UINT8) { - encoder = INT_TO_BYTE; - decoder = BYTE_TO_INT; - } else if (type == DataType.FLOAT64 && encodedType == DataType.INT8) { - encoder = DOUBLE_TO_BYTE; - decoder = BYTE_TO_DOUBLE; - } else if (type == DataType.FLOAT32 && encodedType == DataType.INT8) { - encoder = FLOAT_TO_BYTE; - decoder = BYTE_TO_FLOAT; - } else { - encoder = IDENTITY; - decoder = IDENTITY; - } + encoder = converter(type, encodedType); + decoder = converter(encodedType, type); + } + + @Override + public String getId() { + + return id; } @Override @@ -83,96 +76,301 @@ public static int bytes(DataType type) { } } - public static final BiConsumer IDENTITY_ARR = (x, y) -> { - System.arraycopy(x, 0, y, 0, y.length); - }; - - public static final BiConsumer IDENTITY_ONE_ARR = (x, y) -> { - y[0] = x[0]; - }; + public static BiConsumer converter(final DataType from, final DataType to) { + + // // TODO fill this out + + if (from == to) + return AsTypeCodec::IDENTITY; + else if (from == DataType.INT8) { + + if( to == DataType.INT16 ) + return AsTypeCodec::BYTE_TO_SHORT; + else if( to == DataType.INT32 ) + return AsTypeCodec::BYTE_TO_INT; + else if( to == DataType.INT64 ) + return AsTypeCodec::BYTE_TO_LONG; + else if( to == DataType.FLOAT32 ) + return AsTypeCodec::BYTE_TO_FLOAT; + else if( to == DataType.FLOAT64 ) + return AsTypeCodec::BYTE_TO_DOUBLE; + + } else if (from == DataType.INT16) { + + if (to == DataType.INT8) + return AsTypeCodec::SHORT_TO_BYTE; + else if (to == DataType.INT32) + return AsTypeCodec::SHORT_TO_INT; + else if (to == DataType.INT64) + return AsTypeCodec::SHORT_TO_LONG; + else if (to == DataType.FLOAT32) + return AsTypeCodec::SHORT_TO_FLOAT; + else if (to == DataType.FLOAT64) + return AsTypeCodec::SHORT_TO_DOUBLE; + + } else if (from == DataType.INT32) { + + if (to == DataType.INT8) + return AsTypeCodec::INT_TO_BYTE; + else if (to == DataType.INT16) + return AsTypeCodec::INT_TO_SHORT; + if (to == DataType.INT8) + return AsTypeCodec::DOUBLE_TO_BYTE; + else if (to == DataType.INT16) + return AsTypeCodec::DOUBLE_TO_SHORT; + else if (to == DataType.INT32) + return AsTypeCodec::DOUBLE_TO_INT; + else if (to == DataType.INT64) + return AsTypeCodec::DOUBLE_TO_LONG; + else if (to == DataType.FLOAT32) + return AsTypeCodec::DOUBLE_TO_FLOAT; + else if (to == DataType.INT64) + return AsTypeCodec::INT_TO_LONG; + else if (to == DataType.FLOAT32) + return AsTypeCodec::INT_TO_FLOAT; + else if (to == DataType.FLOAT64) + return AsTypeCodec::INT_TO_DOUBLE; + + } else if (from == DataType.INT64) { + + if (to == DataType.INT8) + return AsTypeCodec::LONG_TO_BYTE; + else if (to == DataType.INT16) + return AsTypeCodec::LONG_TO_SHORT; + else if (to == DataType.INT32) + return AsTypeCodec::LONG_TO_INT; + else if (to == DataType.FLOAT32) + return AsTypeCodec::LONG_TO_FLOAT; + else if (to == DataType.FLOAT64) + return AsTypeCodec::LONG_TO_DOUBLE; + + } else if (from == DataType.FLOAT32) { + + if (to == DataType.INT8) + return AsTypeCodec::FLOAT_TO_BYTE; + else if (to == DataType.INT16) + return AsTypeCodec::FLOAT_TO_SHORT; + else if (to == DataType.INT32) + return AsTypeCodec::FLOAT_TO_INT; + else if (to == DataType.INT64) + return AsTypeCodec::FLOAT_TO_LONG; + else if (to == DataType.FLOAT64) + return AsTypeCodec::FLOAT_TO_DOUBLE; + + } else if (from == DataType.FLOAT64) { + + if (to == DataType.INT8) + return AsTypeCodec::DOUBLE_TO_BYTE; + else if (to == DataType.INT16) + return AsTypeCodec::DOUBLE_TO_SHORT; + else if (to == DataType.INT32) + return AsTypeCodec::DOUBLE_TO_INT; + else if (to == DataType.INT64) + return AsTypeCodec::DOUBLE_TO_LONG; + else if (to == DataType.FLOAT32) + return AsTypeCodec::DOUBLE_TO_FLOAT; + } - public static final BiConsumer BYTE_TO_INT_ARR = (b, i) -> { - i[0] = 0; - i[1] = 0; - i[2] = 0; - i[3] = b[0]; - }; + return AsTypeCodec::IDENTITY; + } - public static final BiConsumer INT_TO_BYTE_ARR = (i, b) -> { - b[0] = i[3]; - }; + public static final void IDENTITY(final ByteBuffer x, final ByteBuffer y) { - public static final BiConsumer INT_TO_FLOAT_ARR = (i, f) -> { - ByteBuffer.wrap(f).putFloat( - (float)ByteBuffer.wrap(i).getInt()); - }; + for (int i = 0; i < y.capacity(); i++) + y.put(x.get()); + } - public static final BiConsumer FLOAT_TO_INT_ARR = (f, i) -> { - ByteBuffer.wrap(i).putInt( - (int)ByteBuffer.wrap(f).getFloat()); - }; + public static final void IDENTITY_ONE(final ByteBuffer x, final ByteBuffer y) { - public static final BiConsumer INT_TO_DOUBLE_ARR = (i, f) -> { - ByteBuffer.wrap(f).putDouble( - (float)ByteBuffer.wrap(i).getInt()); - }; + y.put(x.get()); + } - public static final BiConsumer DOUBLE_TO_INT_ARR = (f, i) -> { - ByteBuffer.wrap(i).putInt( - (int)ByteBuffer.wrap(f).getDouble()); - }; + public static final void BYTE_TO_SHORT(final ByteBuffer b, final ByteBuffer s) { - public static final BiConsumer IDENTITY = (x, y) -> { - for (int i = 0; i < y.capacity(); i++) - y.put(x.get()); - }; + final byte zero = 0; + s.put(zero); + s.put(b.get()); + } - public static final BiConsumer IDENTITY_ONE = (x, y) -> { - y.put(x.get()); - }; + public static final void BYTE_TO_INT(final ByteBuffer b, final ByteBuffer i) { - public static final BiConsumer BYTE_TO_INT = (b, i) -> { final byte zero = 0; i.put(zero); i.put(zero); i.put(zero); i.put(b.get()); - }; + } + + public static final void BYTE_TO_LONG(final ByteBuffer b, final ByteBuffer l) { + + final byte zero = 0; + l.put(zero); + l.put(zero); + l.put(zero); + l.put(zero); + l.put(zero); + l.put(zero); + l.put(zero); + l.put(b.get()); + } + + public static final void BYTE_TO_FLOAT(final ByteBuffer b, final ByteBuffer f) { + + f.putFloat((float)b.get()); + } + + public static final void BYTE_TO_DOUBLE(final ByteBuffer b, final ByteBuffer d) { + + d.putDouble((double)b.get()); + } + + public static final void SHORT_TO_BYTE(final ByteBuffer s, final ByteBuffer b) { + + final byte zero = 0; + b.put(zero); + b.put(s.get()); + } + + public static final void SHORT_TO_INT(final ByteBuffer s, final ByteBuffer i) { + + final byte zero = 0; + i.put(zero); + i.put(zero); + i.put(s.get()); + i.put(s.get()); + } + + public static final void SHORT_TO_LONG(final ByteBuffer s, final ByteBuffer l) { + + final byte zero = 0; + l.put(zero); + l.put(zero); + l.put(zero); + l.put(zero); + l.put(zero); + l.put(zero); + l.put(s.get()); + l.put(s.get()); + } + + public static final void SHORT_TO_FLOAT(final ByteBuffer s, final ByteBuffer f) { + + f.putFloat((float)s.getShort()); + } + + public static final void SHORT_TO_DOUBLE(final ByteBuffer s, final ByteBuffer d) { + + d.putDouble((double)s.getShort()); + } + + public static final void INT_TO_BYTE(final ByteBuffer i, final ByteBuffer b) { - public static final BiConsumer INT_TO_BYTE = (i, b) -> { b.put(i.get(3)); - }; + } + + public static final void INT_TO_SHORT(final ByteBuffer i, final ByteBuffer s) { + + s.put(i.get(2)); + s.put(i.get(3)); + } + + public static final void INT_TO_LONG(final ByteBuffer i, final ByteBuffer l) { + + final byte zero = 0; + l.put(zero); + l.put(zero); + l.put(zero); + l.put(zero); + l.put(i.get()); + l.put(i.get()); + l.put(i.get()); + l.put(i.get()); + } + + public static final void INT_TO_FLOAT(final ByteBuffer i, final ByteBuffer f) { - public static final BiConsumer INT_TO_FLOAT = (i, f) -> { f.putFloat((float)i.getInt()); - }; + } - public static final BiConsumer FLOAT_TO_INT = (f, i) -> { - i.putInt((int)f.getFloat()); - }; + public static final void INT_TO_DOUBLE(final ByteBuffer i, final ByteBuffer f) { - public static final BiConsumer INT_TO_DOUBLE = (i, f) -> { f.putDouble((float)i.getInt()); - }; + } - public static final BiConsumer DOUBLE_TO_INT = (f, i) -> { - i.putInt((int)f.getDouble()); - }; + public static final void LONG_TO_BYTE(final ByteBuffer l, final ByteBuffer b) { - public static final BiConsumer BYTE_TO_FLOAT = (b, f) -> { - f.putFloat((float)b.get()); - }; + b.put((byte)l.getLong()); + } + + public static final void LONG_TO_SHORT(final ByteBuffer l, final ByteBuffer s) { + + s.putShort((short)l.getLong()); + } + + public static final void LONG_TO_INT(final ByteBuffer l, final ByteBuffer i) { + + i.putInt((int)l.getLong()); + } + + public static final void LONG_TO_FLOAT(final ByteBuffer l, final ByteBuffer f) { + + f.putFloat((float)l.getLong()); + } + + public static final void LONG_TO_DOUBLE(final ByteBuffer l, final ByteBuffer f) { + + f.putDouble((float)l.getLong()); + } + + public static final void FLOAT_TO_BYTE(final ByteBuffer f, final ByteBuffer b) { - public static final BiConsumer FLOAT_TO_BYTE = (f, b) -> { b.put((byte)f.getFloat()); - }; + } - public static final BiConsumer BYTE_TO_DOUBLE = (b, d) -> { - d.putDouble((double)b.get()); - }; + public static final void FLOAT_TO_SHORT(final ByteBuffer f, final ByteBuffer s) { + + s.putShort((short)f.getFloat()); + } + + public static final void FLOAT_TO_INT(final ByteBuffer f, final ByteBuffer i) { + + i.putInt((int)f.getFloat()); + } + + public static final void FLOAT_TO_LONG(final ByteBuffer f, final ByteBuffer l) { + + l.putLong((long)f.getFloat()); + } + + public static final void FLOAT_TO_DOUBLE(final ByteBuffer f, final ByteBuffer d) { + + d.putDouble((double)f.getFloat()); + } + + public static final void DOUBLE_TO_BYTE(final ByteBuffer d, final ByteBuffer b) { - public static final BiConsumer DOUBLE_TO_BYTE = (d, b) -> { b.put((byte)d.getDouble()); - }; + } + + public static final void DOUBLE_TO_SHORT(final ByteBuffer d, final ByteBuffer s) { + + s.putShort((short)d.getDouble()); + } + + public static final void DOUBLE_TO_INT(final ByteBuffer d, final ByteBuffer i) { + + i.putInt((int)d.getDouble()); + } + + public static final void DOUBLE_TO_LONG(final ByteBuffer d, final ByteBuffer l) { + + l.putLong((long)d.getDouble()); + } + + public static final void DOUBLE_TO_FLOAT(final ByteBuffer d, final ByteBuffer f) { + + f.putFloat((float)d.getDouble()); + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index 95865dd32..a2e5373f0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -32,4 +32,6 @@ public interface Codec extends Serializable { */ public OutputStream encode(OutputStream out) throws IOException; + public String getId(); + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java index ecf388379..49e6f923d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java @@ -12,17 +12,25 @@ public class ComposedCodec implements Codec { private static final long serialVersionUID = 5068349140842235924L; private final Codec[] filters; + protected String id = "composed"; + public ComposedCodec(final Codec... filters) { this.filters = filters; } + @Override + public String getId() { + + return id; + } + @Override public InputStream decode(InputStream in) throws IOException { - // DOCME : note that decoding is in reverse order + // note that decoding is in reverse order InputStream decoded = in; - for( int i = filters.length - 1; i >= 0; i-- ) + for (int i = filters.length - 1; i >= 0; i--) decoded = filters[i].decode(decoded); return decoded; @@ -32,7 +40,7 @@ public InputStream decode(InputStream in) throws IOException { public OutputStream encode(OutputStream out) throws IOException { OutputStream encoded = out; - for( int i = 0; i < filters.length; i++ ) + for (int i = 0; i < filters.length; i++) encoded = filters[i].encode(encoded); return encoded; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java index 955be2356..34b06ecdf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java @@ -12,14 +12,25 @@ public class FixedScaleOffsetCodec extends AsTypeCodec { private static final long serialVersionUID = 8024945290803548528L; + public static transient final String FIXED_SCALE_OFFSET_CODEC_ID = "fixedscaleoffset"; + @SuppressWarnings("unused") private final double scale; @SuppressWarnings("unused") private final double offset; + protected final String id = FIXED_SCALE_OFFSET_CODEC_ID; + + private transient final ByteBuffer tmpEncoder; + private transient final ByteBuffer tmpDecoder; + public transient final BiConsumer encoder; + public transient final BiConsumer encoderPre; + public transient final BiConsumer encoderPost; public transient final BiConsumer decoder; + public transient final BiConsumer decoderPre; + public transient final BiConsumer decoderPost; public FixedScaleOffsetCodec(final double scale, final double offset, DataType type, DataType encodedType) { @@ -27,19 +38,74 @@ public FixedScaleOffsetCodec(final double scale, final double offset, DataType t this.scale = scale; this.offset = offset; - encoder = (f, i) -> { - final double in = f.getDouble(); - final byte res = (byte)(scale * in + offset); - i.put((byte)(scale * in + offset)); + tmpEncoder = ByteBuffer.wrap(new byte[Double.BYTES]); + tmpDecoder = ByteBuffer.wrap(new byte[Double.BYTES]); + + // encoder goes from type to encoded type + encoderPre = converter(type, DataType.FLOAT64); + encoderPost = converter(DataType.FLOAT64, encodedType); + + // decoder goes from encoded type to type + decoderPre = converter(encodedType, DataType.FLOAT64); + decoderPost = converter(DataType.FLOAT64, type); + + // convert from i type to double, apply scale and offset, then convert to type o + encoder = (i, o) -> { + tmpEncoder.rewind(); + encoderPre.accept(i, tmpEncoder); + tmpEncoder.rewind(); + final double x = tmpEncoder.getDouble(); + final double y = scale * x + offset; + System.out.println("encode: " + y); + tmpEncoder.rewind(); + tmpEncoder.putDouble(scale * x + offset); + tmpEncoder.rewind(); + encoderPost.accept(tmpEncoder, o); }; - decoder = (i, f) -> { - final byte in = i.get(); - final double conv = (((double)in) - offset) / scale; - f.putDouble(conv); + // convert from i type to double, apply scale and offset, then convert to type o + decoder = (i, o) -> { + // System.out.println("decode"); + // System.out.println(i.capacity()); + // System.out.println(tmpDecoder.capacity()); + // System.out.println(o.capacity()); + tmpDecoder.rewind(); + decoderPre.accept(i, tmpDecoder); + tmpDecoder.rewind(); + final double x = tmpDecoder.getDouble(); + tmpDecoder.rewind(); + tmpDecoder.putDouble((x - offset) / scale); + tmpDecoder.rewind(); + decoderPost.accept(tmpDecoder, o); }; } + public double getScale() { + + return scale; + } + + public double getOffset() { + + return offset; + } + + public DataType getType() { + + return super.type; + } + + public DataType getEncodedType() { + + return encodedType; + } + + @Override + public String getId() { + + return id; + } + @Override public InputStream decode(InputStream in) throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java index 83e3925c3..5164c5474 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -8,6 +8,14 @@ public class IdentityCodec implements Codec { private static final long serialVersionUID = 8354269325800855621L; + protected final String id = "id"; + + @Override + public String getId() { + + return id; + } + @Override public InputStream decode(InputStream in) throws IOException { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java index 8cd38d830..9aeeaebc6 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java @@ -9,11 +9,8 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.stream.IntStream; import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; -import org.janelia.saalfeldlab.n5.codec.Codec; import org.junit.Test; public class AsTypeTests { @@ -22,52 +19,48 @@ public class AsTypeTests { public void testInt2Byte() throws IOException { final int N = 16; - final int[] ints = IntStream.rangeClosed(0, N).toArray(); - final ByteBuffer encodedInts = ByteBuffer.allocate(Integer.BYTES * N); - final byte[] bytes = new byte[N]; + final ByteBuffer intsAsBuffer = ByteBuffer.allocate(Integer.BYTES * N); + final byte[] encodedBytes = new byte[N]; for (int i = 0; i < N; i++) { - - bytes[i] = (byte)ints[i]; - encodedInts.putInt(ints[i]); + intsAsBuffer.putInt(i); + encodedBytes[i] = (byte)i; } - final AsTypeCodec int2Byte = new AsTypeCodec(DataType.UINT32, DataType.UINT8); - testEncoding( int2Byte, bytes, encodedInts.array()); - testDecoding( int2Byte, encodedInts.array(), bytes); - - final AsTypeCodec byte2Int = new AsTypeCodec(DataType.UINT8, DataType.UINT32); - testEncoding( byte2Int, encodedInts.array(), bytes); - testDecoding( byte2Int, bytes, encodedInts.array()); + final byte[] decodedInts = intsAsBuffer.array(); + testEncodingAndDecoding(new AsTypeCodec(DataType.INT32, DataType.INT8), encodedBytes, decodedInts); + testEncodingAndDecoding(new AsTypeCodec(DataType.INT8, DataType.INT32), decodedInts, encodedBytes); } @Test public void testDouble2Byte() throws IOException { final int N = 16; - final double[] doubles = new double[N]; - final byte[] bytes = new byte[N]; - final ByteBuffer encodedDoubles = ByteBuffer.allocate(Double.BYTES * N); + final ByteBuffer doublesAsBuffer = ByteBuffer.allocate(Double.BYTES * N); + final byte[] encodedBytes = new byte[N]; for (int i = 0; i < N; i++) { - doubles[i] = i; - encodedDoubles.putDouble(doubles[i]); - - bytes[i] = (byte)i; + doublesAsBuffer.putDouble(i); + encodedBytes[i] = (byte)i; } + final byte[] decodedDoubles = doublesAsBuffer.array(); - final AsTypeCodec double2Byte = new AsTypeCodec(DataType.FLOAT64, DataType.INT8); - testEncoding(double2Byte, bytes, encodedDoubles.array()); - testDecoding(double2Byte, encodedDoubles.array(), bytes); + testEncodingAndDecoding(new AsTypeCodec(DataType.FLOAT64, DataType.INT8), encodedBytes, decodedDoubles); + testEncodingAndDecoding(new AsTypeCodec(DataType.INT8, DataType.FLOAT64), decodedDoubles, encodedBytes); } - protected static void testDecoding( final Codec codec, final byte[] expected, final byte[] input ) throws IOException - { + public static void testEncodingAndDecoding(Codec codec, byte[] encodedBytes, byte[] decodedBytes) throws IOException { + + testEncoding(codec, encodedBytes, decodedBytes); + testDecoding(codec, decodedBytes, encodedBytes); + } + + public static void testDecoding(final Codec codec, final byte[] expected, final byte[] input) throws IOException { + final InputStream result = codec.decode(new ByteArrayInputStream(input)); for (int i = 0; i < expected.length; i++) assertEquals(expected[i], (byte)result.read()); } - protected static void testEncoding( final Codec codec, final byte[] expected, final byte[] data ) throws IOException - { + public static void testEncoding(final Codec codec, final byte[] expected, final byte[] data) throws IOException { final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(expected.length); final OutputStream encodedStream = codec.encode(outputStream); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedInputStreamTest.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedInputStreamTest.java index e23ab07d8..27c744fa4 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedInputStreamTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedInputStreamTest.java @@ -9,8 +9,6 @@ import java.util.Arrays; import java.util.stream.IntStream; -import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; -import org.janelia.saalfeldlab.n5.codec.FixedLengthConvertedInputStream; import org.junit.Test; public class FixedConvertedInputStreamTest { @@ -24,7 +22,7 @@ public void testLengthOne() throws IOException Arrays.fill(data, expected); final FixedLengthConvertedInputStream convertedId = new FixedLengthConvertedInputStream(1, 1, - AsTypeCodec.IDENTITY_ONE, + AsTypeCodec::IDENTITY_ONE, new ByteArrayInputStream(data)); final FixedLengthConvertedInputStream convertedPlusOne = new FixedLengthConvertedInputStream(1, 1, @@ -55,7 +53,7 @@ public void testIntToByte() throws IOException final byte[] data = buf.array(); final FixedLengthConvertedInputStream intToByte = new FixedLengthConvertedInputStream( 4, 1, - AsTypeCodec.INT_TO_BYTE, + AsTypeCodec::INT_TO_BYTE, new ByteArrayInputStream(data)); for( int i = 0; i < N; i++ ) @@ -74,7 +72,7 @@ public void testByteToInt() throws IOException data[i] = (byte)i; final FixedLengthConvertedInputStream byteToInt = new FixedLengthConvertedInputStream( - 1, 4, AsTypeCodec.BYTE_TO_INT, + 1, 4, AsTypeCodec::BYTE_TO_INT, new ByteArrayInputStream(data)); final DataInputStream dataStream = new DataInputStream(byteToInt); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java index a457dfb1b..1035e271a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java @@ -8,8 +8,6 @@ import java.util.Arrays; import java.util.stream.IntStream; -import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; -import org.janelia.saalfeldlab.n5.codec.FixedLengthConvertedOutputStream; import org.junit.Test; public class FixedConvertedOutputStreamTest { @@ -28,7 +26,7 @@ public void testLengthOne() throws IOException final ByteArrayOutputStream outId = new ByteArrayOutputStream(N); final FixedLengthConvertedOutputStream convertedId = new FixedLengthConvertedOutputStream(1, 1, - AsTypeCodec.IDENTITY_ONE, + AsTypeCodec::IDENTITY_ONE, outId); convertedId.write(expectedData); @@ -68,7 +66,7 @@ public void testIntToByte() throws IOException final ByteArrayOutputStream outStream = new ByteArrayOutputStream(N); final FixedLengthConvertedOutputStream intToByte = new FixedLengthConvertedOutputStream( 4, 1, - AsTypeCodec.INT_TO_BYTE, + AsTypeCodec::INT_TO_BYTE, outStream); intToByte.write(buf.array()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java index 098d8ecee..6de676901 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java @@ -1,19 +1,10 @@ package org.janelia.saalfeldlab.n5.codec; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.stream.DoubleStream; import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.FixedScaleOffsetCodec; import org.junit.Test; public class FixedScaleOffsetTests { @@ -35,28 +26,35 @@ public void testDouble2Byte() throws IOException { encodedDoubles.putDouble(i); } - final FixedScaleOffsetCodec double2Byte = new FixedScaleOffsetCodec(scale, offset, DataType.FLOAT64, DataType.UINT8); - testEncoding(double2Byte, bytes, encodedDoubles.array()); - testDecoding(double2Byte, encodedDoubles.array(), bytes); + final FixedScaleOffsetCodec double2Byte = new FixedScaleOffsetCodec(scale, offset, DataType.FLOAT64, DataType.INT8); + AsTypeTests.testEncoding(double2Byte, bytes, encodedDoubles.array()); + AsTypeTests.testDecoding(double2Byte, encodedDoubles.array(), bytes); } - protected static void testDecoding( final Codec codec, final byte[] expected, final byte[] input ) throws IOException - { - final InputStream result = codec.decode(new ByteArrayInputStream(input)); - for (int i = 0; i < expected.length; i++) - assertEquals(expected[i], (byte)result.read()); - } + @Test + public void testLong2Short() throws IOException { - protected static void testEncoding( final Codec codec, final byte[] expected, final byte[] data ) throws IOException - { + final int N = 16; + final ByteBuffer encodedLongs = ByteBuffer.allocate(Double.BYTES * N); + final ByteBuffer encodedShorts = ByteBuffer.allocate(Short.BYTES * N); + + final long scale = 2; + final long offset = 1; - final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(expected.length); - final OutputStream encodedStream = codec.encode(outputStream); - encodedStream.write(data); - encodedStream.flush(); - final byte[] convertedArr = outputStream.toByteArray(); - assertArrayEquals( expected, outputStream.toByteArray()); - encodedStream.close(); + for (int i = 0; i < N; i++) { + final long val = (scale * i + offset); + encodedShorts.putShort((short)val); + encodedLongs.putLong(i); + } + + final byte[] shortBytes = encodedShorts.array(); + final byte[] longBytes = encodedLongs.array(); + + final FixedScaleOffsetCodec long2short = new FixedScaleOffsetCodec(scale, offset, DataType.INT64, DataType.INT16); + AsTypeTests.testEncoding(long2short, shortBytes, longBytes); + AsTypeTests.testDecoding(long2short, longBytes, shortBytes); } + + } From adb84bb7c3dcd282a1e5dae2cc06ce89db0144d3 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 18 Jul 2024 17:54:21 -0400 Subject: [PATCH 003/423] feat: add ChecksumCodec * DeterministicSizeCodec and Crc32cChecksumCodec --- .../n5/codec/DeterministicSizeCodec.java | 13 ++++ .../n5/codec/checksum/ChecksumCodec.java | 73 +++++++++++++++++++ .../codec/checksum/ChecksumInputStream.java | 50 +++++++++++++ .../codec/checksum/ChecksumOutputStream.java | 33 +++++++++ .../codec/checksum/Crc32cChecksumCodec.java | 41 +++++++++++ 5 files changed, 210 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumInputStream.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumOutputStream.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java new file mode 100644 index 000000000..9ac0a1fe0 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java @@ -0,0 +1,13 @@ +package org.janelia.saalfeldlab.n5.codec; + +/** + * A {@link Codec} that can deterministically determine the size of encoded data from the size of the raw data and vice versa from the data length alone (i.e. encoding is data + * independent). + */ +public interface DeterministicSizeCodec extends Codec { + + public abstract long encodedSize(long size); + + public abstract long decodedSize(long size); + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java new file mode 100644 index 000000000..a6bd6a84f --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java @@ -0,0 +1,73 @@ +package org.janelia.saalfeldlab.n5.codec.checksum; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.zip.Checksum; + +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; + +/** + * A {@link Codec} that appends a checksum to data when encoding and can validate against that checksum when decoding. + */ +public abstract class ChecksumCodec implements DeterministicSizeCodec { + + private static final long serialVersionUID = 3141427377277375077L; + + private int numChecksumBytes; + + private Checksum checksum; + + public ChecksumCodec(Checksum checksum, int numChecksumBytes) { + + this.checksum = checksum; + this.numChecksumBytes = numChecksumBytes; + } + + public Checksum getChecksum() { + + return checksum; + } + + public int numChecksumBytes() { + + return numChecksumBytes; + } + + @Override + public ChecksumInputStream decode(final InputStream in) throws IOException { + + // TODO get the correct expected checksum + // TODO write a test with nested checksum codecs + + // has to know the number of it needs to read? + return new ChecksumInputStream(getChecksum(), in); + } + + @Override + public ChecksumOutputStream encode(final OutputStream out) throws IOException { + + // when do we validate + return new ChecksumOutputStream(getChecksum(), out); + } + + @Override + public long encodedSize(final long size) { + + return size + numChecksumBytes(); + } + + @Override + public long decodedSize(final long size) { + + return size - numChecksumBytes(); + } + + public boolean validate() { + + // TODO implement + // does validate go here or in ChecksumOutputStream + return true; + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumInputStream.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumInputStream.java new file mode 100644 index 000000000..af7488197 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumInputStream.java @@ -0,0 +1,50 @@ +package org.janelia.saalfeldlab.n5.codec.checksum; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.zip.Checksum; + +public class ChecksumInputStream extends InputStream { + + private final Checksum checksum; + // private final long expected; + private InputStream in; + + public ChecksumInputStream(Checksum checksum, InputStream in) { + + // this needs to know how many bytes are in its checksum + // Maybe pass the codec here instead of the checksum + this.checksum = checksum; + this.in = in; + } + + @Override + public int read() throws IOException { + + // returns -1 if end of the stream is reached + final int b = in.read(); + checksum.update(b); + return b; + } + + protected long readChecksum() throws IOException { + + final byte[] checksum = new byte[getChecksumSize()]; + in.read(checksum); + return ByteBuffer.wrap(checksum).getLong(); + } + + public boolean validate() throws IOException { + + // TODO consider reading N more bytes (the checksum) + // set expected to that, then validate + return readChecksum() == checksum.getValue(); + } + + public int getChecksumSize() { + + return -1; + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumOutputStream.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumOutputStream.java new file mode 100644 index 000000000..4fac8f622 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumOutputStream.java @@ -0,0 +1,33 @@ +package org.janelia.saalfeldlab.n5.codec.checksum; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.zip.Checksum; + +public class ChecksumOutputStream extends OutputStream { + + private final Checksum checksum; + private OutputStream out; + + public ChecksumOutputStream(Checksum checksum, OutputStream out) { + + this.checksum = checksum; + this.out = out; + } + + @Override + public void write(int b) throws IOException { + + checksum.update(b); + out.write(b); + } + + public void finish() throws IOException { + + final ByteBuffer buf = ByteBuffer.allocate(8); + buf.asLongBuffer().put(checksum.getValue()); + out.write(buf.array()); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java new file mode 100644 index 000000000..511715104 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java @@ -0,0 +1,41 @@ +package org.janelia.saalfeldlab.n5.codec.checksum; + +import java.util.zip.CRC32; + +public class Crc32cChecksumCodec extends ChecksumCodec { + + private static final long serialVersionUID = 7424151868725442500L; + + public static transient final String CRC32C_CHECKSUM_CODEC_ID = "crc32c"; + + public Crc32cChecksumCodec() { + + super(new CRC32(), 4); + } + + @Override + public String getId() { + + return CRC32C_CHECKSUM_CODEC_ID; + } + + @Override + public long encodedSize(final long size) { + + return size + numChecksumBytes(); + } + + @Override + public long decodedSize(final long size) { + + return size - numChecksumBytes(); + } + + @Override + public boolean validate() { + + // TODO implement me + return true; + } + +} From c54bdb98b9e8270b86c7bcf3fc644bca79c30edf Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 19 Jul 2024 10:09:41 -0400 Subject: [PATCH 004/423] wip: use CheckedInput/Output streams from java.util.zip * move some functionality to ChecksumCodec --- .../n5/codec/checksum/ChecksumCodec.java | 58 ++++++++++++++++--- .../codec/checksum/ChecksumInputStream.java | 50 ---------------- .../codec/checksum/ChecksumOutputStream.java | 33 ----------- .../codec/checksum/Crc32cChecksumCodec.java | 8 ++- 4 files changed, 54 insertions(+), 95 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumInputStream.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumOutputStream.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java index a6bd6a84f..4b30265b2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java @@ -3,6 +3,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.zip.CheckedInputStream; +import java.util.zip.CheckedOutputStream; import java.util.zip.Checksum; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; @@ -35,20 +38,39 @@ public int numChecksumBytes() { } @Override - public ChecksumInputStream decode(final InputStream in) throws IOException { + public CheckedInputStream decode(final InputStream in) throws IOException { // TODO get the correct expected checksum // TODO write a test with nested checksum codecs // has to know the number of it needs to read? - return new ChecksumInputStream(getChecksum(), in); + return new CheckedInputStream(in, getChecksum()); } @Override - public ChecksumOutputStream encode(final OutputStream out) throws IOException { + public CheckedOutputStream encode(final OutputStream out) throws IOException { - // when do we validate - return new ChecksumOutputStream(getChecksum(), out); + // when do we validate? + return new CheckedOutputStream(out, getChecksum()); + } + + public void encode(final OutputStream out, ByteBuffer buffer) throws IOException { + + final CheckedOutputStream cout = new CheckedOutputStream(out, getChecksum()); + cout.write(buffer.array()); + writeChecksum(out); + } + + public ByteBuffer decodeAndValidate(final InputStream in, int numBytes) throws IOException { + + final CheckedInputStream cin = decode(in); + final byte[] data = new byte[numBytes]; + cin.read(data); + + if (!valid(in)) + throw new IOException("Invalid checksum"); + + return ByteBuffer.wrap(data); } @Override @@ -63,11 +85,29 @@ public long decodedSize(final long size) { return size - numChecksumBytes(); } - public boolean validate() { + protected boolean valid(InputStream in) throws IOException { - // TODO implement - // does validate go here or in ChecksumOutputStream - return true; + return readChecksum(in) == getChecksum().getValue(); } + protected long readChecksum(InputStream in) throws IOException { + + final byte[] checksum = new byte[numChecksumBytes()]; + in.read(checksum); + return ByteBuffer.wrap(checksum).getLong(); + } + + /** + * Return the value of the checksum as a {@link ByteBuffer} to be serialized. + * + * @return a ByteBuffer representing the checksum value + */ + public abstract ByteBuffer getChecksumValue(); + + public void writeChecksum(OutputStream out) throws IOException { + + out.write(getChecksumValue().array()); + } + + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumInputStream.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumInputStream.java deleted file mode 100644 index af7488197..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumInputStream.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec.checksum; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.util.zip.Checksum; - -public class ChecksumInputStream extends InputStream { - - private final Checksum checksum; - // private final long expected; - private InputStream in; - - public ChecksumInputStream(Checksum checksum, InputStream in) { - - // this needs to know how many bytes are in its checksum - // Maybe pass the codec here instead of the checksum - this.checksum = checksum; - this.in = in; - } - - @Override - public int read() throws IOException { - - // returns -1 if end of the stream is reached - final int b = in.read(); - checksum.update(b); - return b; - } - - protected long readChecksum() throws IOException { - - final byte[] checksum = new byte[getChecksumSize()]; - in.read(checksum); - return ByteBuffer.wrap(checksum).getLong(); - } - - public boolean validate() throws IOException { - - // TODO consider reading N more bytes (the checksum) - // set expected to that, then validate - return readChecksum() == checksum.getValue(); - } - - public int getChecksumSize() { - - return -1; - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumOutputStream.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumOutputStream.java deleted file mode 100644 index 4fac8f622..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumOutputStream.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec.checksum; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.zip.Checksum; - -public class ChecksumOutputStream extends OutputStream { - - private final Checksum checksum; - private OutputStream out; - - public ChecksumOutputStream(Checksum checksum, OutputStream out) { - - this.checksum = checksum; - this.out = out; - } - - @Override - public void write(int b) throws IOException { - - checksum.update(b); - out.write(b); - } - - public void finish() throws IOException { - - final ByteBuffer buf = ByteBuffer.allocate(8); - buf.asLongBuffer().put(checksum.getValue()); - out.write(buf.array()); - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java index 511715104..bab39e413 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.codec.checksum; +import java.nio.ByteBuffer; import java.util.zip.CRC32; public class Crc32cChecksumCodec extends ChecksumCodec { @@ -32,10 +33,11 @@ public long decodedSize(final long size) { } @Override - public boolean validate() { + public ByteBuffer getChecksumValue() { - // TODO implement me - return true; + final ByteBuffer buf = ByteBuffer.allocate(numChecksumBytes()); + buf.putInt((int)getChecksum().getValue()); + return buf; } } From e756f8e18b18013f1cc442963bf42548779cd73e Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 19 Jul 2024 10:24:56 -0400 Subject: [PATCH 005/423] wip: add ChecksumException --- .../saalfeldlab/n5/codec/checksum/ChecksumCodec.java | 4 ++-- .../n5/codec/checksum/ChecksumException.java | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumException.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java index 4b30265b2..84555dd38 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java @@ -61,14 +61,14 @@ public void encode(final OutputStream out, ByteBuffer buffer) throws IOException writeChecksum(out); } - public ByteBuffer decodeAndValidate(final InputStream in, int numBytes) throws IOException { + public ByteBuffer decodeAndValidate(final InputStream in, int numBytes) throws IOException, ChecksumException { final CheckedInputStream cin = decode(in); final byte[] data = new byte[numBytes]; cin.read(data); if (!valid(in)) - throw new IOException("Invalid checksum"); + throw new ChecksumException("Invalid checksum"); return ByteBuffer.wrap(data); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumException.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumException.java new file mode 100644 index 000000000..034343c42 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumException.java @@ -0,0 +1,12 @@ +package org.janelia.saalfeldlab.n5.codec.checksum; + +public class ChecksumException extends Exception { + + private static final long serialVersionUID = 905130066386622561L; + + public ChecksumException(final String message) { + + super(message); + } + +} From 5a821593f31b1b82377834c56ea5ac8888c88992 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 23 Jul 2024 13:35:14 -0400 Subject: [PATCH 006/423] refactor: codec getId to getName * to match zarr v3 --- .../org/janelia/saalfeldlab/n5/CodecAdapter.java | 8 ++++---- .../org/janelia/saalfeldlab/n5/Compression.java | 2 +- .../janelia/saalfeldlab/n5/codec/AsTypeCodec.java | 6 +++--- .../org/janelia/saalfeldlab/n5/codec/Codec.java | 2 +- .../saalfeldlab/n5/codec/ComposedCodec.java | 6 +++--- .../n5/codec/FixedScaleOffsetCodec.java | 14 +++----------- .../saalfeldlab/n5/codec/IdentityCodec.java | 6 +++--- .../n5/codec/checksum/Crc32cChecksumCodec.java | 6 ++++-- 8 files changed, 22 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java index 0933433e4..0390d9a3f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java @@ -47,10 +47,10 @@ public JsonElement serialize( final Type typeOfSrc, final JsonSerializationContext context) { - if (codec.getId().equals(FixedScaleOffsetCodec.FIXED_SCALE_OFFSET_CODEC_ID)) { + if (codec.getName().equals(FixedScaleOffsetCodec.FIXED_SCALE_OFFSET_CODEC_ID)) { final FixedScaleOffsetCodec c = (FixedScaleOffsetCodec)codec; final JsonObject obj = new JsonObject(); - obj.addProperty("id", c.getId()); + obj.addProperty("name", c.getName()); obj.addProperty("scale", c.getScale()); obj.addProperty("offset", c.getOffset()); obj.addProperty("type", c.getType().toString().toLowerCase()); @@ -73,9 +73,9 @@ else if (!json.isJsonObject()) return null; final JsonObject jsonObject = json.getAsJsonObject(); - if (jsonObject.has("id")) { + if (jsonObject.has("name")) { - final String id = jsonObject.get("id").getAsString(); + final String id = jsonObject.get("name").getAsString(); if (id.equals(FixedScaleOffsetCodec.FIXED_SCALE_OFFSET_CODEC_ID)) { return new FixedScaleOffsetCodec( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index 3092bbe45..a6ca1be04 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -133,7 +133,7 @@ public OutputStream encode(OutputStream out) throws IOException { } @Override - public String getId() { + public String getName() { return compression.getType(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java index d4852d4a0..f7bf9945d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java @@ -22,7 +22,7 @@ public class AsTypeCodec implements Codec { protected final DataType type; protected final DataType encodedType; - protected final String id = "astype"; + protected final String name = "astype"; public AsTypeCodec( DataType type, DataType encodedType ) { @@ -37,9 +37,9 @@ public AsTypeCodec( DataType type, DataType encodedType ) } @Override - public String getId() { + public String getName() { - return id; + return name; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index a2e5373f0..b3225b7e8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -32,6 +32,6 @@ public interface Codec extends Serializable { */ public OutputStream encode(OutputStream out) throws IOException; - public String getId(); + public String getName(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java index 49e6f923d..10f8adb01 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java @@ -12,7 +12,7 @@ public class ComposedCodec implements Codec { private static final long serialVersionUID = 5068349140842235924L; private final Codec[] filters; - protected String id = "composed"; + protected String name = "composed"; public ComposedCodec(final Codec... filters) { @@ -20,9 +20,9 @@ public ComposedCodec(final Codec... filters) { } @Override - public String getId() { + public String getName() { - return id; + return name; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java index 34b06ecdf..0538498db 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java @@ -14,13 +14,11 @@ public class FixedScaleOffsetCodec extends AsTypeCodec { public static transient final String FIXED_SCALE_OFFSET_CODEC_ID = "fixedscaleoffset"; - @SuppressWarnings("unused") private final double scale; - @SuppressWarnings("unused") private final double offset; - protected final String id = FIXED_SCALE_OFFSET_CODEC_ID; + protected final String name = FIXED_SCALE_OFFSET_CODEC_ID; private transient final ByteBuffer tmpEncoder; private transient final ByteBuffer tmpDecoder; @@ -55,8 +53,6 @@ public FixedScaleOffsetCodec(final double scale, final double offset, DataType t encoderPre.accept(i, tmpEncoder); tmpEncoder.rewind(); final double x = tmpEncoder.getDouble(); - final double y = scale * x + offset; - System.out.println("encode: " + y); tmpEncoder.rewind(); tmpEncoder.putDouble(scale * x + offset); tmpEncoder.rewind(); @@ -65,10 +61,6 @@ public FixedScaleOffsetCodec(final double scale, final double offset, DataType t // convert from i type to double, apply scale and offset, then convert to type o decoder = (i, o) -> { - // System.out.println("decode"); - // System.out.println(i.capacity()); - // System.out.println(tmpDecoder.capacity()); - // System.out.println(o.capacity()); tmpDecoder.rewind(); decoderPre.accept(i, tmpDecoder); tmpDecoder.rewind(); @@ -101,9 +93,9 @@ public DataType getEncodedType() { } @Override - public String getId() { + public String getName() { - return id; + return name; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java index 5164c5474..4383669aa 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -8,12 +8,12 @@ public class IdentityCodec implements Codec { private static final long serialVersionUID = 8354269325800855621L; - protected final String id = "id"; + protected final String name = "id"; @Override - public String getId() { + public String getName() { - return id; + return name; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java index bab39e413..0a16d435a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java @@ -9,15 +9,17 @@ public class Crc32cChecksumCodec extends ChecksumCodec { public static transient final String CRC32C_CHECKSUM_CODEC_ID = "crc32c"; + private final String name = CRC32C_CHECKSUM_CODEC_ID; + public Crc32cChecksumCodec() { super(new CRC32(), 4); } @Override - public String getId() { + public String getName() { - return CRC32C_CHECKSUM_CODEC_ID; + return name; } @Override From 1cd307c42879dbda6a7ce7b8d73443dee776eeec Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 23 Jul 2024 13:43:55 -0400 Subject: [PATCH 007/423] feat: add BytesCodec --- .../saalfeldlab/n5/codec/BytesCodec.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java new file mode 100644 index 000000000..65cff6a21 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java @@ -0,0 +1,21 @@ +package org.janelia.saalfeldlab.n5.codec; + + +public class BytesCodec extends IdentityCodec { + + private static final long serialVersionUID = 3523505403978222360L; + + public static final String ID = "bytes"; + + protected final String name = ID; + + protected final String endian = "little"; + + // TODO implement me + + @Override + public String getName() { + + return name; + } +} From 72cd669af0ecfc8e122b3b7b7d8c9f77ef6b1363 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 23 Jul 2024 13:44:16 -0400 Subject: [PATCH 008/423] feat: add ShardingCodec --- .../saalfeldlab/n5/shard/ShardingCodec.java | 86 ++++++++++++++++++ .../n5/shard/ShardingConfiguration.java | 89 +++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingConfiguration.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java new file mode 100644 index 000000000..9f9f978a3 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -0,0 +1,86 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Type; + +import org.janelia.saalfeldlab.n5.codec.Codec; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +public class ShardingCodec implements Codec { + + private static final long serialVersionUID = -5879797314954717810L; + + public static final String ID = "sharding_indexed"; + + private final ShardingConfiguration configuration; + + private final String name = ID; + + public ShardingCodec(ShardingConfiguration configuration) { + + this.configuration = configuration; + } + + public ShardingConfiguration getConfiguration() { + + return configuration; + } + + @Override + public InputStream decode(InputStream in) throws IOException { + + // TODO Auto-generated method stub + return in; + } + + @Override + public OutputStream encode(OutputStream out) throws IOException { + + // TODO Auto-generated method stub + return out; + } + + @Override + public String getName() { + + return name; + } + + public static boolean isShardingCodec(final Codec codec) { + + return codec instanceof ShardingCodec; + } + + // public static void TypeAd + public static class ShardingCodecAdapter implements JsonDeserializer, JsonSerializer { + + @Override + public JsonElement serialize(ShardingCodec src, Type typeOfSrc, JsonSerializationContext context) { + + final JsonObject jsonObj = new JsonObject(); + + jsonObj.addProperty("name", ShardingCodec.ID); + // context.serialize(typeOfSrc); + + return jsonObj; + } + + @Override + public ShardingCodec deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) + throws JsonParseException { + + return null; + } + + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingConfiguration.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingConfiguration.java new file mode 100644 index 000000000..2a29a1162 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingConfiguration.java @@ -0,0 +1,89 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.lang.reflect.Type; +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.codec.Codec; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +public class ShardingConfiguration { + + public static final String CHUNK_SHAPE_KEY = "chunk_shape"; + public static final String INDEX_LOCATION_KEY = "index_location"; + public static final String CODECS_KEY = "codecs"; + public static final String INDEX_CODECS_KEY = "index_codecs"; + + public static enum IndexLocation { + start, end + }; + + protected int[] blockSize; + protected Codec[] codecs; + protected Codec[] indexCodecs; + protected IndexLocation indexLocation; + + public ShardingConfiguration(final int[] blockSize, final Codec[] codecs, final Codec[] indexCodecs, + final IndexLocation indexLocation) { + + this.blockSize = blockSize; + this.codecs = codecs; + this.indexCodecs = indexCodecs; + this.indexLocation = indexLocation; + } + + public int[] getBlockSize() { + + return blockSize; + } + + public boolean areIndexesAtStart() { + + return indexLocation == IndexLocation.start; + } + + public static class ShardingConfigurationAdapter + implements JsonDeserializer, JsonSerializer { + + @Override + public JsonElement serialize(ShardingConfiguration src, Type typeOfSrc, JsonSerializationContext context) { + + if( anyShardingCodecs(src.codecs) || anyShardingCodecs(src.indexCodecs)) + return JsonNull.INSTANCE; + + final JsonObject jsonObj = new JsonObject(); + jsonObj.add(CHUNK_SHAPE_KEY, context.serialize(src.blockSize)); + jsonObj.add(INDEX_LOCATION_KEY, context.serialize(src.indexLocation.toString())); + jsonObj.add(CODECS_KEY, context.serialize(src.codecs)); + jsonObj.add(INDEX_CODECS_KEY, context.serialize(src.indexCodecs)); + + return jsonObj; + } + + @Override + public ShardingConfiguration deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) + throws JsonParseException { + + return null; + } + + public boolean anyShardingCodecs(final Codec[] codecs) { + + if (codecs == null) + return false; + + return Arrays.stream(codecs).anyMatch(c -> { + return (c instanceof ShardingCodec); + }); + } + + } + +} From 6fbb729271104f2a1ef27c2827019ac1f9b58670 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 23 Jul 2024 13:45:00 -0400 Subject: [PATCH 009/423] feat(wip): add prelim Shard classes --- .../n5/ShardedDatasetAttributes.java | 90 ++++++++++++ .../saalfeldlab/n5/shard/AbstractShard.java | 50 +++++++ .../saalfeldlab/n5/shard/InMemoryShard.java | 28 ++++ .../janelia/saalfeldlab/n5/shard/Shard.java | 111 ++++++++++++++ .../saalfeldlab/n5/shard/ShardException.java | 14 ++ .../saalfeldlab/n5/shard/ShardImpl.java | 20 +++ .../saalfeldlab/n5/shard/ShardIndex.java | 120 +++++++++++++++ .../saalfeldlab/n5/shard/ShardReader.java | 90 ++++++++++++ .../saalfeldlab/n5/shard/ShardWriter.java | 137 ++++++++++++++++++ .../janelia/saalfeldlab/n5/shard/Shards.java | 106 ++++++++++++++ 10 files changed, 766 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardException.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardImpl.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/Shards.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java new file mode 100644 index 000000000..9b69ad6e9 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -0,0 +1,90 @@ +package org.janelia.saalfeldlab.n5; + +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; +import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration; +import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.Shards; + +public class ShardedDatasetAttributes extends DatasetAttributes { + + private static final long serialVersionUID = -4559068841006651814L; + + private final int[] shardSize; + + private final IndexLocation indexLocation; + + public ShardedDatasetAttributes(final long[] dimensions, final int[] shardSize, final DataType dataType, + final Compression compression, + final Codec[] codecs) { + + super(dimensions, getBlockSize(codecs), dataType, compression, codecs); + final ShardingConfiguration config = getShardConfiguration(codecs); + this.indexLocation = config.areIndexesAtStart() ? IndexLocation.start : IndexLocation.end; + this.shardSize = shardSize; + } + + public ShardedDatasetAttributes(final long[] dimensions, + final int[] shardSize, + final int[] blockSize, + final IndexLocation shardIndexLocation, + final DataType dataType, + final Compression compression, + final Codec[] codecs) { + + super(dimensions, blockSize, dataType, compression, codecs); + this.shardSize = shardSize; + this.indexLocation = shardIndexLocation; + // this.config = new ShardingConfiguration(blockSize, null, null, shardIndexLocation); + + // TODO figure out codecs + } + + public int[] getShardSize() { + + return shardSize; + } + + public Shards getShards() { + + return new Shards(this); + } + + public ShardingConfiguration getShardingConfiguration() { + + return Arrays.stream(getCodecs()) + .filter(ShardingCodec::isShardingCodec) + .map(x -> { + return ((ShardingCodec)x).getConfiguration(); + }) + .findFirst().orElse(null); + } + + public static boolean isSharded(Codec[] codecs) { + + return Arrays.stream(codecs).anyMatch(ShardingCodec::isShardingCodec); + } + + public static ShardingConfiguration getShardConfiguration(Codec[] codecs) { + + return Arrays.stream(codecs) + .filter(ShardingCodec::isShardingCodec) + .map(x -> { + return ((ShardingCodec)x).getConfiguration(); + }) + .findFirst().orElse(null); + } + + public static int[] getBlockSize(Codec[] codecs) { + + return Arrays.stream(codecs) + .filter(ShardingCodec::isShardingCodec) + .map(x -> { + return ((ShardingCodec)x).getConfiguration(); + }) + .map(ShardingConfiguration::getBlockSize).findFirst().orElse(null); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java new file mode 100644 index 000000000..05d63144e --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java @@ -0,0 +1,50 @@ +package org.janelia.saalfeldlab.n5.shard; + +import org.janelia.saalfeldlab.n5.DataBlock; + +public abstract class AbstractShard implements Shard { + + protected final long[] size; + protected final long[] gridPosition; + protected final int[] blockSize; + + public AbstractShard(final long[] size, final long[] gridPosition, + final int[] blockSize, final T type) { + + this.size = size; + this.gridPosition = gridPosition; + this.blockSize = blockSize; + } + + @Override + public long[] getSize() { + + return size; + } + + @Override + public int[] getBlockSize() { + + return blockSize; + } + + @Override + public long[] getGridPosition() { + + return gridPosition; + } + + @Override + public DataBlock getBlock(int... position) { + + return null; + } + + @Override + public ShardIndex getIndexes() { + + return null; + } + + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java new file mode 100644 index 000000000..72f102b5f --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -0,0 +1,28 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.ArrayList; +import java.util.List; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; + +public class InMemoryShard extends AbstractShard { + + private List> blocks; + + private ShardIndex shardIndex; + + public InMemoryShard(final long[] size, final long[] gridPosition, final int[] blockSize, T type) { + + super(size, gridPosition, blockSize, type); + blocks = new ArrayList<>(); + } + + @Override + public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) { + + // TODO Auto-generated method stub + return null; + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java new file mode 100644 index 000000000..a587f735c --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -0,0 +1,111 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; + +public interface Shard { + + /** + * Returns the number of blocks this shard contains along all dimensions. + * + * The size of a shard expected to be smaller than or equal to the spacing of the shard grid. The dimensionality of size is expected to be equal to the dimensionality of the + * dataset. Consistency is not enforced. + * + * @return size of the shard in units of blocks + */ + default int[] getBlockGridSize() { + + final long[] sz = getSize(); + final int[] blkSz = getBlockSize(); + final int[] blockGridSize = new int[sz.length]; + for (int i = 0; i < sz.length; i++) + blockGridSize[i] = (int)(sz[i] / blkSz[i]); + + return blockGridSize; + } + + /** + * Returns the size of shards in pixel units. + * + * @return shard size + */ + public long[] getSize(); + + /** + * Returns the size of blocks in pixel units. + * + * @return block size + */ + public int[] getBlockSize(); + + /** + * Returns the position of this shard on the shard grid. + * + * The dimensionality of the grid position is expected to be equal to the dimensionality of the dataset. Consistency is not enforced. + * + * @return position on the shard grid + */ + public long[] getGridPosition(); + + /** + * Returns of the block at the given position relative to this shard, or null if this shard does not contain the given block. + * + * @return the shard position + */ + default int[] getBlockPosition(long... blockPosition) { + + final long[] shardPos = getShard(blockPosition); + if (!Arrays.equals(getGridPosition(), shardPos)) + return null; + + final long[] shardSize = getSize(); + final int[] blkSize = getBlockSize(); + final int[] blkGridSize = getBlockGridSize(); + + final int[] blockShardPos = new int[shardSize.length]; + for (int i = 0; i < shardSize.length; i++) { + final long shardP = shardPos[i] * shardSize[i]; + final long blockP = blockPosition[i] * blkSize[i]; + blockShardPos[i] = (int)((blockP - shardP) / blkGridSize[i]); + } + + return blockShardPos; + } + + /** + * Returns the position of the shard containing the block with the given block position. + * + * @return the shard position + */ + default long[] getShard(long... blockPosition) { + + final int[] shardBlockDimensions = getBlockGridSize(); + final long[] shardGridPosition = new long[shardBlockDimensions.length]; + for (int i = 0; i < shardGridPosition.length; i++) { + shardGridPosition[i] = (long)Math.floor((double)blockPosition[i] / shardBlockDimensions[i]); + } + + return shardGridPosition; + } + + public DataBlock getBlock(int... position); + + public ShardIndex getIndexes(); + + public DataBlock readBlock(final String pathName, final DatasetAttributes datasetAttributes, long... gridPosition); + + + /** + * Say we want async datablock access + * + * Say we construct shard then getBlockAt + * + * (this could be how we do the aggregation) multiple getblockAt calls don't trigger reading read triggers reading of all blocks that were requested + * + * Shard doesn't hold the data directly, but is the metadata about how the blocks are stored + * + */ + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardException.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardException.java new file mode 100644 index 000000000..d208c62ed --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardException.java @@ -0,0 +1,14 @@ +package org.janelia.saalfeldlab.n5.shard; + +import org.janelia.saalfeldlab.n5.N5Exception; + +public class ShardException extends N5Exception { + + private static final long serialVersionUID = -77907634621557855L; + + public static class IndexException extends ShardException { + + private static final long serialVersionUID = 3924426352575114063L; + + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardImpl.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardImpl.java new file mode 100644 index 000000000..11803b3b9 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardImpl.java @@ -0,0 +1,20 @@ +package org.janelia.saalfeldlab.n5.shard; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; + +public class ShardImpl extends AbstractShard { + + public ShardImpl(final long[] size, final long[] gridPosition, final int[] blockSize, T type) { + + super(size, gridPosition, blockSize, type); + } + + @Override + public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) { + + // TODO Auto-generated method stub + return null; + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java new file mode 100644 index 000000000..b2a19472f --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -0,0 +1,120 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; + +public class ShardIndex extends LongArrayDataBlock { + + private static final int BYTES_PER_LONG = 8; + + public ShardIndex(int[] size, long[] data) { + + super(indexBlockSize(size), new long[]{0}, data); + } + + public ShardIndex(int[] size) { + + super(indexBlockSize(size), new long[]{0}, createData(size)); + } + + private static long[] createData(final int[] size) { + + final int N = 2 * Arrays.stream(size).reduce(1, (x, y) -> x * y); + return new long[N]; + } + + private static int[] indexBlockSize(final int[] size) { + + final int[] indexBlockSize = new int[size.length + 1]; + indexBlockSize[0] = 2; + System.arraycopy(size, 0, indexBlockSize, 1, size.length); + return indexBlockSize; + } + + public long getOffset(long... gridPosition) { + + return data[getOffsetIndex(gridPosition)]; + } + + public long getNumBytes(long... gridPosition) { + + return data[getNumBytesIndex(gridPosition)]; + } + + public void set(long offset, long nbytes, long[] gridPosition) { + + final int i = getOffsetIndex(gridPosition); + data[i] = offset; + data[i + 1] = nbytes; + } + + + private int getOffsetIndex(long... gridPosition) { + + int idx = 0; + long stride = 2; + for (int i = 0; i < gridPosition.length; i++) { + idx += gridPosition[i] * stride; + stride *= size[i]; + } + + return idx; + } + + private int getNumBytesIndex(long... gridPosition) { + + return getOffsetIndex() + 1; + } + + public void printData() { + + System.out.println(Arrays.toString(data)); + } + + public static ShardIndex read(FileChannel channel, ShardedDatasetAttributes datasetAttributes) throws IOException { + + // TODO need codecs + // TODO FileChannel is too specific - generalize + final Shards shards = new Shards(datasetAttributes); + final int[] indexShape = indexBlockSize(shards.getShardBlockGridSize()); + final int indexSize = (int)Arrays.stream(indexShape).reduce(1, (x, y) -> x * y); + final int indexBytes = BYTES_PER_LONG * indexSize; + + if (!datasetAttributes.getShardingConfiguration().areIndexesAtStart()) { + channel.position(channel.size() - indexBytes); + } + + final InputStream is = Channels.newInputStream(channel); + final DataInputStream dis = new DataInputStream(is); + + final long[] indexes = new long[indexSize]; + for (int i = 0; i < indexSize; i++) { + indexes[i] = dis.readLong(); + } + + return new ShardIndex(indexShape, indexes); + } + + public static void main(String[] args) { + + final ShardIndex ib = new ShardIndex(new int[]{2, 2}); + + ib.set(8, 9, new long[]{1, 1}); + ib.printData(); + + // System.out.println(ib.getIndex(0, 0)); + // System.out.println(ib.getIndex(1, 0)); + // System.out.println(ib.getIndex(0, 1)); + // System.out.println(ib.getIndex(1, 1)); + + System.out.println("done"); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java new file mode 100644 index 000000000..d68a03c01 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java @@ -0,0 +1,90 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.DefaultBlockReader; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; + +public class ShardReader { + + + private final ShardedDatasetAttributes datasetAttributes; + private long[] indexes; + private Shards shards; + + public ShardReader(final ShardedDatasetAttributes datasetAttributes) { + + this.datasetAttributes = datasetAttributes; + this.shards = new Shards(datasetAttributes); + } + + public ShardIndex readIndexes(FileChannel channel) throws IOException { + + return ShardIndex.read(channel, datasetAttributes); + } + + public InMemoryShard readShardFully( + final FileChannel channel, + long... gridPosition) throws IOException { + + final DatasetAttributes dsetAttrs = shards.getDatasetAttributes(); + + final ShardIndex si = readIndexes(channel); + return null; + } + + public DataBlock readBlock( + final FileChannel in, + long... blockPosition) throws IOException { + + // TODO generalize from FileChannel + // TODO this assumes the "file" holding the shard is known, + // the logic to figure that out will have to go somewhere + + final ShardIndex index = readIndexes(in); + + final long[] shardPosition = shards.getShardPositionForBlock(blockPosition); + in.position(index.getOffset(shardPosition)); + final InputStream is = Channels.newInputStream(in); + return DefaultBlockReader.readBlock(is, datasetAttributes, indexes); + } + + private long getIndexIndex(long... shardPosition) { + + final int[] indexDimensions = shards.getShardBlockGridSize(); + long idx = 0; + for (int i = 0; i < indexDimensions.length; i++) { + idx += shardPosition[i] * indexDimensions[i]; + } + + return idx; + } + + public static void main(String[] args) { + + final ShardReader reader = new ShardReader(buildTestAttributes()); + + System.out.println(reader.getIndexIndex(0, 0)); + System.out.println(reader.getIndexIndex(0, 1)); + System.out.println(reader.getIndexIndex(1, 0)); + System.out.println(reader.getIndexIndex(1, 1)); + } + + private static ShardedDatasetAttributes buildTestAttributes() { + + final Codec[] codecs = new Codec[]{ + new ShardingCodec(new ShardingConfiguration(new int[]{2, 2}, null, null, IndexLocation.end))}; + + return new ShardedDatasetAttributes(new long[]{4, 4}, new int[]{2, 2}, DataType.INT32, new RawCompression(), codecs); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java new file mode 100644 index 000000000..5705d583d --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java @@ -0,0 +1,137 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DefaultBlockWriter; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; + +public class ShardWriter { + + private static final int BYTES_PER_LONG = 8; + + private final List> blocks; + + private final ShardedDatasetAttributes datasetAttributes; + + private ByteBuffer blockSizes; + + private ByteBuffer blockIndexes; + + private ShardIndex indexData; + + private List blockBytes; + + public ShardWriter(final ShardedDatasetAttributes datasetAttributes) { + + blocks = new ArrayList<>(); + this.datasetAttributes = datasetAttributes; + } + + public void reset() { + + blocks.clear(); + blockSizes = null; + blockBytes.clear(); + + indexData = null; + } + + public void addBlock(final DataBlock block) { + + blocks.add(block); + } + + public void write(final OutputStream out) throws IOException { + + // TODO need codecs + + // prepareForWriting(); + // if (datasetAttributes.getShardingConfiguration().areIndexesAtStart()) { + // writeIndexes(out); + // writeBlocks(out); + // } else { + // writeBlocks(out); + // writeIndexes(out); + // } + + prepareForWritingDataBlock(); + if (datasetAttributes.getShardingConfiguration().areIndexesAtStart()) { + writeIndexBlock(out); + writeBlocks(out); + } else { + writeBlocks(out); + writeIndexBlock(out); + } + } + + private void prepareForWritingDataBlock() throws IOException { + + // final ShardingProperties shardProps = new ShardingProperties(datasetAttributes); + // indexData = new ShardIndexDataBlock(shardProps.getIndexDimensions()); + + indexData = new ShardIndex(new int[]{blocks.size()}); + blockBytes = new ArrayList<>(); + long cumulativeBytes = 0; + final long[] shardPosition = new long[1]; + for (int i = 0; i < blocks.size(); i++) { + + final ByteArrayOutputStream blockOut = new ByteArrayOutputStream(); + DefaultBlockWriter.writeBlock(blockOut, datasetAttributes, blocks.get(i)); + System.out.println(String.format("block %d is %d bytes", i, blockOut.size())); + + shardPosition[0] = i; + indexData.set(cumulativeBytes, blockOut.size(), shardPosition); + cumulativeBytes += blockOut.size(); + + blockBytes.add(blockOut.toByteArray()); + } + + indexData.printData(); + } + + private void prepareForWriting() throws IOException { + + blockSizes = ByteBuffer.allocate(BYTES_PER_LONG * blocks.size()); + blockIndexes = ByteBuffer.allocate(BYTES_PER_LONG * blocks.size()); + blockBytes = new ArrayList<>(); + long cumulativeBytes = 0; + for (int i = 0; i < blocks.size(); i++) { + + final ByteArrayOutputStream blockOut = new ByteArrayOutputStream(); + DefaultBlockWriter.writeBlock(blockOut, datasetAttributes, blocks.get(i)); + System.out.println(String.format("block %d is %d bytes", i, blockOut.size())); + + blockIndexes.putLong(cumulativeBytes); + blockSizes.putLong(blockOut.size()); + cumulativeBytes += blockOut.size(); + + blockBytes.add(blockOut.toByteArray()); + } + } + + private void writeBlocks(final OutputStream out) throws IOException { + + for (final byte[] bytes : blockBytes) + out.write(bytes); + } + + private void writeIndexes(final OutputStream out) throws IOException { + + out.write(blockSizes.array()); + } + + private void writeIndexBlock(final OutputStream out) throws IOException { + + final DataOutputStream dos = new DataOutputStream(out); + for (final long l : indexData.getData()) + dos.writeLong(l); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shards.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shards.java new file mode 100644 index 000000000..fb0f6aca9 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shards.java @@ -0,0 +1,106 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; + +/** + * Manages the set of shards that comprise a dataset. + */ +public class Shards { + + public static long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; + + private final ShardedDatasetAttributes datasetAttributes; + + public Shards(final ShardedDatasetAttributes datasetAttributes) { + + this.datasetAttributes = datasetAttributes; + } + + public ShardedDatasetAttributes getDatasetAttributes() { + + return datasetAttributes; + } + + public int[] getShardSize() { + + return getDatasetAttributes().getShardSize(); + } + + public int[] getBlockSize() { + + return getDatasetAttributes().getBlockSize(); + } + + /** + * Returns the number of blocks a shard contains along all dimensions. + * + * @return the size of the block grid of a shard + */ + public int[] getShardBlockGridSize() { + + final int nd = getDatasetAttributes().getNumDimensions(); + final int[] shardBlockGridSize = new int[nd]; + final int[] blockSize = getBlockSize(); + for (int i = 0; i < nd; i++) + shardBlockGridSize[i] = (int)(Math + .ceil((double)getDatasetAttributes().getDimensions()[i] / blockSize[i])); + + return shardBlockGridSize; + } + + /** + * Given a block's position relative to the array, returns the position of the shard containing that block relative to the shard grid. + * + * @param gridPosition + * position of a block relative to the array + * @return the position of the containing shard in the shard grid + */ + public long[] getShardPositionForBlock(final long... blockGridPosition) { + + // TODO have this return a shard + final int[] shardBlockDimensions = getShardBlockGridSize(); + final long[] shardGridPosition = new long[blockGridPosition.length]; + for (int i = 0; i < shardGridPosition.length; i++) { + shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / shardBlockDimensions[i]); + } + + return shardGridPosition; + } + + /** + * Returns of the block at the given position relative to this shard, or null if this shard does not contain the given block. + * + * @return the shard position + */ + public int[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { + + final long[] shardPos = getShardPositionForBlock(blockPosition); + if (!Arrays.equals(shardPosition, shardPos)) + return null; + + final int[] shardSize = getShardSize(); + final int[] blkSize = getBlockSize(); + final int[] blkGridSize = getShardBlockGridSize(); + + final int[] blockShardPos = new int[shardSize.length]; + for (int i = 0; i < shardSize.length; i++) { + final long shardP = shardPos[i] * shardSize[i]; + final long blockP = blockPosition[i] * blkSize[i]; + blockShardPos[i] = (int)((blockP - shardP) / blkGridSize[i]); + } + + return blockShardPos; + } + + /** + * @return the number of blocks per shard + */ + public long getNumBlocks() { + + return Arrays.stream(getShardBlockGridSize()).reduce(1, (x, y) -> x * y); + } + + +} From a2131902c62774cb22be894e034cfc212fa71f87 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 23 Jul 2024 13:45:38 -0400 Subject: [PATCH 010/423] feat(wip): update core n5 api with sharding --- .../janelia/saalfeldlab/n5/CodecAdapter.java | 29 +++++++++++++++ .../saalfeldlab/n5/DefaultBlockReader.java | 16 ++++++-- .../n5/FileSystemKeyValueAccess.java | 16 ++++++-- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 25 +++++++++---- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 13 +++++++ .../org/janelia/saalfeldlab/n5/GsonUtils.java | 3 ++ .../org/janelia/saalfeldlab/n5/N5Writer.java | 37 +++++++++++++++++++ 7 files changed, 125 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java index 0390d9a3f..db8a6daaf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java @@ -27,8 +27,11 @@ import java.lang.reflect.Type; +import org.janelia.saalfeldlab.n5.codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.FixedScaleOffsetCodec; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; +import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; @@ -57,6 +60,24 @@ public JsonElement serialize( obj.addProperty("encodedType", c.getEncodedType().toString().toLowerCase()); return obj; } + else if (codec.getName().equals(ShardingCodec.ID)) { + final ShardingCodec sharding = (ShardingCodec)codec; + final JsonObject obj = new JsonObject(); + obj.addProperty("name", sharding.getName()); + obj.add("configuration", context.serialize(sharding.getConfiguration())); + return obj; + } + else if (codec.getName().equals(BytesCodec.ID)) { + final BytesCodec bytes = (BytesCodec)codec; + final JsonObject obj = new JsonObject(); + obj.addProperty("name", bytes.getName()); + + final JsonObject config = new JsonObject(); + config.addProperty("endian", bytes.getName()); + obj.add("configuration", config); + + return obj; + } return JsonNull.INSTANCE; } @@ -84,6 +105,14 @@ else if (!json.isJsonObject()) DataType.valueOf(jsonObject.get("type").getAsString().toUpperCase()), DataType.valueOf(jsonObject.get("encodedType").getAsString().toUpperCase())); } + else if (id.equals(ShardingCodec.ID)) { + return new ShardingCodec( + context.deserialize(jsonObject.get("configuration"), ShardingConfiguration.class)); + } else if (id.equals(BytesCodec.ID)) { + + // TODO + return new BytesCodec(); + } } return null; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 4e5858193..0f70baea7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -30,6 +30,8 @@ import java.io.InputStream; import java.nio.ByteBuffer; +import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration; + /** * Default implementation of {@link BlockReader}. * @@ -46,10 +48,11 @@ public default > void read( final InputStream in) throws IOException { final ByteBuffer buffer = dataBlock.toByteBuffer(); - try (final InputStream inflater = getInputStream(in)) { - final DataInputStream dis = new DataInputStream(inflater); - dis.readFully(buffer.array()); - } + + // do not try with this input stream because subsequent block reads may happen if the stream points to a shard + final InputStream inflater = getInputStream(in); + final DataInputStream dis = new DataInputStream(inflater); + dis.readFully(buffer.array()); dataBlock.readData(buffer); } @@ -146,5 +149,10 @@ public static > void readFromStream(final B dataBlock, dataBlock.readData(buffer); } + public static long getShardIndex(final ShardingConfiguration shardingConfiguration, final long[] gridPosition) { + + // TODO implement + return -1; + } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index cfb455920..b7ea36d54 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -73,11 +73,21 @@ protected class LockedFileChannel implements LockedChannel { protected LockedFileChannel(final String path, final boolean readOnly) throws IOException { - this(fileSystem.getPath(path), readOnly); + this(fileSystem.getPath(path), readOnly, -1, -1); + } + + protected LockedFileChannel(final String path, final boolean readOnly, final long startByte, final long lastByte) throws IOException { + + this(fileSystem.getPath(path), readOnly, startByte, lastByte); } protected LockedFileChannel(final Path path, final boolean readOnly) throws IOException { + this(path, readOnly, -1, -1); + } + + protected LockedFileChannel(final Path path, final boolean readOnly, final long startByte, final long lastByte) throws IOException { + final OpenOption[] options; if (readOnly) { options = new OpenOption[]{StandardOpenOption.READ}; @@ -162,7 +172,7 @@ public LockedFileChannel lockForReading(final String normalPath) throws IOExcept try { return new LockedFileChannel(normalPath, true); - } catch (NoSuchFileException e) { + } catch (final NoSuchFileException e) { throw new N5Exception.N5NoSuchKeyException("No such file", e); } } @@ -177,7 +187,7 @@ public LockedFileChannel lockForReading(final Path path) throws IOException { try { return new LockedFileChannel(path, true); - } catch (NoSuchFileException e) { + } catch (final NoSuchFileException e) { throw new N5Exception.N5NoSuchKeyException("No such file", e); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 12867071b..9b8e288c0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -30,6 +30,8 @@ import java.util.Arrays; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.Shards; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -86,24 +88,32 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { } + default Shard readShard(final String pathName, + final ShardedDatasetAttributes datasetAttributes, + long... gridPosition) { + + final Shards shards = new Shards(datasetAttributes); + // TODO throw exception if this dataset is not sharded? + + return null; + } + @Override default DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { + if (ShardedDatasetAttributes.isSharded(datasetAttributes)) { + // TODO + } + final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), gridPosition); try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(path)) { -<<<<<<< HEAD - return DefaultBlockReader.readBlock(lockedChannel.newInputStream(), datasetAttributes, gridPosition); + return DefaultBlockReader.readBlockWithCodecs(lockedChannel.newInputStream(), datasetAttributes, gridPosition); } catch (final N5Exception.N5NoSuchKeyException e) { return null; -||||||| 6b6d4d2 - return DefaultBlockReader.readBlock(lockedChannel.newInputStream(), datasetAttributes, gridPosition); -======= - return DefaultBlockReader.readBlockWithCodecs(lockedChannel.newInputStream(), datasetAttributes, gridPosition); ->>>>>>> origin/codecs } catch (final IOException | UncheckedIOException e) { throw new N5IOException( "Failed to read block " + Arrays.toString(gridPosition) + " from dataset " + path, @@ -151,6 +161,7 @@ default String absoluteDataBlockPath( return getKeyValueAccess().compose(getURI(), components); } + /** * Constructs the absolute path (in terms of this store) for the group or * dataset. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index a784dcc6f..9e5253d76 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -32,6 +32,7 @@ import java.util.Map; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.shard.Shard; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -227,6 +228,18 @@ default void writeBlock( } } + @Override + default void writeShard( + final String path, + final DatasetAttributes datasetAttributes, + final Shard shard) throws N5Exception { + + if (!(datasetAttributes instanceof ShardedDatasetAttributes)) + throw new N5IOException("Can not write shard into non-sharded dataset " + path); + + // TODO implement me + } + @Override default boolean remove(final String path) throws N5Exception { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 96cbeff98..7423b57ea 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -35,6 +35,7 @@ import java.util.regex.Matcher; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -56,6 +57,8 @@ static Gson registerGson(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(ShardingConfiguration.class, + new ShardingConfiguration.ShardingConfigurationAdapter()); gsonBuilder.registerTypeHierarchyAdapter(Codec.class, new CodecAdapter()); gsonBuilder.disableHtmlEscaping(); return gsonBuilder.create(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 867044dba..6067b9f18 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -35,6 +35,10 @@ import java.util.Map; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; +import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration; +import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; /** * A simple structured container API for hierarchies of chunked @@ -208,6 +212,20 @@ default void createDataset( setDatasetAttributes(normalPath, datasetAttributes); } + default void createDataset( + final String datasetPath, + final long[] dimensions, + final int[] shardSize, + final int[] blockSize, + final DataType dataType, + final Compression compression) throws N5Exception { + + final Codec[] codecs = new Codec[]{new ShardingCodec( + new ShardingConfiguration(blockSize, null, null, IndexLocation.end))}; + + createDataset(datasetPath, new DatasetAttributes(dimensions, shardSize, dataType, compression, codecs)); + } + /** * Creates a dataset. This does not create any data but the path and * mandatory attributes only. @@ -266,6 +284,25 @@ void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception; + /** + * Writes a complete {@link Shard} to a dataset. + * + * @param datasetPath + * dataset path + * @param datasetAttributes + * the dataset attributes + * @param dataBlock + * the data block + * @param + * the data block data type + * @throws N5Exception + * if the requested dataset is not sharded + */ + void writeShard( + final String datasetPath, + final DatasetAttributes datasetAttributes, + final Shard shard) throws N5Exception; + /** * Deletes the block at {@code gridPosition} * From 352427e09285da36f2f7b45999b894143e9946da Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 23 Jul 2024 15:49:51 -0400 Subject: [PATCH 011/423] wip: more shard/codec work --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 6 +- .../org/janelia/saalfeldlab/n5/N5Writer.java | 2 +- .../n5/ShardedDatasetAttributes.java | 102 +++++++++++------ .../saalfeldlab/n5/shard/AbstractShard.java | 16 +-- .../saalfeldlab/n5/shard/InMemoryShard.java | 11 +- .../janelia/saalfeldlab/n5/shard/Shard.java | 30 +++-- .../saalfeldlab/n5/shard/ShardImpl.java | 20 ---- .../saalfeldlab/n5/shard/ShardIndex.java | 5 +- .../saalfeldlab/n5/shard/ShardReader.java | 33 ++++-- .../saalfeldlab/n5/shard/ShardWriter.java | 2 +- .../n5/shard/ShardingConfiguration.java | 13 ++- .../janelia/saalfeldlab/n5/shard/Shards.java | 106 ------------------ 12 files changed, 137 insertions(+), 209 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardImpl.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/Shards.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 9b8e288c0..2aeb21b1f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -31,7 +31,6 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.shard.Shard; -import org.janelia.saalfeldlab.n5.shard.Shards; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -88,11 +87,10 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { } - default Shard readShard(final String pathName, + default Shard getShard(final String pathName, final ShardedDatasetAttributes datasetAttributes, long... gridPosition) { - final Shards shards = new Shards(datasetAttributes); // TODO throw exception if this dataset is not sharded? return null; @@ -104,7 +102,7 @@ default DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { - if (ShardedDatasetAttributes.isSharded(datasetAttributes)) { + if (datasetAttributes instanceof ShardedDatasetAttributes) { // TODO } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 6067b9f18..f4645a802 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -221,7 +221,7 @@ default void createDataset( final Compression compression) throws N5Exception { final Codec[] codecs = new Codec[]{new ShardingCodec( - new ShardingConfiguration(blockSize, null, null, IndexLocation.end))}; + new ShardingConfiguration(blockSize, null, null, IndexLocation.END))}; createDataset(datasetPath, new DatasetAttributes(dimensions, shardSize, dataType, compression, codecs)); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index 9b69ad6e9..2825eff91 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -6,7 +6,6 @@ import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration; import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; -import org.janelia.saalfeldlab.n5.shard.Shards; public class ShardedDatasetAttributes extends DatasetAttributes { @@ -16,17 +15,8 @@ public class ShardedDatasetAttributes extends DatasetAttributes { private final IndexLocation indexLocation; - public ShardedDatasetAttributes(final long[] dimensions, final int[] shardSize, final DataType dataType, - final Compression compression, - final Codec[] codecs) { - - super(dimensions, getBlockSize(codecs), dataType, compression, codecs); - final ShardingConfiguration config = getShardConfiguration(codecs); - this.indexLocation = config.areIndexesAtStart() ? IndexLocation.start : IndexLocation.end; - this.shardSize = shardSize; - } - - public ShardedDatasetAttributes(final long[] dimensions, + public ShardedDatasetAttributes( + final long[] dimensions, final int[] shardSize, final int[] blockSize, final IndexLocation shardIndexLocation, @@ -37,7 +27,6 @@ public ShardedDatasetAttributes(final long[] dimensions, super(dimensions, blockSize, dataType, compression, codecs); this.shardSize = shardSize; this.indexLocation = shardIndexLocation; - // this.config = new ShardingConfiguration(blockSize, null, null, shardIndexLocation); // TODO figure out codecs } @@ -47,44 +36,85 @@ public int[] getShardSize() { return shardSize; } - public Shards getShards() { - - return new Shards(this); - } + /** + * Returns the number of blocks a shard contains along all dimensions. + * + * @return the size of the block grid of a shard + */ + public int[] getShardBlockGridSize() { - public ShardingConfiguration getShardingConfiguration() { + final int nd = getNumDimensions(); + final int[] shardBlockGridSize = new int[nd]; + final int[] blockSize = getBlockSize(); + for (int i = 0; i < nd; i++) + shardBlockGridSize[i] = (int)(Math.ceil((double)getDimensions()[i] / blockSize[i])); - return Arrays.stream(getCodecs()) - .filter(ShardingCodec::isShardingCodec) - .map(x -> { - return ((ShardingCodec)x).getConfiguration(); - }) - .findFirst().orElse(null); + return shardBlockGridSize; } - public static boolean isSharded(Codec[] codecs) { + /** + * Given a block's position relative to the array, returns the position of the shard containing that block relative to the shard grid. + * + * @param blockGridPosition + * position of a block relative to the array + * @return the position of the containing shard in the shard grid + */ + public long[] getShardPositionForBlock(final long... blockGridPosition) { + + // TODO have this return a shard + final int[] shardBlockDimensions = getShardBlockGridSize(); + final long[] shardGridPosition = new long[blockGridPosition.length]; + for (int i = 0; i < shardGridPosition.length; i++) { + shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / shardBlockDimensions[i]); + } + + return shardGridPosition; + } - return Arrays.stream(codecs).anyMatch(ShardingCodec::isShardingCodec); + /** + * Returns of the block at the given position relative to this shard, or null if this shard does not contain the given block. + * + * @return the shard position + */ + public int[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { + + final long[] shardPos = getShardPositionForBlock(blockPosition); + if (!Arrays.equals(shardPosition, shardPos)) + return null; + + final int[] shardSize = getShardSize(); + final int[] blkSize = getBlockSize(); + final int[] blkGridSize = getShardBlockGridSize(); + + final int[] blockShardPos = new int[shardSize.length]; + for (int i = 0; i < shardSize.length; i++) { + final long shardP = shardPos[i] * shardSize[i]; + final long blockP = blockPosition[i] * blkSize[i]; + blockShardPos[i] = (int)((blockP - shardP) / blkGridSize[i]); + } + + return blockShardPos; } - public static ShardingConfiguration getShardConfiguration(Codec[] codecs) { + /** + * @return the number of blocks per shard + */ + public long getNumBlocks() { - return Arrays.stream(codecs) - .filter(ShardingCodec::isShardingCodec) - .map(x -> { - return ((ShardingCodec)x).getConfiguration(); - }) - .findFirst().orElse(null); + return Arrays.stream(getShardBlockGridSize()).reduce(1, (x, y) -> x * y); } public static int[] getBlockSize(Codec[] codecs) { + //TODO Caleb: Move this? return Arrays.stream(codecs) .filter(ShardingCodec::isShardingCodec) - .map(x -> { - return ((ShardingCodec)x).getConfiguration(); - }) + .map(x -> ((ShardingCodec)x).getConfiguration()) .map(ShardingConfiguration::getBlockSize).findFirst().orElse(null); } + public IndexLocation getIndexLocation() { + + return indexLocation; + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java index 05d63144e..34237084f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java @@ -4,20 +4,22 @@ public abstract class AbstractShard implements Shard { - protected final long[] size; + protected final int[] size; protected final long[] gridPosition; protected final int[] blockSize; + private final ShardIndex index; - public AbstractShard(final long[] size, final long[] gridPosition, - final int[] blockSize, final T type) { + public AbstractShard(final int[] shardSize, final long[] gridPosition, + final int[] blockSize, final ShardIndex index) { - this.size = size; + this.size = shardSize; this.gridPosition = gridPosition; this.blockSize = blockSize; + this.index = index; } @Override - public long[] getSize() { + public int[] getSize() { return size; } @@ -41,9 +43,9 @@ public DataBlock getBlock(int... position) { } @Override - public ShardIndex getIndexes() { + public ShardIndex getIndex() { - return null; + return index; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 72f102b5f..977620e17 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -12,17 +12,10 @@ public class InMemoryShard extends AbstractShard { private ShardIndex shardIndex; - public InMemoryShard(final long[] size, final long[] gridPosition, final int[] blockSize, T type) { + public InMemoryShard(final int[] shardSize, final long[] gridPosition, final int[] blockSize, ShardIndex index) { - super(size, gridPosition, blockSize, type); + super(shardSize, gridPosition, blockSize, index); blocks = new ArrayList<>(); } - @Override - public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) { - - // TODO Auto-generated method stub - return null; - } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index a587f735c..716d6d4b9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -3,10 +3,12 @@ import java.util.Arrays; import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; public interface Shard { + long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; + /** * Returns the number of blocks this shard contains along all dimensions. * @@ -17,7 +19,7 @@ public interface Shard { */ default int[] getBlockGridSize() { - final long[] sz = getSize(); + final int[] sz = getSize(); final int[] blkSz = getBlockSize(); final int[] blockGridSize = new int[sz.length]; for (int i = 0; i < sz.length; i++) @@ -31,7 +33,7 @@ default int[] getBlockGridSize() { * * @return shard size */ - public long[] getSize(); + public int[] getSize(); /** * Returns the size of blocks in pixel units. @@ -60,7 +62,7 @@ default int[] getBlockPosition(long... blockPosition) { if (!Arrays.equals(getGridPosition(), shardPos)) return null; - final long[] shardSize = getSize(); + final int[] shardSize = getSize(); final int[] blkSize = getBlockSize(); final int[] blkGridSize = getBlockGridSize(); @@ -92,10 +94,25 @@ default long[] getShard(long... blockPosition) { public DataBlock getBlock(int... position); - public ShardIndex getIndexes(); + default DataBlock[] getAllBlocks(int... position) { + //TODO Caleb: Do we want this? + return null; + } + + public ShardIndex getIndex(); - public DataBlock readBlock(final String pathName, final DatasetAttributes datasetAttributes, long... gridPosition); + public static Shard createEmpty(final ShardedDatasetAttributes attributes, long... shardPosition) { + final long[] emptyIndex = new long[(int)(2 * attributes.getNumBlocks())]; + Arrays.fill(emptyIndex, EMPTY_INDEX_NBYTES); + final ShardIndex shardIndex = new ShardIndex(attributes.getShardBlockGridSize(), emptyIndex); + + return new InMemoryShard( + attributes.getShardSize(), + shardPosition, + attributes.getBlockSize(), + shardIndex); + } /** * Say we want async datablock access @@ -107,5 +124,4 @@ default long[] getShard(long... blockPosition) { * Shard doesn't hold the data directly, but is the metadata about how the blocks are stored * */ - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardImpl.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardImpl.java deleted file mode 100644 index 11803b3b9..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardImpl.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; - -public class ShardImpl extends AbstractShard { - - public ShardImpl(final long[] size, final long[] gridPosition, final int[] blockSize, T type) { - - super(size, gridPosition, blockSize, type); - } - - @Override - public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) { - - // TODO Auto-generated method stub - return null; - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index b2a19472f..113a8ad43 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -82,12 +82,11 @@ public static ShardIndex read(FileChannel channel, ShardedDatasetAttributes data // TODO need codecs // TODO FileChannel is too specific - generalize - final Shards shards = new Shards(datasetAttributes); - final int[] indexShape = indexBlockSize(shards.getShardBlockGridSize()); + final int[] indexShape = indexBlockSize(datasetAttributes.getShardBlockGridSize()); final int indexSize = (int)Arrays.stream(indexShape).reduce(1, (x, y) -> x * y); final int indexBytes = BYTES_PER_LONG * indexSize; - if (!datasetAttributes.getShardingConfiguration().areIndexesAtStart()) { + if (datasetAttributes.getIndexLocation() == ShardingConfiguration.IndexLocation.END) { channel.position(channel.size() - indexBytes); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java index d68a03c01..530bb9426 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java @@ -5,26 +5,28 @@ import java.nio.channels.Channels; import java.nio.channels.FileChannel; +import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.DefaultBlockReader; +import org.janelia.saalfeldlab.n5.N5FSReader; +import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.IdentityCodec; +import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; public class ShardReader { - private final ShardedDatasetAttributes datasetAttributes; private long[] indexes; - private Shards shards; public ShardReader(final ShardedDatasetAttributes datasetAttributes) { this.datasetAttributes = datasetAttributes; - this.shards = new Shards(datasetAttributes); } public ShardIndex readIndexes(FileChannel channel) throws IOException { @@ -36,8 +38,6 @@ public InMemoryShard readShardFully( final FileChannel channel, long... gridPosition) throws IOException { - final DatasetAttributes dsetAttrs = shards.getDatasetAttributes(); - final ShardIndex si = readIndexes(channel); return null; } @@ -52,7 +52,7 @@ public DataBlock readBlock( final ShardIndex index = readIndexes(in); - final long[] shardPosition = shards.getShardPositionForBlock(blockPosition); + final long[] shardPosition = datasetAttributes.getShardPositionForBlock(blockPosition); in.position(index.getOffset(shardPosition)); final InputStream is = Channels.newInputStream(in); return DefaultBlockReader.readBlock(is, datasetAttributes, indexes); @@ -60,7 +60,7 @@ public DataBlock readBlock( private long getIndexIndex(long... shardPosition) { - final int[] indexDimensions = shards.getShardBlockGridSize(); + final int[] indexDimensions = datasetAttributes.getShardBlockGridSize(); long idx = 0; for (int i = 0; i < indexDimensions.length; i++) { idx += shardPosition[i] * indexDimensions[i]; @@ -77,14 +77,27 @@ public static void main(String[] args) { System.out.println(reader.getIndexIndex(0, 1)); System.out.println(reader.getIndexIndex(1, 0)); System.out.println(reader.getIndexIndex(1, 1)); + + final N5Reader n5 = new N5FSReader("shard.n5"); + final ShardedDatasetAttributes datasetAttributes = buildTestAttributes(); + n5.readBlock("dataset", datasetAttributes, 0, 0, 0); + } private static ShardedDatasetAttributes buildTestAttributes() { final Codec[] codecs = new Codec[]{ - new ShardingCodec(new ShardingConfiguration(new int[]{2, 2}, null, null, IndexLocation.end))}; - - return new ShardedDatasetAttributes(new long[]{4, 4}, new int[]{2, 2}, DataType.INT32, new RawCompression(), codecs); + new IdentityCodec(), + new ShardingCodec( + new ShardingConfiguration( + new int[]{2, 2}, + new Codec[]{new Compression.CompressionCodec(new RawCompression()), new IdentityCodec()}, + new Codec[]{new Crc32cChecksumCodec()}, + IndexLocation.END) + ) + }; + + return new ShardedDatasetAttributes(new long[]{4, 4}, new int[]{2, 2}, new int[]{2, 2}, IndexLocation.END, DataType.INT32, new RawCompression(), codecs); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java index 5705d583d..24d82a4c6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java @@ -62,7 +62,7 @@ public void write(final OutputStream out) throws IOException { // } prepareForWritingDataBlock(); - if (datasetAttributes.getShardingConfiguration().areIndexesAtStart()) { + if (datasetAttributes.getIndexLocation() == ShardingConfiguration.IndexLocation.START) { writeIndexBlock(out); writeBlocks(out); } else { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingConfiguration.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingConfiguration.java index 2a29a1162..ce520700d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingConfiguration.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingConfiguration.java @@ -21,16 +21,19 @@ public class ShardingConfiguration { public static final String CODECS_KEY = "codecs"; public static final String INDEX_CODECS_KEY = "index_codecs"; - public static enum IndexLocation { - start, end - }; + public enum IndexLocation { + START, END + } protected int[] blockSize; protected Codec[] codecs; protected Codec[] indexCodecs; protected IndexLocation indexLocation; - public ShardingConfiguration(final int[] blockSize, final Codec[] codecs, final Codec[] indexCodecs, + public ShardingConfiguration( + final int[] blockSize, + final Codec[] codecs, + final Codec[] indexCodecs, final IndexLocation indexLocation) { this.blockSize = blockSize; @@ -46,7 +49,7 @@ public int[] getBlockSize() { public boolean areIndexesAtStart() { - return indexLocation == IndexLocation.start; + return indexLocation == IndexLocation.START; } public static class ShardingConfigurationAdapter diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shards.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shards.java deleted file mode 100644 index fb0f6aca9..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shards.java +++ /dev/null @@ -1,106 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import java.util.Arrays; - -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; - -/** - * Manages the set of shards that comprise a dataset. - */ -public class Shards { - - public static long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; - - private final ShardedDatasetAttributes datasetAttributes; - - public Shards(final ShardedDatasetAttributes datasetAttributes) { - - this.datasetAttributes = datasetAttributes; - } - - public ShardedDatasetAttributes getDatasetAttributes() { - - return datasetAttributes; - } - - public int[] getShardSize() { - - return getDatasetAttributes().getShardSize(); - } - - public int[] getBlockSize() { - - return getDatasetAttributes().getBlockSize(); - } - - /** - * Returns the number of blocks a shard contains along all dimensions. - * - * @return the size of the block grid of a shard - */ - public int[] getShardBlockGridSize() { - - final int nd = getDatasetAttributes().getNumDimensions(); - final int[] shardBlockGridSize = new int[nd]; - final int[] blockSize = getBlockSize(); - for (int i = 0; i < nd; i++) - shardBlockGridSize[i] = (int)(Math - .ceil((double)getDatasetAttributes().getDimensions()[i] / blockSize[i])); - - return shardBlockGridSize; - } - - /** - * Given a block's position relative to the array, returns the position of the shard containing that block relative to the shard grid. - * - * @param gridPosition - * position of a block relative to the array - * @return the position of the containing shard in the shard grid - */ - public long[] getShardPositionForBlock(final long... blockGridPosition) { - - // TODO have this return a shard - final int[] shardBlockDimensions = getShardBlockGridSize(); - final long[] shardGridPosition = new long[blockGridPosition.length]; - for (int i = 0; i < shardGridPosition.length; i++) { - shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / shardBlockDimensions[i]); - } - - return shardGridPosition; - } - - /** - * Returns of the block at the given position relative to this shard, or null if this shard does not contain the given block. - * - * @return the shard position - */ - public int[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { - - final long[] shardPos = getShardPositionForBlock(blockPosition); - if (!Arrays.equals(shardPosition, shardPos)) - return null; - - final int[] shardSize = getShardSize(); - final int[] blkSize = getBlockSize(); - final int[] blkGridSize = getShardBlockGridSize(); - - final int[] blockShardPos = new int[shardSize.length]; - for (int i = 0; i < shardSize.length; i++) { - final long shardP = shardPos[i] * shardSize[i]; - final long blockP = blockPosition[i] * blkSize[i]; - blockShardPos[i] = (int)((blockP - shardP) / blkGridSize[i]); - } - - return blockShardPos; - } - - /** - * @return the number of blocks per shard - */ - public long getNumBlocks() { - - return Arrays.stream(getShardBlockGridSize()).reduce(1, (x, y) -> x * y); - } - - -} From ce0b25dcf87d3eae0ffd9f0d441d6d470b827d93 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Jul 2024 12:00:55 -0400 Subject: [PATCH 012/423] feat/wip: add size partial read lockForReading methods --- .../n5/FileSystemKeyValueAccess.java | 32 +++++++++++++++++-- .../saalfeldlab/n5/KeyValueAccess.java | 8 +++++ .../janelia/saalfeldlab/n5/LockedChannel.java | 2 ++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index b7ea36d54..d5d38ffe0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -86,7 +86,8 @@ protected LockedFileChannel(final Path path, final boolean readOnly) throws IOEx this(path, readOnly, -1, -1); } - protected LockedFileChannel(final Path path, final boolean readOnly, final long startByte, final long lastByte) throws IOException { + protected LockedFileChannel(final Path path, final boolean readOnly, final long startByte, final long endByte) + throws IOException { final OpenOption[] options; if (readOnly) { @@ -106,10 +107,13 @@ protected LockedFileChannel(final Path path, final boolean readOnly, final long } } + if (startByte != 0) + channel.position(startByte); + for (boolean waiting = true; waiting;) { waiting = false; try { - channel.lock(0L, Long.MAX_VALUE, readOnly); + channel.lock(startByte, endByte, readOnly); } catch (final OverlappingFileLockException e) { waiting = true; try { @@ -122,6 +126,12 @@ protected LockedFileChannel(final Path path, final boolean readOnly, final long } } + @Override + public long size() throws IOException { + + return channel.size(); + } + @Override public Reader newReader() throws IOException { @@ -177,6 +187,17 @@ public LockedFileChannel lockForReading(final String normalPath) throws IOExcept } } + @Override + public LockedFileChannel lockForReading(final String normalPath, final long startByte, final long endByte) + throws IOException { + + try { + return new LockedFileChannel(normalPath, true, startByte, endByte); + } catch (final NoSuchFileException e) { + throw new N5Exception.N5NoSuchKeyException("No such file", e); + } + } + @Override public LockedFileChannel lockForWriting(final String normalPath) throws IOException { @@ -218,6 +239,13 @@ public boolean exists(final String normalPath) { return Files.exists(path); } + @Override + public long size(final String normalPath) throws IOException { + + final Path path = fileSystem.getPath(normalPath); + return Files.size(path); + } + @Override public String[] listDirectories(final String normalPath) throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index ea09269ba..8b6fe5aa4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -30,6 +30,8 @@ import java.net.URISyntaxException; import java.nio.file.FileSystem; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess.LockedFileChannel; + /** * Key value read primitives used by {@link N5KeyValueReader} * implementations. This interface implements a subset of access primitives @@ -117,6 +119,8 @@ public default String compose(final URI uri, final String... components) { */ public boolean exists(final String normalPath); + public long size(final String normalPath) throws IOException; + /** * Test whether the path is a directory. * @@ -155,6 +159,9 @@ public default String compose(final URI uri, final String... components) { */ public LockedChannel lockForReading(final String normalPath) throws IOException; + public LockedFileChannel lockForReading(String normalPath, final long startByte, final long endByte) + throws IOException; + /** * Create an exclusive lock on a path for writing. If the file doesn't * exist yet, it will be created, including all directories leading up to @@ -222,4 +229,5 @@ public default String compose(final URI uri, final String... components) { * if an error occurs during deletion */ public void delete(final String normalPath) throws IOException; + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java index bd34a59da..c3c53f500 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java @@ -40,6 +40,8 @@ */ public interface LockedChannel extends Closeable { + public long size() throws IOException; + /** * Create a UTF-8 {@link Reader}. * From 8aabcf43cad0a553f773349ab775c06c706e5c7c Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Jul 2024 12:04:20 -0400 Subject: [PATCH 013/423] wip/feat: ShardIndex progress, VirtualShard progress --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 14 +- .../saalfeldlab/n5/shard/AbstractShard.java | 28 ++-- .../saalfeldlab/n5/shard/InMemoryShard.java | 9 +- .../janelia/saalfeldlab/n5/shard/Shard.java | 26 ++-- .../saalfeldlab/n5/shard/ShardIndex.java | 122 ++++++++++++++---- .../saalfeldlab/n5/shard/ShardReader.java | 9 -- .../saalfeldlab/n5/shard/ShardWriter.java | 3 +- 7 files changed, 149 insertions(+), 62 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 2aeb21b1f..d12c6e7eb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -31,6 +31,7 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.VirtualShard; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -87,13 +88,13 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { } + @SuppressWarnings("rawtypes") default Shard getShard(final String pathName, final ShardedDatasetAttributes datasetAttributes, - long... gridPosition) { + long... shardGridPosition) { - // TODO throw exception if this dataset is not sharded? - - return null; + final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), shardGridPosition); + return new VirtualShard(datasetAttributes, shardGridPosition, getKeyValueAccess(), path); } @Override @@ -103,7 +104,10 @@ default DataBlock readBlock( final long... gridPosition) throws N5Exception { if (datasetAttributes instanceof ShardedDatasetAttributes) { - // TODO + final ShardedDatasetAttributes shardedAttrs = (ShardedDatasetAttributes)datasetAttributes; + final long[] shardPosition = shardedAttrs.getShardPositionForBlock(gridPosition); + final Shard shard = getShard(pathName, shardedAttrs, shardPosition); + return shard.getBlock(gridPosition); } final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), gridPosition); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java index 34237084f..270358b5f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java @@ -1,33 +1,39 @@ package org.janelia.saalfeldlab.n5.shard; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; public abstract class AbstractShard implements Shard { - protected final int[] size; - protected final long[] gridPosition; - protected final int[] blockSize; - private final ShardIndex index; + protected final ShardedDatasetAttributes datasetAttributes; - public AbstractShard(final int[] shardSize, final long[] gridPosition, - final int[] blockSize, final ShardIndex index) { + protected final ShardIndex index; - this.size = shardSize; + private final long[] gridPosition; + + public AbstractShard(final ShardedDatasetAttributes datasetAttributes, final long[] gridPosition, + final ShardIndex index) { + + this.datasetAttributes = datasetAttributes; this.gridPosition = gridPosition; - this.blockSize = blockSize; this.index = index; } + public ShardedDatasetAttributes getDatasetAttributes() { + + return datasetAttributes; + } + @Override public int[] getSize() { - return size; + return datasetAttributes.getShardSize(); } @Override public int[] getBlockSize() { - return blockSize; + return datasetAttributes.getShardSize(); } @Override @@ -37,7 +43,7 @@ public long[] getGridPosition() { } @Override - public DataBlock getBlock(int... position) { + public DataBlock getBlock(long... position) { return null; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 977620e17..346e69db8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -4,17 +4,16 @@ import java.util.List; import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; public class InMemoryShard extends AbstractShard { private List> blocks; - private ShardIndex shardIndex; + public InMemoryShard(final ShardedDatasetAttributes datasetAttributes, final long[] gridPosition, + ShardIndex index) { - public InMemoryShard(final int[] shardSize, final long[] gridPosition, final int[] blockSize, ShardIndex index) { - - super(shardSize, gridPosition, blockSize, index); + super(datasetAttributes, gridPosition, index); blocks = new ArrayList<>(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 716d6d4b9..27c3cd227 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -28,6 +28,8 @@ default int[] getBlockGridSize() { return blockGridSize; } + public ShardedDatasetAttributes getDatasetAttributes(); + /** * Returns the size of shards in pixel units. * @@ -56,7 +58,7 @@ default int[] getBlockGridSize() { * * @return the shard position */ - default int[] getBlockPosition(long... blockPosition) { + default long[] getBlockPosition(long... blockPosition) { final long[] shardPos = getShard(blockPosition); if (!Arrays.equals(getGridPosition(), shardPos)) @@ -66,7 +68,7 @@ default int[] getBlockPosition(long... blockPosition) { final int[] blkSize = getBlockSize(); final int[] blkGridSize = getBlockGridSize(); - final int[] blockShardPos = new int[shardSize.length]; + final long[] blockShardPos = new long[shardSize.length]; for (int i = 0; i < shardSize.length; i++) { final long shardP = shardPos[i] * shardSize[i]; final long blockP = blockPosition[i] * blkSize[i]; @@ -92,9 +94,9 @@ default long[] getShard(long... blockPosition) { return shardGridPosition; } - public DataBlock getBlock(int... position); + public DataBlock getBlock(long... position); - default DataBlock[] getAllBlocks(int... position) { + default DataBlock[] getAllBlocks(long... position) { //TODO Caleb: Do we want this? return null; } @@ -106,12 +108,18 @@ public static Shard createEmpty(final ShardedDatasetAttributes attributes final long[] emptyIndex = new long[(int)(2 * attributes.getNumBlocks())]; Arrays.fill(emptyIndex, EMPTY_INDEX_NBYTES); final ShardIndex shardIndex = new ShardIndex(attributes.getShardBlockGridSize(), emptyIndex); + return new InMemoryShard(attributes, shardPosition, shardIndex); + } - return new InMemoryShard( - attributes.getShardSize(), - shardPosition, - attributes.getBlockSize(), - shardIndex); + public static long flatIndex(long[] gridPosition, int[] gridSize) { + + long index = gridPosition[0]; + long cumSizes = gridSize[0]; + for (int i = 1; i < gridSize.length; i++) { + index += gridPosition[i] * cumSizes; + cumSizes *= gridSize[i]; + } + return index; } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 113a8ad43..95bd4b491 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -3,39 +3,40 @@ import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.util.Arrays; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; public class ShardIndex extends LongArrayDataBlock { private static final int BYTES_PER_LONG = 8; - public ShardIndex(int[] size, long[] data) { + private static final int LONGS_PER_BLOCK = 2; - super(indexBlockSize(size), new long[]{0}, data); - } - - public ShardIndex(int[] size) { - - super(indexBlockSize(size), new long[]{0}, createData(size)); - } + private static final long[] DUMMY_GRID_POSITION = null; - private static long[] createData(final int[] size) { + public ShardIndex(int[] shardBlockGridSize, long[] data) { - final int N = 2 * Arrays.stream(size).reduce(1, (x, y) -> x * y); - return new long[N]; + super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, data); } - private static int[] indexBlockSize(final int[] size) { + public ShardIndex(int[] shardBlockGridSize) { - final int[] indexBlockSize = new int[size.length + 1]; - indexBlockSize[0] = 2; - System.arraycopy(size, 0, indexBlockSize, 1, size.length); - return indexBlockSize; + super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, emptyIndexData(shardBlockGridSize)); } public long getOffset(long... gridPosition) { @@ -55,7 +56,6 @@ public void set(long offset, long nbytes, long[] gridPosition) { data[i + 1] = nbytes; } - private int getOffsetIndex(long... gridPosition) { int idx = 0; @@ -73,20 +73,83 @@ private int getNumBytesIndex(long... gridPosition) { return getOffsetIndex() + 1; } - public void printData() { + public static ShardIndex read(final KeyValueAccess keyValueAccess, final String key, + final ShardedDatasetAttributes datasetAttributes) throws IOException { + + return read(keyValueAccess, key, datasetAttributes.getShardBlockGridSize(), + datasetAttributes.getIndexLocation()); + } + + public static ShardIndex read(final KeyValueAccess keyValueAccess, final String key, final int[] shardBlockGridSize, + final IndexLocation indexLocation) throws IOException { + + final ShardIndex idx = new ShardIndex(shardBlockGridSize); + final IndexByteBounds byteBounds = byteBounds(idx.getSize(), indexLocation, keyValueAccess.size(key)); + try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(key, byteBounds.start, byteBounds.end)) { + + final byte[] bytes = new byte[idx.getNumElements() * ShardIndex.BYTES_PER_LONG]; + lockedChannel.newInputStream().read(bytes); + idx.readData(ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)); // TODO generalize byte order + return idx; + + } catch (final N5Exception.N5NoSuchKeyException e) { + return null; + } catch (final IOException | UncheckedIOException e) { + throw new N5IOException("Failed to read from " + key, e); + } + } + + public static DatasetAttributes indexDatasetAttributes(final int[] indexBlockSize) { + + final int[] blkSize = new int[indexBlockSize.length]; + final long[] size = new long[indexBlockSize.length]; + for (int i = 0; i < blkSize.length; i++) { + blkSize[i] = (int)indexBlockSize[i]; + } + + // TODO codecs + return new DatasetAttributes(size, blkSize, DataType.UINT64, new RawCompression(), null); + } + + public static IndexByteBounds byteBounds(ShardedDatasetAttributes datasetAttributes, final long objectSize) { - System.out.println(Arrays.toString(data)); + final int[] indexShape = prepend(2, datasetAttributes.getShardBlockGridSize()); + return byteBounds(indexShape, datasetAttributes.getIndexLocation(), objectSize); + } + + public static IndexByteBounds byteBounds(final int[] indexShape, final IndexLocation indexLocation, + final long objectSize) { + + final int indexSize = (int)Arrays.stream(indexShape).reduce(1, (x, y) -> x * y); + + if (indexLocation == IndexLocation.START) { + return new IndexByteBounds(0L, indexSize); + } else { + return new IndexByteBounds(objectSize - (BYTES_PER_LONG * indexSize), objectSize - 1); + } + } + + private static class IndexByteBounds { + + private final long start; + private final long end; + + private IndexByteBounds(long start, long end) { + + this.start = start; + this.end = end; + } } public static ShardIndex read(FileChannel channel, ShardedDatasetAttributes datasetAttributes) throws IOException { // TODO need codecs // TODO FileChannel is too specific - generalize - final int[] indexShape = indexBlockSize(datasetAttributes.getShardBlockGridSize()); + final int[] indexShape = prepend(2, datasetAttributes.getShardBlockGridSize()); final int indexSize = (int)Arrays.stream(indexShape).reduce(1, (x, y) -> x * y); final int indexBytes = BYTES_PER_LONG * indexSize; - if (datasetAttributes.getIndexLocation() == ShardingConfiguration.IndexLocation.END) { + if (datasetAttributes.getIndexLocation() == IndexLocation.END) { channel.position(channel.size() - indexBytes); } @@ -101,12 +164,27 @@ public static ShardIndex read(FileChannel channel, ShardedDatasetAttributes data return new ShardIndex(indexShape, indexes); } + private static long[] emptyIndexData(final int[] size) { + + final int N = 2 * Arrays.stream(size).reduce(1, (x, y) -> x * y); + final long[] data = new long[N]; + Arrays.fill(data, Shard.EMPTY_INDEX_NBYTES); + return data; + } + + private static int[] prepend(final int value, final int[] array) { + + final int[] indexBlockSize = new int[array.length + 1]; + indexBlockSize[0] = value; + System.arraycopy(array, 0, indexBlockSize, 1, array.length); + return indexBlockSize; + } + public static void main(String[] args) { final ShardIndex ib = new ShardIndex(new int[]{2, 2}); ib.set(8, 9, new long[]{1, 1}); - ib.printData(); // System.out.println(ib.getIndex(0, 0)); // System.out.println(ib.getIndex(1, 0)); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java index 530bb9426..ed4e71288 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java @@ -8,7 +8,6 @@ import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.DefaultBlockReader; import org.janelia.saalfeldlab.n5.N5FSReader; import org.janelia.saalfeldlab.n5.N5Reader; @@ -34,14 +33,6 @@ public ShardIndex readIndexes(FileChannel channel) throws IOException { return ShardIndex.read(channel, datasetAttributes); } - public InMemoryShard readShardFully( - final FileChannel channel, - long... gridPosition) throws IOException { - - final ShardIndex si = readIndexes(channel); - return null; - } - public DataBlock readBlock( final FileChannel in, long... blockPosition) throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java index 24d82a4c6..d1ade67f4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java @@ -6,6 +6,7 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.janelia.saalfeldlab.n5.DataBlock; @@ -93,7 +94,7 @@ private void prepareForWritingDataBlock() throws IOException { blockBytes.add(blockOut.toByteArray()); } - indexData.printData(); + System.out.println(Arrays.toString(indexData.getData())); } private void prepareForWriting() throws IOException { From 77c096d96ccead5050bcae516e07fb12470bd67f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Jul 2024 12:41:56 -0400 Subject: [PATCH 014/423] feat/wip: add VirtualShard --- .../saalfeldlab/n5/shard/VirtualShard.java | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java new file mode 100644 index 000000000..2dab08cc2 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -0,0 +1,70 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.DefaultBlockReader; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; + +public class VirtualShard extends AbstractShard { + + private KeyValueAccess keyValueAccess; + private String path; + + public VirtualShard(final ShardedDatasetAttributes datasetAttributes, long[] gridPosition, + final KeyValueAccess keyValueAccess, final String path) { + + super(datasetAttributes, gridPosition, null); + this.keyValueAccess = keyValueAccess; + this.path = path; + } + + @SuppressWarnings("unchecked") + @Override + public DataBlock getBlock(long... blockGridPosition) { + + final long[] relativePosition = getBlockPosition(blockGridPosition); + final ShardIndex idx = getIndex(); + final long startByte = idx.getOffset(relativePosition); + final long endByte = startByte + idx.getNumBytes(relativePosition); + try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path, startByte, endByte)) { + + // TODO add codecs, generalize to use any BlockReader + final DataBlock dataBlock = (DataBlock)datasetAttributes.getDataType().createDataBlock( + datasetAttributes.getBlockSize(), + blockGridPosition, + numBlockElements(datasetAttributes)); + + DefaultBlockReader.readFromStream(dataBlock, lockedChannel.newInputStream()); + return dataBlock; + + } catch (final N5Exception.N5NoSuchKeyException e) { + return null; + } catch (final IOException | UncheckedIOException e) { + throw new N5IOException("Failed to read block from " + path, e); + } + } + + private static int numBlockElements(DatasetAttributes datasetAttributes) { + + return Arrays.stream(datasetAttributes.getBlockSize()).reduce(1, (x, y) -> x * y); + } + + @Override + public ShardIndex getIndex() { + + try { + return ShardIndex.read(keyValueAccess, path, datasetAttributes); + } catch (final IOException e) { + throw new N5IOException("Failed to read index at " + path, e); + } + } + +} From 6874c39981c6341751cbc3f1a6828e798306f395 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Jul 2024 13:27:08 -0400 Subject: [PATCH 015/423] test: reading a zarr shard demo --- .../saalfeldlab/n5/shard/ShardDemos.java | 35 ++++++++++++ .../shardExamples/test.zarr/mid_sharded/c/0/0 | Bin 0 -> 88 bytes .../test.zarr/mid_sharded/zarr.json | 54 ++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java create mode 100644 src/test/resources/shardExamples/test.zarr/mid_sharded/c/0/0 create mode 100644 src/test/resources/shardExamples/test.zarr/mid_sharded/zarr.json diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java new file mode 100644 index 000000000..d969a3aba --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -0,0 +1,35 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.net.MalformedURLException; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; + +public class ShardDemos { + + public static void main(String[] args) throws MalformedURLException { + + final Path p = Paths.get("src/test/resources/shardExamples/test.zarr/mid_sharded/c/0/0"); + System.out.println(p); + + final String key = p.toString(); + final ShardedDatasetAttributes dsetAttrs = new ShardedDatasetAttributes(new long[]{6, 4}, new int[]{6, 4}, + new int[]{3, 2}, IndexLocation.END, DataType.UINT8, new RawCompression(), null); + + final FileSystemKeyValueAccess kva = new FileSystemKeyValueAccess(FileSystems.getDefault()); + + final VirtualShard shard = new VirtualShard(dsetAttrs, new long[]{0, 0}, kva, key); + final DataBlock blk00 = shard.getBlock(0, 0); + System.out.println(Arrays.toString((byte[])blk00.getData())); + + } + +} diff --git a/src/test/resources/shardExamples/test.zarr/mid_sharded/c/0/0 b/src/test/resources/shardExamples/test.zarr/mid_sharded/c/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..eafe18073dbcea989cb3bb6ee0d9246357a15a45 GIT binary patch literal 88 tcmZQzWMX6I;Nj&H5*A@*VddoF<`)nY6%%KG0yZekgT{x6!)PHic>q@K0Z{+| literal 0 HcmV?d00001 diff --git a/src/test/resources/shardExamples/test.zarr/mid_sharded/zarr.json b/src/test/resources/shardExamples/test.zarr/mid_sharded/zarr.json new file mode 100644 index 000000000..a80cb9d96 --- /dev/null +++ b/src/test/resources/shardExamples/test.zarr/mid_sharded/zarr.json @@ -0,0 +1,54 @@ +{ + "shape": [ + 4, + 6 + ], + "fill_value": 0, + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 4, + 6 + ] + } + }, + "attributes": {}, + "zarr_format": 3, + "data_type": "uint8", + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2, + 3 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + } + ], + "index_location": "end" + } + } + ], + "node_type": "array" +} From be870132f371e9cb2feb9603f03c461d49f29644 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Jul 2024 17:25:28 -0400 Subject: [PATCH 016/423] feat: make partial writes possible for key value access --- .../n5/FileSystemKeyValueAccess.java | 26 ++++++++++++++++--- .../saalfeldlab/n5/KeyValueAccess.java | 3 +++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index d5d38ffe0..fdaae9e01 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -71,6 +71,8 @@ protected class LockedFileChannel implements LockedChannel { protected final FileChannel channel; + protected final boolean truncate; + protected LockedFileChannel(final String path, final boolean readOnly) throws IOException { this(fileSystem.getPath(path), readOnly, -1, -1); @@ -89,6 +91,11 @@ protected LockedFileChannel(final Path path, final boolean readOnly) throws IOEx protected LockedFileChannel(final Path path, final boolean readOnly, final long startByte, final long endByte) throws IOException { + truncate = (startByte < 0 && endByte < 0); + + final long start = startByte < 0 ? 0L : startByte; + final long end = endByte < 0 ? Long.MAX_VALUE : endByte; + final OpenOption[] options; if (readOnly) { options = new OpenOption[]{StandardOpenOption.READ}; @@ -108,12 +115,12 @@ protected LockedFileChannel(final Path path, final boolean readOnly, final long } if (startByte != 0) - channel.position(startByte); + channel.position(start); for (boolean waiting = true; waiting;) { waiting = false; try { - channel.lock(startByte, endByte, readOnly); + channel.lock(start, end, readOnly); } catch (final OverlappingFileLockException e) { waiting = true; try { @@ -141,7 +148,9 @@ public Reader newReader() throws IOException { @Override public Writer newWriter() throws IOException { - channel.truncate(0); + if (truncate) + channel.truncate(0); + return Channels.newWriter(channel, StandardCharsets.UTF_8.name()); } @@ -154,7 +163,9 @@ public InputStream newInputStream() throws IOException { @Override public OutputStream newOutputStream() throws IOException { - channel.truncate(0); + if (truncate) + channel.truncate(0); + return Channels.newOutputStream(channel); } @@ -204,6 +215,13 @@ public LockedFileChannel lockForWriting(final String normalPath) throws IOExcept return new LockedFileChannel(normalPath, false); } + @Override + public LockedFileChannel lockForWriting(final String normalPath, final long startByte, final long endByte) + throws IOException { + + return new LockedFileChannel(normalPath, false, startByte, endByte); + } + public LockedFileChannel lockForReading(final Path path) throws IOException { try { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 8b6fe5aa4..bd20d575b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -182,6 +182,9 @@ public LockedFileChannel lockForReading(String normalPath, final long startByte, */ public LockedChannel lockForWriting(final String normalPath) throws IOException; + public LockedFileChannel lockForWriting(String normalPath, final long startByte, final long endByte) + throws IOException; + /** * List all 'directory'-like children of a path. * From c5ee84dc59ee86150cbdee6a929c786e391b4340 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Jul 2024 17:26:44 -0400 Subject: [PATCH 017/423] fix: AbstractShard getBlockSize --- .../java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java index 270358b5f..f642075a4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java @@ -19,6 +19,7 @@ public AbstractShard(final ShardedDatasetAttributes datasetAttributes, final lon this.index = index; } + @Override public ShardedDatasetAttributes getDatasetAttributes() { return datasetAttributes; @@ -33,7 +34,7 @@ public int[] getSize() { @Override public int[] getBlockSize() { - return datasetAttributes.getShardSize(); + return datasetAttributes.getBlockSize(); } @Override From 0b4d73c33d38e9a5853ba64e893a14d7493d2289 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Jul 2024 17:31:24 -0400 Subject: [PATCH 018/423] wip: toward block writing through shard --- .../saalfeldlab/n5/shard/InMemoryShard.java | 14 +++++++ .../janelia/saalfeldlab/n5/shard/Shard.java | 6 ++- .../saalfeldlab/n5/shard/ShardIndex.java | 29 +++++++++++++- .../saalfeldlab/n5/shard/VirtualShard.java | 39 +++++++++++++++++++ .../saalfeldlab/n5/shard/ShardDemos.java | 19 +++++++-- 5 files changed, 102 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 346e69db8..baacc8c58 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -17,4 +17,18 @@ public InMemoryShard(final ShardedDatasetAttributes datasetAttributes, final lon blocks = new ArrayList<>(); } + @Override + public void writeBlock(DataBlock block) { + + // TODO Auto-generated method stub + + } + + @Override + public void writeShard() { + + // TODO Auto-generated method stub + + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 27c3cd227..43a90ca6f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -88,7 +88,7 @@ default long[] getShard(long... blockPosition) { final int[] shardBlockDimensions = getBlockGridSize(); final long[] shardGridPosition = new long[shardBlockDimensions.length]; for (int i = 0; i < shardGridPosition.length; i++) { - shardGridPosition[i] = (long)Math.floor((double)blockPosition[i] / shardBlockDimensions[i]); + shardGridPosition[i] = (long)Math.floor((double)(blockPosition[i]) / shardBlockDimensions[i]); } return shardGridPosition; @@ -96,6 +96,10 @@ default long[] getShard(long... blockPosition) { public DataBlock getBlock(long... position); + public void writeBlock(DataBlock block); + + public void writeShard(); + default DataBlock[] getAllBlocks(long... position) { //TODO Caleb: Do we want this? return null; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 95bd4b491..1d0da8a49 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -3,6 +3,7 @@ import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -39,6 +40,12 @@ public ShardIndex(int[] shardBlockGridSize) { super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, emptyIndexData(shardBlockGridSize)); } + public boolean exists(long... gridPosition) { + + return getOffset(gridPosition) != Shard.EMPTY_INDEX_NBYTES && + getNumBytes(gridPosition) != Shard.EMPTY_INDEX_NBYTES; + } + public long getOffset(long... gridPosition) { return data[getOffsetIndex(gridPosition)]; @@ -80,7 +87,10 @@ public static ShardIndex read(final KeyValueAccess keyValueAccess, final String datasetAttributes.getIndexLocation()); } - public static ShardIndex read(final KeyValueAccess keyValueAccess, final String key, final int[] shardBlockGridSize, + public static ShardIndex read( + final KeyValueAccess keyValueAccess, + final String key, + final int[] shardBlockGridSize, final IndexLocation indexLocation) throws IOException { final ShardIndex idx = new ShardIndex(shardBlockGridSize); @@ -99,6 +109,23 @@ public static ShardIndex read(final KeyValueAccess keyValueAccess, final String } } + public static void write(ShardIndex index, + final KeyValueAccess keyValueAccess, + final String key, + final int[] shardBlockGridSize, + final IndexLocation indexLocation) throws IOException { + + final IndexByteBounds byteBounds = byteBounds(index.getSize(), indexLocation, keyValueAccess.size(key)); + try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(key, byteBounds.start, byteBounds.end)) { + + final OutputStream os = lockedChannel.newOutputStream(); + os.write(index.toByteBuffer().array()); + + } catch (final IOException | UncheckedIOException e) { + throw new N5IOException("Failed to read from " + key, e); + } + } + public static DatasetAttributes indexDatasetAttributes(final int[] indexBlockSize) { final int[] blkSize = new int[indexBlockSize.length]; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 2dab08cc2..fd2f38543 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -31,6 +31,9 @@ public VirtualShard(final ShardedDatasetAttributes datasetAttributes, long[] gri public DataBlock getBlock(long... blockGridPosition) { final long[] relativePosition = getBlockPosition(blockGridPosition); + if (relativePosition == null) + throw new N5IOException("Attempted to read a block from the wrong shard."); + final ShardIndex idx = getIndex(); final long startByte = idx.getOffset(relativePosition); final long endByte = startByte + idx.getNumBytes(relativePosition); @@ -52,6 +55,42 @@ public DataBlock getBlock(long... blockGridPosition) { } } + @Override + public void writeBlock(final DataBlock block) { + + final long[] relativePosition = getBlockPosition(block.getGridPosition()); + if (relativePosition == null) + throw new N5IOException("Attempted to write block in the wrong shard."); + + final ShardIndex idx = getIndex(); + final long startByte = idx.getOffset(relativePosition); + final long endByte = startByte + idx.getNumBytes(relativePosition); + + // TODO this assumes that the block exists in the shard and + // that the available space is sufficient. Should generalize + + // // A starting point: + // if (!idx.exists(block.getGridPosition())) { + // + // } + try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(path, startByte, endByte)) { + + // TODO codecs + datasetAttributes.getCompression().getWriter().write(block, lockedChannel.newOutputStream()); + + // TODO update index when we know how many bytes were written + + } catch (final IOException | UncheckedIOException e) { + throw new N5IOException("Failed to read block from " + path, e); + } + + } + + @Override + public void writeShard() { + + } + private static int numBlockElements(DatasetAttributes datasetAttributes) { return Arrays.stream(datasetAttributes.getBlockSize()).reduce(1, (x, y) -> x * y); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java index d969a3aba..2568eb6ee 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -20,16 +20,29 @@ public static void main(String[] args) throws MalformedURLException { final Path p = Paths.get("src/test/resources/shardExamples/test.zarr/mid_sharded/c/0/0"); System.out.println(p); + final String key = p.toString(); final ShardedDatasetAttributes dsetAttrs = new ShardedDatasetAttributes(new long[]{6, 4}, new int[]{6, 4}, new int[]{3, 2}, IndexLocation.END, DataType.UINT8, new RawCompression(), null); final FileSystemKeyValueAccess kva = new FileSystemKeyValueAccess(FileSystems.getDefault()); + final VirtualShard shard = new VirtualShard<>(dsetAttrs, new long[]{0, 0}, kva, key); + + final DataBlock blk = shard.getBlock(0, 0); + + final byte[] data = (byte[])blk.getData(); + System.out.println(Arrays.toString(data)); + + // fill the block with a weird value + Arrays.fill(data, (byte)123); - final VirtualShard shard = new VirtualShard(dsetAttrs, new long[]{0, 0}, kva, key); - final DataBlock blk00 = shard.getBlock(0, 0); - System.out.println(Arrays.toString((byte[])blk00.getData())); + // write the block + shard.writeBlock(blk); + // re-read the block and check the data it contains + final DataBlock blkReread = shard.getBlock(0, 0); + final byte[] dataReRead = (byte[])blkReread.getData(); + System.out.println(Arrays.toString(dataReRead)); } } From 083cb548e5c68af48d63f7ac3f2b370360f267fb Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 1 Aug 2024 14:49:43 -0400 Subject: [PATCH 019/423] fix: partial write defaults with 0 test,build: include n5-universe for tests --- pom.xml | 27 ++++++++++++------ .../n5/FileSystemKeyValueAccess.java | 18 ++++++------ .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 28 +++++++++++++++++++ 3 files changed, 57 insertions(+), 16 deletions(-) diff --git a/pom.xml b/pom.xml index 8b683eb44..be2913baf 100644 --- a/pom.xml +++ b/pom.xml @@ -161,6 +161,14 @@ com.google.code.gson gson + + org.scijava + scijava-common + + + org.apache.commons + commons-compress + @@ -168,6 +176,17 @@ junit test + + org.janelia.saalfeldlab + n5-universe + + + org.janelia.saalfeldlab + n5 + + + test + net.imagej ij @@ -194,14 +213,6 @@ ${commons-collections4.version} test - - org.scijava - scijava-common - - - org.apache.commons - commons-compress - diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index fdaae9e01..967cb6e2b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -75,26 +75,28 @@ protected class LockedFileChannel implements LockedChannel { protected LockedFileChannel(final String path, final boolean readOnly) throws IOException { - this(fileSystem.getPath(path), readOnly, -1, -1); + this(fileSystem.getPath(path), readOnly, 0, 0); } - protected LockedFileChannel(final String path, final boolean readOnly, final long startByte, final long lastByte) throws IOException { + protected LockedFileChannel(final String path, final boolean readOnly, final long startByte, final long size) throws IOException { - this(fileSystem.getPath(path), readOnly, startByte, lastByte); + this(fileSystem.getPath(path), readOnly, startByte, size); } protected LockedFileChannel(final Path path, final boolean readOnly) throws IOException { - this(path, readOnly, -1, -1); + this(path, readOnly, 0, 0); } - protected LockedFileChannel(final Path path, final boolean readOnly, final long startByte, final long endByte) + protected LockedFileChannel(final Path path, final boolean readOnly, final long startByte, final long size) throws IOException { - truncate = (startByte < 0 && endByte < 0); final long start = startByte < 0 ? 0L : startByte; - final long end = endByte < 0 ? Long.MAX_VALUE : endByte; + final long len = size < 0 ? 0L : size; + + //TODO Caleb: How does this handle if manually overwriting the entire file? (e.g. len > file size) + truncate = (start == 0 && len == 0); final OpenOption[] options; if (readOnly) { @@ -120,7 +122,7 @@ protected LockedFileChannel(final Path path, final boolean readOnly, final long for (boolean waiting = true; waiting;) { waiting = false; try { - channel.lock(start, end, readOnly); + channel.lock(start, len, readOnly); } catch (final OverlappingFileLockException e) { waiting = true; try { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index d12c6e7eb..3f44eebcb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -163,6 +163,34 @@ default String absoluteDataBlockPath( return getKeyValueAccess().compose(getURI(), components); } + /** + * Constructs the path for a shard in a dataset at a given grid position. + *

      + * The returned path is + * + * 

      +	 * $basePath/datasetPathName/$shardPosition[0]/$shardPosition[1]/.../$shardPosition[n]
+	 *
      + *

      + * This is the file into which the shard will be stored. + * + * @param normalPath normalized dataset path + * @param shardGridPosition to the target shard + * @return the absolute path to the shard at shardGridPosition + */ + default String absoluteShardPath( + final String normalPath, + final long... shardGridPosition) { + + final String[] components = new String[shardGridPosition.length + 1]; + components[0] = normalPath; + int i = 0; + for (final long p : shardGridPosition) + components[++i] = Long.toString(p); + + return getKeyValueAccess().compose(getURI(), components); + } + /** * Constructs the absolute path (in terms of this store) for the group or From bdadb582ec849dbc65497bb465dc19d28b690a0c Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 2 Aug 2024 16:17:23 -0400 Subject: [PATCH 020/423] wip: zone serialization and sharding? --- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 11 ++ .../saalfeldlab/n5/shard/ShardDemos.java | 120 +++++++++++++++++- .../test.zarr/mid_sharded/attributes.json | 54 ++++++++ .../shardExamples/test.zarr/mid_sharded/c/0/0 | Bin 88 -> 88 bytes 4 files changed, 179 insertions(+), 6 deletions(-) create mode 100644 src/test/resources/shardExamples/test.zarr/mid_sharded/attributes.json diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 9e5253d76..69e77fc21 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -39,6 +39,7 @@ import com.google.gson.JsonNull; import com.google.gson.JsonObject; import com.google.gson.JsonSyntaxException; +import org.janelia.saalfeldlab.n5.shard.VirtualShard; /** * Default implementation of {@link N5Writer} with JSON attributes parsed with @@ -217,6 +218,16 @@ default void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception { + /* Delegate to shard for writing block? How to know what type of shard? */ + if (datasetAttributes instanceof ShardedDatasetAttributes) { + ShardedDatasetAttributes shardDatasetAttrs = (ShardedDatasetAttributes)datasetAttributes; + final long[] shardPos = shardDatasetAttrs.getShardPositionForBlock(dataBlock.getGridPosition()); + final String shardPath = absoluteShardPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition()); + final VirtualShard shard = new VirtualShard<>(shardDatasetAttrs, shardPos, getKeyValueAccess(), shardPath); + shard.writeBlock(dataBlock); + return; + } + final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition()); try (final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath)) { DefaultBlockWriter.writeBlockWithCodecs(lock.newOutputStream(), datasetAttributes, dataBlock); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java index 2568eb6ee..31491634b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -1,17 +1,34 @@ package org.janelia.saalfeldlab.n5.shard; -import java.net.MalformedURLException; -import java.nio.file.FileSystems; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; - +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.IdentityCodec; +import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; +import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.junit.Test; + +import java.lang.reflect.Type; +import java.net.MalformedURLException; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; public class ShardDemos { @@ -45,4 +62,95 @@ public static void main(String[] args) throws MalformedURLException { System.out.println(Arrays.toString(dataReRead)); } + @Test + public void writeReadBlockTest() { + + final N5Writer writer = N5Factory.createWriter("src/test/resources/shardExamples/test.n5"); + + final ShardedDatasetAttributes datasetAttributes = new ShardedDatasetAttributes( + new long[]{8, 8}, + new int[]{4, 4}, + new int[]{2, 2}, + IndexLocation.END, + DataType.UINT8, + new RawCompression(), + new Codec[]{ + new IdentityCodec(), + new ShardingCodec( + new ShardingConfiguration( + new int[]{2, 2}, + new Codec[]{new Compression.CompressionCodec(new RawCompression()), new IdentityCodec()}, + new Codec[]{new Crc32cChecksumCodec()}, + IndexLocation.END) + ) + } + ); + writer.createDataset("shard", datasetAttributes); + final DataBlock dataBlock = datasetAttributes.getDataType().createDataBlock(datasetAttributes.getBlockSize(), new long[]{0, 0}, 2 * 2); + + writer.writeBlock("shard", datasetAttributes, dataBlock); + writer.readBlock("shard", datasetAttributes, 0,0); + } + + private static class ZarrConfig { + final String name; + final T configuration; + + private ZarrConfig() { + name = ""; + configuration = null; + } + } + + private class GridConfig extends ZarrConfig {} + private class KeyEncodingConfig extends ZarrConfig {} + + private class ZarrChunk {} + + private class ZarrChunkAdapter implements com.google.gson.JsonSerializer, com.google.gson.JsonDeserializer { + final ZarrConfig grid; + final ZarrConfig keyEncoding; + + public ZarrChunkAdapter() { + grid = null; + keyEncoding = null; + } + public ZarrChunkAdapter(ZarrConfig grid, ZarrConfig key_encoding) { + + this.grid = grid; + this.keyEncoding = key_encoding; + } + + @Override public ZarrChunk deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { + + if (!json.isJsonObject()) return null; + + final JsonObject obj = json.getAsJsonObject(); + final JsonObject grid = obj.getAsJsonObject("chunk_grid"); + + return null; + } + + @Override public JsonElement serialize(ZarrChunk src, Type typeOfSrc, JsonSerializationContext context) { + + return null; + } + } + + + @Test + public void nameConfigurationGsonTest() { + + final N5Factory factory = new N5Factory(); + final GsonBuilder gson = new GsonBuilder(); + + + gson.registerTypeHierarchyAdapter() + factory.gsonBuilder(gson); + final N5Reader n5 = factory.openReader("src/test/resources/shardExamples/test.zarr/mid_sharded"); + + final JsonObject zarrJson = n5.getAttribute("/", "/", JsonObject.class); + zarrJson.remove("shard") + } + } diff --git a/src/test/resources/shardExamples/test.zarr/mid_sharded/attributes.json b/src/test/resources/shardExamples/test.zarr/mid_sharded/attributes.json new file mode 100644 index 000000000..a80cb9d96 --- /dev/null +++ b/src/test/resources/shardExamples/test.zarr/mid_sharded/attributes.json @@ -0,0 +1,54 @@ +{ + "shape": [ + 4, + 6 + ], + "fill_value": 0, + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 4, + 6 + ] + } + }, + "attributes": {}, + "zarr_format": 3, + "data_type": "uint8", + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2, + 3 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + } + ], + "index_location": "end" + } + } + ], + "node_type": "array" +} diff --git a/src/test/resources/shardExamples/test.zarr/mid_sharded/c/0/0 b/src/test/resources/shardExamples/test.zarr/mid_sharded/c/0/0 index eafe18073dbcea989cb3bb6ee0d9246357a15a45..19ad91e1627a7c0e559a2b3e7fb030cbb62f9934 100644 GIT binary patch delta 12 Pcma!uV5 Date: Fri, 2 Aug 2024 16:23:22 -0400 Subject: [PATCH 021/423] wip/feat: add VirtualShard.createIndex * edit behavior of getIndex --- .../saalfeldlab/n5/shard/VirtualShard.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index fd2f38543..699c45bf0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -68,11 +68,6 @@ public void writeBlock(final DataBlock block) { // TODO this assumes that the block exists in the shard and // that the available space is sufficient. Should generalize - - // // A starting point: - // if (!idx.exists(block.getGridPosition())) { - // - // } try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(path, startByte, endByte)) { // TODO codecs @@ -89,6 +84,7 @@ public void writeBlock(final DataBlock block) { @Override public void writeShard() { + // TODO } private static int numBlockElements(DatasetAttributes datasetAttributes) { @@ -96,11 +92,18 @@ private static int numBlockElements(DatasetAttributes datasetAttributes) { return Arrays.stream(datasetAttributes.getBlockSize()).reduce(1, (x, y) -> x * y); } + public ShardIndex createIndex() { + + // Empty index of the correct size + return new ShardIndex(datasetAttributes.getShardBlockGridSize()); + } + @Override public ShardIndex getIndex() { try { - return ShardIndex.read(keyValueAccess, path, datasetAttributes); + final ShardIndex result = ShardIndex.read(keyValueAccess, path, datasetAttributes); + return result == null ? createIndex() : result; } catch (final IOException e) { throw new N5IOException("Failed to read index at " + path, e); } From 1931530a758e87350fc29698eee9ce304bf383aa Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 2 Aug 2024 16:23:58 -0400 Subject: [PATCH 022/423] fix: return KVA return for types for ranged lockForReading/Writing --- .../java/org/janelia/saalfeldlab/n5/KeyValueAccess.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index bd20d575b..d199bf670 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -30,8 +30,6 @@ import java.net.URISyntaxException; import java.nio.file.FileSystem; -import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess.LockedFileChannel; - /** * Key value read primitives used by {@link N5KeyValueReader} * implementations. This interface implements a subset of access primitives @@ -159,7 +157,7 @@ public default String compose(final URI uri, final String... components) { */ public LockedChannel lockForReading(final String normalPath) throws IOException; - public LockedFileChannel lockForReading(String normalPath, final long startByte, final long endByte) + public LockedChannel lockForReading(String normalPath, final long startByte, final long endByte) throws IOException; /** @@ -182,7 +180,7 @@ public LockedFileChannel lockForReading(String normalPath, final long startByte, */ public LockedChannel lockForWriting(final String normalPath) throws IOException; - public LockedFileChannel lockForWriting(String normalPath, final long startByte, final long endByte) + public LockedChannel lockForWriting(String normalPath, final long startByte, final long endByte) throws IOException; /** From 786ec1fb5545319167d860465282a7b5f313a8c5 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 5 Aug 2024 12:50:36 -0400 Subject: [PATCH 023/423] feat: add getAttributesKey * this may make more sense in a more basic interface --- .../n5/CachedGsonKeyValueN5Reader.java | 16 +++++++++------- .../n5/CachedGsonKeyValueN5Writer.java | 10 +++++----- .../janelia/saalfeldlab/n5/N5KeyValueReader.java | 6 ++++++ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index 812c6ebb0..4cdb8d92f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -27,7 +27,6 @@ import java.lang.reflect.Type; -import com.google.gson.JsonSyntaxException; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; import org.janelia.saalfeldlab.n5.cache.N5JsonCacheableContainer; @@ -35,6 +34,7 @@ import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import com.google.gson.JsonSyntaxException; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -52,6 +52,8 @@ default N5JsonCache newCache() { N5JsonCache getCache(); + public String getAttributesKey(); + @Override default JsonElement getAttributesFromContainer(final String normalPathName, final String normalCacheKey) { @@ -70,7 +72,7 @@ default DatasetAttributes getDatasetAttributes(final String pathName) { return null; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPath, N5KeyValueReader.ATTRIBUTES_JSON); + attributes = getCache().getAttributes(normalPath, getAttributesKey()); } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPath); } @@ -96,7 +98,7 @@ default T getAttribute( final JsonElement attributes; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + attributes = getCache().getAttributes(normalPathName, getAttributesKey()); } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName); } @@ -117,7 +119,7 @@ default T getAttribute( final String normalizedAttributePath = N5URI.normalizeAttributePath(key); JsonElement attributes; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + attributes = getCache().getAttributes(normalPathName, getAttributesKey()); } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName); } @@ -133,7 +135,7 @@ default boolean exists(final String pathName) { final String normalPathName = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) - return getCache().isGroup(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + return getCache().isGroup(normalPathName, getAttributesKey()); else { return existsFromContainer(normalPathName, null); } @@ -176,7 +178,7 @@ default boolean datasetExists(final String pathName) throws N5IOException { final String normalPathName = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) { - return getCache().isDataset(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + return getCache().isDataset(normalPathName, getAttributesKey()); } return isDatasetFromContainer(normalPathName); } @@ -208,7 +210,7 @@ default JsonElement getAttributes(final String pathName) throws N5IOException { /* If cached, return the cache */ if (cacheMeta()) { - return getCache().getAttributes(groupPath, N5KeyValueReader.ATTRIBUTES_JSON); + return getCache().getAttributes(groupPath, getAttributesKey()); } else { return GsonKeyValueN5Reader.super.getAttributes(groupPath); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index d95f6345b..9d509051f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -59,9 +59,9 @@ default void createGroup(final String path) throws N5Exception { // else if exists is true (then a dataset is present) so throw an exception to avoid // overwriting / invalidating existing data if (cacheMeta()) { - if (getCache().isGroup(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) + if (getCache().isGroup(normalPath, getAttributesKey())) return; - else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { + else if (getCache().exists(normalPath, getAttributesKey())) { throw new N5Exception("Can't make a group on existing path."); } } @@ -88,8 +88,8 @@ else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { for (final String child : pathParts) { final String childPath = parent.isEmpty() ? child : parent + "/" + child; - getCache().initializeNonemptyCache(childPath, N5KeyValueReader.ATTRIBUTES_JSON); - getCache().updateCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON); + getCache().initializeNonemptyCache(childPath, getAttributesKey()); + getCache().updateCacheInfo(childPath, getAttributesKey()); // only add if the parent exists and has children cached already if (parent != null && !child.isEmpty()) @@ -130,7 +130,7 @@ default void writeAndCacheAttributes( nullRespectingAttributes = getGson().toJsonTree(attributes); } /* Update the cache, and write to the writer */ - getCache().updateCacheInfo(normalGroupPath, N5KeyValueReader.ATTRIBUTES_JSON, nullRespectingAttributes); + getCache().updateCacheInfo(normalGroupPath, getAttributesKey(), nullRespectingAttributes); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 29e43ce60..edbc6947e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -144,6 +144,12 @@ protected N5KeyValueReader( throw new N5Exception.N5IOException("No container exists at " + basePath); } + @Override + public String getAttributesKey() { + + return ATTRIBUTES_JSON; + } + @Override public Gson getGson() { From bf4f63e79aaa827fa5d4fab066ee06f5cebb588b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 5 Aug 2024 13:27:26 -0400 Subject: [PATCH 024/423] wip: move getAttributesKey to GsonN5Reader --- .../org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java | 2 -- .../java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java | 2 +- src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java | 2 ++ 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index 4cdb8d92f..324d242e8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -52,8 +52,6 @@ default N5JsonCache newCache() { N5JsonCache getCache(); - public String getAttributesKey(); - @Override default JsonElement getAttributesFromContainer(final String normalPathName, final String normalCacheKey) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 3f44eebcb..003820b1c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -215,6 +215,6 @@ default String absoluteGroupPath(final String normalGroupPath) { */ default String absoluteAttributesPath(final String normalPath) { - return getKeyValueAccess().compose(getURI(), normalPath, N5KeyValueReader.ATTRIBUTES_JSON); + return getKeyValueAccess().compose(getURI(), normalPath, getAttributesKey()); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index ea7ea878b..c611a9e7d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -42,6 +42,8 @@ public interface GsonN5Reader extends N5Reader { Gson getGson(); + public String getAttributesKey(); + @Override default Map> listAttributes(final String pathName) throws N5Exception { From b01b6becfbeab6ff9479e4538c843631bb106c8b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 6 Aug 2024 16:57:36 -0400 Subject: [PATCH 025/423] wip add constant N5_DATASET_ATTRIBUTES --- .../java/org/janelia/saalfeldlab/n5/DatasetAttributes.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 9f67d0d06..958cd4be3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -35,6 +35,10 @@ public class DatasetAttributes implements Serializable { public static final String COMPRESSION_KEY = "compression"; public static final String CODEC_KEY = "codecs"; + public static final String[] N5_DATASET_ATTRIBUTES = new String[]{ + DIMENSIONS_KEY, BLOCK_SIZE_KEY, DATA_TYPE_KEY, COMPRESSION_KEY, CODEC_KEY + }; + /* version 0 */ protected static final String compressionTypeKey = "compressionType"; @@ -54,8 +58,8 @@ public DatasetAttributes( this.dimensions = dimensions; this.blockSize = blockSize; this.dataType = dataType; - this.compression = compression; this.codecs = codecs; + this.compression = compression; } public DatasetAttributes( From 0cb5a0f5fd99a7c0b8cc19e3140be4612d38bd23 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 8 Aug 2024 16:10:47 -0400 Subject: [PATCH 026/423] refactor: Compression interface extends Codec * no need for getCompressionAsCodec * use getType for codecs --- .../janelia/saalfeldlab/n5/CodecAdapter.java | 28 ++++++------ .../janelia/saalfeldlab/n5/Compression.java | 45 ++----------------- .../saalfeldlab/n5/DatasetAttributes.java | 8 ++-- .../saalfeldlab/n5/codec/AsTypeCodec.java | 7 +-- .../saalfeldlab/n5/codec/BytesCodec.java | 23 +++++++--- .../janelia/saalfeldlab/n5/codec/Codec.java | 2 +- .../saalfeldlab/n5/codec/ComposedCodec.java | 9 ++-- .../n5/codec/FixedScaleOffsetCodec.java | 13 +++--- .../saalfeldlab/n5/codec/IdentityCodec.java | 14 +++--- .../codec/checksum/Crc32cChecksumCodec.java | 16 +++---- .../saalfeldlab/n5/shard/ShardReader.java | 3 +- .../saalfeldlab/n5/shard/ShardingCodec.java | 30 ++++++++----- .../saalfeldlab/n5/shard/ShardDemos.java | 34 +++++++------- 13 files changed, 103 insertions(+), 129 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java index db8a6daaf..785380ad6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java @@ -50,30 +50,30 @@ public JsonElement serialize( final Type typeOfSrc, final JsonSerializationContext context) { - if (codec.getName().equals(FixedScaleOffsetCodec.FIXED_SCALE_OFFSET_CODEC_ID)) { + if (codec.getType().equals(FixedScaleOffsetCodec.TYPE)) { final FixedScaleOffsetCodec c = (FixedScaleOffsetCodec)codec; final JsonObject obj = new JsonObject(); - obj.addProperty("name", c.getName()); + obj.addProperty("name", c.getType()); obj.addProperty("scale", c.getScale()); obj.addProperty("offset", c.getOffset()); obj.addProperty("type", c.getType().toString().toLowerCase()); - obj.addProperty("encodedType", c.getEncodedType().toString().toLowerCase()); + obj.addProperty("encodedType", c.getEncodedDataType().toString().toLowerCase()); return obj; } - else if (codec.getName().equals(ShardingCodec.ID)) { + else if (codec.getType().equals(ShardingCodec.TYPE)) { final ShardingCodec sharding = (ShardingCodec)codec; final JsonObject obj = new JsonObject(); - obj.addProperty("name", sharding.getName()); + obj.addProperty("name", sharding.getType()); obj.add("configuration", context.serialize(sharding.getConfiguration())); return obj; } - else if (codec.getName().equals(BytesCodec.ID)) { + else if (codec.getType().equals(BytesCodec.TYPE)) { final BytesCodec bytes = (BytesCodec)codec; final JsonObject obj = new JsonObject(); - obj.addProperty("name", bytes.getName()); + obj.addProperty("type", bytes.getType()); final JsonObject config = new JsonObject(); - config.addProperty("endian", bytes.getName()); + config.addProperty("endian", bytes.getType()); obj.add("configuration", config); return obj; @@ -94,10 +94,10 @@ else if (!json.isJsonObject()) return null; final JsonObject jsonObject = json.getAsJsonObject(); - if (jsonObject.has("name")) { + if (jsonObject.has("type")) { - final String id = jsonObject.get("name").getAsString(); - if (id.equals(FixedScaleOffsetCodec.FIXED_SCALE_OFFSET_CODEC_ID)) { + final String type = jsonObject.get("type").getAsString(); + if (type.equals(FixedScaleOffsetCodec.TYPE)) { return new FixedScaleOffsetCodec( jsonObject.get("scale").getAsDouble(), @@ -105,12 +105,12 @@ else if (!json.isJsonObject()) DataType.valueOf(jsonObject.get("type").getAsString().toUpperCase()), DataType.valueOf(jsonObject.get("encodedType").getAsString().toUpperCase())); } - else if (id.equals(ShardingCodec.ID)) { + else if (type.equals(ShardingCodec.TYPE)) { return new ShardingCodec( context.deserialize(jsonObject.get("configuration"), ShardingConfiguration.class)); - } else if (id.equals(BytesCodec.ID)) { + } else if (type.equals(BytesCodec.TYPE)) { - // TODO + // TODO implement return new BytesCodec(); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index a6ca1be04..ba78cecf7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -43,13 +43,7 @@ * * @author Stephan Saalfeld */ -public interface Compression extends Serializable { - - // @Override - // public default String getId() { - // - // return getType(); - // } +public interface Compression extends Serializable, Codec { /** * Annotation for runtime discovery of compression schemes. @@ -73,6 +67,7 @@ public interface Compression extends Serializable { @Target(ElementType.FIELD) public static @interface CompressionParameter {} + @Override public default String getType() { final CompressionType compressionType = getClass().getAnnotation(CompressionType.class); @@ -94,6 +89,7 @@ public default String getType() { * input stream * @return the decoded input stream */ + @Override public InputStream decode(InputStream in) throws IOException; /** @@ -103,40 +99,7 @@ public default String getType() { * the output stream * @return the encoded output stream */ + @Override public OutputStream encode(OutputStream out) throws IOException; - public static Codec getCompressionAsCodec(Compression compression) { - - return new CompressionCodec(compression); - } - - public static class CompressionCodec implements Codec { - - private static final long serialVersionUID = -7931131454184340637L; - private Compression compression; - - public CompressionCodec(Compression compression) { - - this.compression = compression; - } - - @Override - public InputStream decode(InputStream in) throws IOException { - - return compression.decode(in); - } - - @Override - public OutputStream encode(OutputStream out) throws IOException { - - return compression.encode(out); - } - - @Override - public String getName() { - - return compression.getType(); - } - - } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 958cd4be3..3205312fe 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -103,18 +103,16 @@ public Codec[] getCodecs() { public Codec collectCodecs() { - final Codec compressionCodec = Compression.getCompressionAsCodec(compression); - if (codecs == null || codecs.length == 0) - return compressionCodec; + return compression; else if (codecs.length == 1) - return new ComposedCodec(codecs[0], compressionCodec); + return new ComposedCodec(codecs[0], compression); else { final Codec[] codecsAndCompresor = new Codec[codecs.length + 1]; for (int i = 0; i < codecs.length; i++) codecsAndCompresor[i] = codecs[i]; - codecsAndCompresor[codecs.length] = compressionCodec; + codecsAndCompresor[codecs.length] = compression; return new ComposedCodec(codecsAndCompresor); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java index f7bf9945d..9e4ce0087 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java @@ -13,6 +13,8 @@ public class AsTypeCodec implements Codec { private static final long serialVersionUID = 1031322606191894484L; + public static final String TYPE = "astype"; + protected transient final int numBytes; protected transient final int numEncodedBytes; @@ -22,7 +24,6 @@ public class AsTypeCodec implements Codec { protected final DataType type; protected final DataType encodedType; - protected final String name = "astype"; public AsTypeCodec( DataType type, DataType encodedType ) { @@ -37,9 +38,9 @@ public AsTypeCodec( DataType type, DataType encodedType ) } @Override - public String getName() { + public String getType() { - return name; + return TYPE; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java index 65cff6a21..8c2b9e024 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java @@ -5,17 +5,28 @@ public class BytesCodec extends IdentityCodec { private static final long serialVersionUID = 3523505403978222360L; - public static final String ID = "bytes"; + public static final String TYPE = "bytes"; - protected final String name = ID; + private final String endian; - protected final String endian = "little"; + public BytesCodec() { - // TODO implement me + this("little"); + } + + public BytesCodec(final String endian) { + + this.endian = endian; + } @Override - public String getName() { + public String getType() { + + return TYPE; + } + + public String getEndian() { - return name; + return endian; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index b3225b7e8..862a255e7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -32,6 +32,6 @@ public interface Codec extends Serializable { */ public OutputStream encode(OutputStream out) throws IOException; - public String getName(); + public String getType(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java index 10f8adb01..3b07ad2b6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java @@ -10,9 +10,10 @@ public class ComposedCodec implements Codec { private static final long serialVersionUID = 5068349140842235924L; - private final Codec[] filters; - protected String name = "composed"; + protected static final String TYPE = "composed"; + + private final Codec[] filters; public ComposedCodec(final Codec... filters) { @@ -20,9 +21,9 @@ public ComposedCodec(final Codec... filters) { } @Override - public String getName() { + public String getType() { - return name; + return TYPE; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java index 0538498db..eba6e12e2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java @@ -12,14 +12,11 @@ public class FixedScaleOffsetCodec extends AsTypeCodec { private static final long serialVersionUID = 8024945290803548528L; - public static transient final String FIXED_SCALE_OFFSET_CODEC_ID = "fixedscaleoffset"; + public static transient final String TYPE = "fixedscaleoffset"; private final double scale; - private final double offset; - protected final String name = FIXED_SCALE_OFFSET_CODEC_ID; - private transient final ByteBuffer tmpEncoder; private transient final ByteBuffer tmpDecoder; @@ -82,20 +79,20 @@ public double getOffset() { return offset; } - public DataType getType() { + public DataType getDataType() { return super.type; } - public DataType getEncodedType() { + public DataType getEncodedDataType() { return encodedType; } @Override - public String getName() { + public String getType() { - return name; + return TYPE; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java index 4383669aa..6ee5c64c8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -8,13 +8,7 @@ public class IdentityCodec implements Codec { private static final long serialVersionUID = 8354269325800855621L; - protected final String name = "id"; - - @Override - public String getName() { - - return name; - } + protected static final String TYPE = "id"; @Override public InputStream decode(InputStream in) throws IOException { @@ -28,4 +22,10 @@ public OutputStream encode(OutputStream out) throws IOException { return out; } + @Override + public String getType() { + + return TYPE; + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java index 0a16d435a..0b85ed5a9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java @@ -7,21 +7,13 @@ public class Crc32cChecksumCodec extends ChecksumCodec { private static final long serialVersionUID = 7424151868725442500L; - public static transient final String CRC32C_CHECKSUM_CODEC_ID = "crc32c"; - - private final String name = CRC32C_CHECKSUM_CODEC_ID; + public static transient final String TYPE = "crc32c"; public Crc32cChecksumCodec() { super(new CRC32(), 4); } - @Override - public String getName() { - - return name; - } - @Override public long encodedSize(final long size) { @@ -42,4 +34,10 @@ public ByteBuffer getChecksumValue() { return buf; } + @Override + public String getType() { + + return TYPE; + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java index ed4e71288..0ab5d276a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java @@ -5,7 +5,6 @@ import java.nio.channels.Channels; import java.nio.channels.FileChannel; -import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DefaultBlockReader; @@ -82,7 +81,7 @@ private static ShardedDatasetAttributes buildTestAttributes() { new ShardingCodec( new ShardingConfiguration( new int[]{2, 2}, - new Codec[]{new Compression.CompressionCodec(new RawCompression()), new IdentityCodec()}, + new Codec[]{new RawCompression(), new IdentityCodec()}, new Codec[]{new Crc32cChecksumCodec()}, IndexLocation.END) ) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 9f9f978a3..aa0f66846 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -6,6 +6,7 @@ import java.lang.reflect.Type; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; @@ -19,17 +20,24 @@ public class ShardingCodec implements Codec { private static final long serialVersionUID = -5879797314954717810L; - public static final String ID = "sharding_indexed"; + public static final String TYPE = "sharding_indexed"; private final ShardingConfiguration configuration; - private final String name = ID; - public ShardingCodec(ShardingConfiguration configuration) { this.configuration = configuration; } + public ShardingCodec( + final int[] blockSize, + final Codec[] codecs, + final Codec[] indexCodecs, + final IndexLocation indexLocation) { + + this.configuration = new ShardingConfiguration(blockSize, codecs, indexCodecs, indexLocation); + } + public ShardingConfiguration getConfiguration() { return configuration; @@ -39,6 +47,7 @@ public ShardingConfiguration getConfiguration() { public InputStream decode(InputStream in) throws IOException { // TODO Auto-generated method stub + // This method actually makes no sense for a sharding codec return in; } @@ -46,15 +55,10 @@ public InputStream decode(InputStream in) throws IOException { public OutputStream encode(OutputStream out) throws IOException { // TODO Auto-generated method stub + // This method actually makes no sense for a sharding codec return out; } - @Override - public String getName() { - - return name; - } - public static boolean isShardingCodec(final Codec codec) { return codec instanceof ShardingCodec; @@ -68,7 +72,7 @@ public JsonElement serialize(ShardingCodec src, Type typeOfSrc, JsonSerializatio final JsonObject jsonObj = new JsonObject(); - jsonObj.addProperty("name", ShardingCodec.ID); + jsonObj.addProperty("name", ShardingCodec.TYPE); // context.serialize(typeOfSrc); return jsonObj; @@ -83,4 +87,10 @@ public ShardingCodec deserialize(JsonElement json, Type typeOfT, JsonDeserializa } + @Override + public String getType() { + + return TYPE; + } + } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java index 31491634b..9ea523b16 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -1,14 +1,12 @@ package org.janelia.saalfeldlab.n5.shard; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonSerializationContext; -import org.janelia.saalfeldlab.n5.Compression; +import java.lang.reflect.Type; +import java.net.MalformedURLException; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; + import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; @@ -23,12 +21,12 @@ import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.junit.Test; -import java.lang.reflect.Type; -import java.net.MalformedURLException; -import java.nio.file.FileSystems; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; public class ShardDemos { @@ -79,7 +77,7 @@ public void writeReadBlockTest() { new ShardingCodec( new ShardingConfiguration( new int[]{2, 2}, - new Codec[]{new Compression.CompressionCodec(new RawCompression()), new IdentityCodec()}, + new Codec[]{new RawCompression(), new IdentityCodec()}, new Codec[]{new Crc32cChecksumCodec()}, IndexLocation.END) ) @@ -144,13 +142,11 @@ public void nameConfigurationGsonTest() { final N5Factory factory = new N5Factory(); final GsonBuilder gson = new GsonBuilder(); - - gson.registerTypeHierarchyAdapter() factory.gsonBuilder(gson); final N5Reader n5 = factory.openReader("src/test/resources/shardExamples/test.zarr/mid_sharded"); final JsonObject zarrJson = n5.getAttribute("/", "/", JsonObject.class); - zarrJson.remove("shard") + zarrJson.remove("shard"); } } From 553c85e3dfe2e3a662a48d30e97f2830ef777c96 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 9 Aug 2024 15:25:19 -0400 Subject: [PATCH 027/423] wip: BytesCodec update --- .../saalfeldlab/n5/codec/BytesCodec.java | 73 +++++++++++++++++-- 1 file changed, 65 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java index 8c2b9e024..a66657b6d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java @@ -1,22 +1,56 @@ package org.janelia.saalfeldlab.n5.codec; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteOrder; -public class BytesCodec extends IdentityCodec { +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +public class BytesCodec implements Codec { private static final long serialVersionUID = 3523505403978222360L; - public static final String TYPE = "bytes"; + public static String TYPE = "bytes"; + + protected final ByteOrder byteOrder; - private final String endian; + protected transient final byte[] array; public BytesCodec() { - this("little"); + this(ByteOrder.LITTLE_ENDIAN); + } + + public BytesCodec(final ByteOrder byteOrder) { + + this(byteOrder, 256); } - public BytesCodec(final String endian) { + public BytesCodec(final ByteOrder byteOrder, final int N) { - this.endian = endian; + this.byteOrder = byteOrder; + this.array = new byte[N]; + } + + @Override + public InputStream decode(InputStream in) throws IOException { + + // TODO not applicable for array -> bytes + return in; + } + + @Override + public OutputStream encode(OutputStream out) throws IOException { + + // TODO not applicable for array -> bytes + return out; } @Override @@ -25,8 +59,31 @@ public String getType() { return TYPE; } - public String getEndian() { + public static final ByteOrderAdapter byteOrderAdapter = new ByteOrderAdapter(); + + public static class ByteOrderAdapter implements JsonDeserializer, JsonSerializer { + + @Override + public JsonElement serialize(ByteOrder src, java.lang.reflect.Type typeOfSrc, + JsonSerializationContext context) { + + if (src.equals(ByteOrder.LITTLE_ENDIAN)) + return new JsonPrimitive("little"); + else + return new JsonPrimitive("big"); + } + + @Override + public ByteOrder deserialize(JsonElement json, java.lang.reflect.Type typeOfT, + JsonDeserializationContext context) throws JsonParseException { + + if (json.getAsString().equals("little")) + return ByteOrder.LITTLE_ENDIAN; + if (json.getAsString().equals("big")) + return ByteOrder.BIG_ENDIAN; + + return null; + } - return endian; } } From 35a4d354ee54d1c22849f92c49fad50269b069e9 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 9 Aug 2024 15:21:45 -0400 Subject: [PATCH 028/423] test: remove outdated wip config parsing --- .../saalfeldlab/n5/shard/ShardDemos.java | 69 +------------------ 1 file changed, 1 insertion(+), 68 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java index 9ea523b16..d9a1a6042 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -1,6 +1,5 @@ package org.janelia.saalfeldlab.n5.shard; -import java.lang.reflect.Type; import java.net.MalformedURLException; import java.nio.file.FileSystems; import java.nio.file.Path; @@ -10,7 +9,6 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; -import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; @@ -21,12 +19,6 @@ import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.junit.Test; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonSerializationContext; public class ShardDemos { @@ -77,7 +69,7 @@ public void writeReadBlockTest() { new ShardingCodec( new ShardingConfiguration( new int[]{2, 2}, - new Codec[]{new RawCompression(), new IdentityCodec()}, + new Codec[]{new Compression.CompressionCodec(new RawCompression()), new IdentityCodec()}, new Codec[]{new Crc32cChecksumCodec()}, IndexLocation.END) ) @@ -90,63 +82,4 @@ public void writeReadBlockTest() { writer.readBlock("shard", datasetAttributes, 0,0); } - private static class ZarrConfig { - final String name; - final T configuration; - - private ZarrConfig() { - name = ""; - configuration = null; - } - } - - private class GridConfig extends ZarrConfig {} - private class KeyEncodingConfig extends ZarrConfig {} - - private class ZarrChunk {} - - private class ZarrChunkAdapter implements com.google.gson.JsonSerializer, com.google.gson.JsonDeserializer { - final ZarrConfig grid; - final ZarrConfig keyEncoding; - - public ZarrChunkAdapter() { - grid = null; - keyEncoding = null; - } - public ZarrChunkAdapter(ZarrConfig grid, ZarrConfig key_encoding) { - - this.grid = grid; - this.keyEncoding = key_encoding; - } - - @Override public ZarrChunk deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { - - if (!json.isJsonObject()) return null; - - final JsonObject obj = json.getAsJsonObject(); - final JsonObject grid = obj.getAsJsonObject("chunk_grid"); - - return null; - } - - @Override public JsonElement serialize(ZarrChunk src, Type typeOfSrc, JsonSerializationContext context) { - - return null; - } - } - - - @Test - public void nameConfigurationGsonTest() { - - final N5Factory factory = new N5Factory(); - final GsonBuilder gson = new GsonBuilder(); - - factory.gsonBuilder(gson); - final N5Reader n5 = factory.openReader("src/test/resources/shardExamples/test.zarr/mid_sharded"); - - final JsonObject zarrJson = n5.getAttribute("/", "/", JsonObject.class); - zarrJson.remove("shard"); - } - } From d636b83133476e037bd9b466d4f6a0cc41196cc6 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 12 Aug 2024 10:24:33 -0400 Subject: [PATCH 029/423] feat: annotations for extensible serialization for codecs --- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 5 +- .../saalfeldlab/n5/NameConfigAdapter.java | 233 ++++++++++++++++++ .../saalfeldlab/n5/codec/BytesCodec.java | 3 + .../janelia/saalfeldlab/n5/codec/Codec.java | 3 + .../saalfeldlab/n5/codec/IdentityCodec.java | 3 + .../n5/serialization/JsonArrayUtils.java | 20 ++ .../n5/serialization/N5Annotations.java | 18 ++ .../n5/serialization/NameConfig.java | 43 ++++ .../saalfeldlab/n5/codec/BytesTests.java | 53 ++++ .../saalfeldlab/n5/shard/ShardDemos.java | 2 +- 10 files changed, 379 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/serialization/JsonArrayUtils.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/serialization/N5Annotations.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 7423b57ea..c09d1e147 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -56,10 +56,9 @@ public interface GsonUtils { static Gson registerGson(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(ShardingConfiguration.class, new ShardingConfiguration.ShardingConfigurationAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(ShardingConfiguration.class, - new ShardingConfiguration.ShardingConfigurationAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Codec.class, new CodecAdapter()); gsonBuilder.disableHtmlEscaping(); return gsonBuilder.create(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java new file mode 100644 index 000000000..3384fb0d6 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java @@ -0,0 +1,233 @@ +/** + * Copyright (c) 2017, Stephan Saalfeld + * All rights reserved. + *

      + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + *

      + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

      + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.JsonArray; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import org.janelia.saalfeldlab.n5.serialization.N5Annotations; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; +import org.scijava.annotations.Index; +import org.scijava.annotations.IndexItem; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map.Entry; + +/** + * T adapter, auto-discovers annotated T implementations in the classpath. + * + * @author Caleb Hulbert + */ +public class NameConfigAdapter implements JsonDeserializer, JsonSerializer { + + private static HashMap, NameConfigAdapter> adapters = new HashMap<>(); + + private static void registerAdapter(Class cls) { + + adapters.put(cls, new NameConfigAdapter(cls)); + update(adapters.get(cls)); + } + private final HashMap> constructors = new HashMap<>(); + + private final HashMap> parameters = new HashMap<>(); + private final HashMap> parameterNames = new HashMap<>(); + private static ArrayList getDeclaredFields(Class clazz) { + + final ArrayList fields = new ArrayList<>(); + fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + for (clazz = clazz.getSuperclass(); clazz != null; clazz = clazz.getSuperclass()) + fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + return fields; + } + + @SuppressWarnings("unchecked") + public static synchronized void update(final NameConfigAdapter adapter) { + + final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + final Index annotationIndex = Index.load(NameConfig.Name.class, classLoader); + for (final IndexItem item : annotationIndex) { + Class clazz; + try { + clazz = (Class)Class.forName(item.className()); + final String name = clazz.getAnnotation(NameConfig.Name.class).value(); + final String prefix = adapter.type.getAnnotation(NameConfig.Prefix.class).value(); + final String type = prefix + "." + name; + + final Constructor constructor = clazz.getDeclaredConstructor(); + + final HashMap parameters = new HashMap<>(); + final HashMap parameterNames = new HashMap<>(); + final ArrayList fields = getDeclaredFields(clazz); + for (final Field field : fields) { + final NameConfig.Parameter parameter = field.getAnnotation(NameConfig.Parameter.class); + if (parameter != null) { + + final String parameterName; + if (parameter.value().equals("")) + parameterName = field.getName(); + else + parameterName = parameter.value(); + + parameterNames.put(field.getName(), parameterName); + + parameters.put(field.getName(), field); + } + } + + adapter.constructors.put(type, constructor); + adapter.parameters.put(type, parameters); + adapter.parameterNames.put(type, parameterNames); + } catch (final ClassNotFoundException | NoSuchMethodException | ClassCastException + | UnsatisfiedLinkError e) { + System.err.println("T '" + item.className() + "' could not be registered"); + } + } + } + + private final Class type; + + public NameConfigAdapter(Class cls) { + this.type = cls; + } + + @Override + public JsonElement serialize( + final T object, + final Type typeOfSrc, + final JsonSerializationContext context) { + + final Class clazz = (Class)object.getClass(); + final String name = clazz.getAnnotation(NameConfig.Name.class).value(); + final String prefix = type.getAnnotation(NameConfig.Prefix.class).value(); + final String type = prefix + "." + name; + + final JsonObject json = new JsonObject(); + json.addProperty("name", name); + final JsonObject configuration = new JsonObject(); + json.add("configuration", configuration); + + final HashMap parameterTypes = parameters.get(type); + final HashMap parameterNameMap = parameterNames.get(type); + try { + for (final Entry parameterType : parameterTypes.entrySet()) { + final String fieldName = parameterType.getKey(); + final Field field = clazz.getDeclaredField(fieldName); + final boolean isAccessible = field.isAccessible(); + field.setAccessible(true); + final Object value = field.get(object); + field.setAccessible(isAccessible); + final JsonElement serialized = context.serialize(value); + if (field.getAnnotation(N5Annotations.ReverseArray.class) != null) { + final JsonArray reversedArray = reverseJsonArray(serialized.getAsJsonArray()); + configuration.add(parameterNameMap.get(fieldName), reversedArray); + } else + configuration.add(parameterNameMap.get(fieldName), serialized); + + } + } catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) { + e.printStackTrace(System.err); + return null; + } + + return json; + } + + @Override + public T deserialize( + final JsonElement json, + final Type typeOfT, + final JsonDeserializationContext context) throws JsonParseException { + + final String prefix = type.getAnnotation(NameConfig.Prefix.class).value(); + + final JsonObject chunkGridJson = json.getAsJsonObject(); + final String name = chunkGridJson.getAsJsonPrimitive("name").getAsString(); + if (name == null) + return null; + final JsonObject configuration = chunkGridJson.getAsJsonObject("configuration"); + if (configuration == null) + return null; + + final String type = prefix + "." + name; + + final Constructor constructor = constructors.get(type); + constructor.setAccessible(true); + final T chunkGrid; + try { + chunkGrid = constructor.newInstance(); + final HashMap parameterTypes = parameters.get(type); + final HashMap parameterNameMap = parameterNames.get(type); + for (final Entry parameterType : parameterTypes.entrySet()) { + final String fieldName = parameterType.getKey(); + final String paramName = parameterNameMap.get(fieldName); + final JsonElement paramJson = configuration.get(paramName); + if (paramJson != null) { + final Field field = parameterType.getValue(); + final Object parameter; + if (field.getAnnotation(N5Annotations.ReverseArray.class) != null) { + final JsonArray reversedArray = reverseJsonArray(paramJson); + parameter = context.deserialize(reversedArray, field.getType()); + } else + parameter = context.deserialize(paramJson, field.getType()); + ReflectionUtils.setFieldValue(chunkGrid, fieldName, parameter); + } + } + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException + | SecurityException | NoSuchFieldException e) { + e.printStackTrace(System.err); + return null; + } + + return chunkGrid; + } + + private static JsonArray reverseJsonArray(JsonElement paramJson) { + + final JsonArray reversedJson = new JsonArray(paramJson.getAsJsonArray().size()); + for (int i = paramJson.getAsJsonArray().size() - 1; i >= 0; i--) { + reversedJson.add(paramJson.getAsJsonArray().get(i)); + } + return reversedJson; + } + + public static NameConfigAdapter getJsonAdapter(Class cls) { + + if (adapters.get(cls) == null) + registerAdapter(cls); + return (NameConfigAdapter) adapters.get(cls); + } +} \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java index a66657b6d..5f0bb4abb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java @@ -12,13 +12,16 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; +@NameConfig.Name("bytes") public class BytesCodec implements Codec { private static final long serialVersionUID = 3523505403978222360L; public static String TYPE = "bytes"; + @NameConfig.Parameter("endian") protected final ByteOrder byteOrder; protected transient final byte[] array; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index 862a255e7..2c76438ca 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -1,5 +1,7 @@ package org.janelia.saalfeldlab.n5.codec; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -12,6 +14,7 @@ * Modeled after Filters in * Zarr. */ +@NameConfig.Prefix("codec") public interface Codec extends Serializable { /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java index 6ee5c64c8..79c532b17 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -1,9 +1,12 @@ package org.janelia.saalfeldlab.n5.codec; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +@NameConfig.Name(IdentityCodec.TYPE) public class IdentityCodec implements Codec { private static final long serialVersionUID = 8354269325800855621L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/serialization/JsonArrayUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/serialization/JsonArrayUtils.java new file mode 100644 index 000000000..b65fbb6c1 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/serialization/JsonArrayUtils.java @@ -0,0 +1,20 @@ +package org.janelia.saalfeldlab.n5.serialization; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; + +public class JsonArrayUtils { + + public static void reverse(final JsonArray array) { + + JsonElement a; + final int max = array.size() - 1; + for (int i = (max - 1) / 2; i >= 0; --i) { + final int j = max - i; + a = array.get(i); + array.set(i, array.get(j)); + array.set(j, a); + } + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/serialization/N5Annotations.java b/src/main/java/org/janelia/saalfeldlab/n5/serialization/N5Annotations.java new file mode 100644 index 000000000..500f139f1 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/serialization/N5Annotations.java @@ -0,0 +1,18 @@ +package org.janelia.saalfeldlab.n5.serialization; + +import java.io.Serializable; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +public interface N5Annotations extends Serializable { + + @Inherited + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.FIELD) + @interface ReverseArray { + } +} + diff --git a/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java b/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java new file mode 100644 index 000000000..229653826 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java @@ -0,0 +1,43 @@ +package org.janelia.saalfeldlab.n5.serialization; + +import org.scijava.annotations.Indexable; + +import java.io.Serializable; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +public interface NameConfig extends Serializable { + + @Retention(RetentionPolicy.RUNTIME) + @Inherited + @Target(ElementType.TYPE) + @interface Prefix { + String value(); + } + + @Retention(RetentionPolicy.RUNTIME) + @Inherited + @Target(ElementType.TYPE) + @Indexable + @interface Name { + String value(); + } + + @Retention(RetentionPolicy.RUNTIME) + @Inherited + @Target(ElementType.FIELD) + @interface Parameter { + String value() default ""; + } + + default String getType() { + + final Name type = getClass().getAnnotation(Name.class); + return type == null ? null : type.value(); + + } +} + diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java new file mode 100644 index 000000000..8eeb83455 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java @@ -0,0 +1,53 @@ +package org.janelia.saalfeldlab.n5.codec; + +import com.google.gson.GsonBuilder; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.NameConfigAdapter; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.junit.Test; + +import java.nio.ByteOrder; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class BytesTests { + + @Test + public void testSerialization() { + + final N5Factory factory = new N5Factory(); + factory.cacheAttributes(false); + final GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); + gsonBuilder.registerTypeAdapter(ByteOrder.class, BytesCodec.byteOrderAdapter); + factory.gsonBuilder(gsonBuilder); + + final N5Writer reader = factory.openWriter("n5:src/test/resources/shardExamples/test.zarr"); + final Codec bytes = reader.getAttribute("mid_sharded", "codecs[0]/configuration/codecs[0]", Codec.class); + assertTrue("as BytesCodec", bytes instanceof BytesCodec); + + final N5Writer writer = factory.openWriter("n5:src/test/resources/shardExamples/test.n5"); + + final DatasetAttributes datasetAttributes = new DatasetAttributes( + new long[]{8, 8}, + new int[]{4, 4}, + DataType.UINT8, + new RawCompression(), + new Codec[]{ + new IdentityCodec(), + new BytesCodec(ByteOrder.LITTLE_ENDIAN) + } + ); + writer.setAttribute("shard", "/", datasetAttributes); + final DatasetAttributes deserialized = writer.getAttribute("shard", "/", DatasetAttributes.class); + + assertEquals("2 codecs", 2, deserialized.getCodecs().length); + assertTrue("Identity", deserialized.getCodecs()[0] instanceof IdentityCodec); + assertTrue("Bytes", deserialized.getCodecs()[1] instanceof BytesCodec); + assertEquals("LittleEndian",ByteOrder.LITTLE_ENDIAN, ((BytesCodec)deserialized.getCodecs()[1]).byteOrder); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java index d9a1a6042..32574d4b5 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -69,7 +69,7 @@ public void writeReadBlockTest() { new ShardingCodec( new ShardingConfiguration( new int[]{2, 2}, - new Codec[]{new Compression.CompressionCodec(new RawCompression()), new IdentityCodec()}, + new Codec[]{new RawCompression(), new IdentityCodec()}, new Codec[]{new Crc32cChecksumCodec()}, IndexLocation.END) ) From 1d9c0c2414afb309ae1def918b36c761d136b61d Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 12 Aug 2024 11:37:49 -0400 Subject: [PATCH 030/423] refactor: rename former 'chunkGrid' variables --- .../janelia/saalfeldlab/n5/NameConfigAdapter.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java index 3384fb0d6..85768d180 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java @@ -174,11 +174,11 @@ public T deserialize( final String prefix = type.getAnnotation(NameConfig.Prefix.class).value(); - final JsonObject chunkGridJson = json.getAsJsonObject(); - final String name = chunkGridJson.getAsJsonPrimitive("name").getAsString(); + final JsonObject objectJson = json.getAsJsonObject(); + final String name = objectJson.getAsJsonPrimitive("name").getAsString(); if (name == null) return null; - final JsonObject configuration = chunkGridJson.getAsJsonObject("configuration"); + final JsonObject configuration = objectJson.getAsJsonObject("configuration"); if (configuration == null) return null; @@ -186,9 +186,9 @@ public T deserialize( final Constructor constructor = constructors.get(type); constructor.setAccessible(true); - final T chunkGrid; + final T object; try { - chunkGrid = constructor.newInstance(); + object = constructor.newInstance(); final HashMap parameterTypes = parameters.get(type); final HashMap parameterNameMap = parameterNames.get(type); for (final Entry parameterType : parameterTypes.entrySet()) { @@ -203,7 +203,7 @@ public T deserialize( parameter = context.deserialize(reversedArray, field.getType()); } else parameter = context.deserialize(paramJson, field.getType()); - ReflectionUtils.setFieldValue(chunkGrid, fieldName, parameter); + ReflectionUtils.setFieldValue(object, fieldName, parameter); } } } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException @@ -212,7 +212,7 @@ public T deserialize( return null; } - return chunkGrid; + return object; } private static JsonArray reverseJsonArray(JsonElement paramJson) { From c7fb316082ab1ee3bc794ba8bb700b03fec469c0 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 12 Aug 2024 13:18:34 -0400 Subject: [PATCH 031/423] test: start CodecSerialization test * make AsType, FixedScaleOffset, Identity codecs serializable --- .../saalfeldlab/n5/codec/AsTypeCodec.java | 53 ++++++++++---- .../n5/codec/FixedScaleOffsetCodec.java | 40 +++++------ .../saalfeldlab/n5/codec/IdentityCodec.java | 7 +- .../n5/serialization/CodecSerialization.java | 70 +++++++++++++++++++ 4 files changed, 133 insertions(+), 37 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java index 9e4ce0087..3a437ffa2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java @@ -7,34 +7,37 @@ import java.util.function.BiConsumer; import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; - +@NameConfig.Name(AsTypeCodec.TYPE) +@NameConfig.Prefix("codec") public class AsTypeCodec implements Codec { private static final long serialVersionUID = 1031322606191894484L; public static final String TYPE = "astype"; - protected transient final int numBytes; - protected transient final int numEncodedBytes; + protected transient int numBytes; + protected transient int numEncodedBytes; + + protected transient BiConsumer encoder; + protected transient BiConsumer decoder; - protected transient final BiConsumer encoder; - protected transient final BiConsumer decoder; + @NameConfig.Parameter + protected final DataType dataType; - protected final DataType type; + @NameConfig.Parameter protected final DataType encodedType; + private AsTypeCodec() { - public AsTypeCodec( DataType type, DataType encodedType ) - { - this.type = type; - this.encodedType = encodedType; + this(null, null); + } - numBytes = bytes(type); - numEncodedBytes = bytes(encodedType); + public AsTypeCodec(DataType dataType, DataType encodedType) { - encoder = converter(type, encodedType); - decoder = converter(encodedType, type); + this.dataType = dataType; + this.encodedType = encodedType; } @Override @@ -43,15 +46,37 @@ public String getType() { return TYPE; } + public DataType getDataType() { + + return dataType; + } + + public DataType getEncodedDataType() { + + return encodedType; + } + @Override public InputStream decode(InputStream in) throws IOException { + numBytes = bytes(dataType); + numEncodedBytes = bytes(encodedType); + + encoder = converter(dataType, encodedType); + decoder = converter(encodedType, dataType); + return new FixedLengthConvertedInputStream(numEncodedBytes, numBytes, decoder, in); } @Override public OutputStream encode(OutputStream out) throws IOException { + numBytes = bytes(dataType); + numEncodedBytes = bytes(encodedType); + + encoder = converter(dataType, encodedType); + decoder = converter(encodedType, dataType); + return new FixedLengthConvertedOutputStream(numBytes, numEncodedBytes, encoder, out); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java index eba6e12e2..55613b573 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java @@ -7,25 +7,35 @@ import java.util.function.BiConsumer; import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; +@NameConfig.Name(FixedScaleOffsetCodec.TYPE) public class FixedScaleOffsetCodec extends AsTypeCodec { private static final long serialVersionUID = 8024945290803548528L; public static transient final String TYPE = "fixedscaleoffset"; - private final double scale; - private final double offset; + @NameConfig.Parameter + protected final double scale; - private transient final ByteBuffer tmpEncoder; - private transient final ByteBuffer tmpDecoder; + @NameConfig.Parameter + protected final double offset; - public transient final BiConsumer encoder; - public transient final BiConsumer encoderPre; - public transient final BiConsumer encoderPost; - public transient final BiConsumer decoder; - public transient final BiConsumer decoderPre; - public transient final BiConsumer decoderPost; + private transient ByteBuffer tmpEncoder; + private transient ByteBuffer tmpDecoder; + + public transient BiConsumer encoder; + public transient BiConsumer encoderPre; + public transient BiConsumer encoderPost; + public transient BiConsumer decoder; + public transient BiConsumer decoderPre; + public transient BiConsumer decoderPost; + + private FixedScaleOffsetCodec() { + + this(1, 0, null, null); + } public FixedScaleOffsetCodec(final double scale, final double offset, DataType type, DataType encodedType) { @@ -79,16 +89,6 @@ public double getOffset() { return offset; } - public DataType getDataType() { - - return super.type; - } - - public DataType getEncodedDataType() { - - return encodedType; - } - @Override public String getType() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java index 79c532b17..41fd10f12 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -1,17 +1,18 @@ package org.janelia.saalfeldlab.n5.codec; -import org.janelia.saalfeldlab.n5.serialization.NameConfig; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + @NameConfig.Name(IdentityCodec.TYPE) +@NameConfig.Prefix("codec") public class IdentityCodec implements Codec { private static final long serialVersionUID = 8354269325800855621L; - protected static final String TYPE = "id"; + public static final String TYPE = "id"; @Override public InputStream decode(InputStream in) throws IOException { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java new file mode 100644 index 000000000..148618ee3 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java @@ -0,0 +1,70 @@ +package org.janelia.saalfeldlab.n5.serialization; + +import static org.junit.Assert.assertEquals; + +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.NameConfigAdapter; +import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.FixedScaleOffsetCodec; +import org.janelia.saalfeldlab.n5.codec.IdentityCodec; +import org.junit.Before; +import org.junit.Test; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +public class CodecSerialization { + + private Gson gson; + + @Before + public void before() { + + final GsonBuilder builder = new GsonBuilder(); + builder.registerTypeAdapter(IdentityCodec.class, NameConfigAdapter.getJsonAdapter(IdentityCodec.class)); + builder.registerTypeAdapter(AsTypeCodec.class, NameConfigAdapter.getJsonAdapter(AsTypeCodec.class)); + builder.registerTypeAdapter(FixedScaleOffsetCodec.class, + NameConfigAdapter.getJsonAdapter(FixedScaleOffsetCodec.class)); + gson = builder.create(); + } + + @Test + public void testSerializeIdentity() { + + final IdentityCodec id = new IdentityCodec(); + final JsonObject jsonId = gson.toJsonTree(id).getAsJsonObject(); + final JsonElement expected = gson.fromJson("{\"name\":\"id\", \"configuration\":{}}", JsonElement.class); + assertEquals("identity", expected, jsonId.getAsJsonObject()); + } + + @Test + public void testSerializeAsType() { + + final AsTypeCodec asTypeCodec = new AsTypeCodec(DataType.FLOAT64, DataType.INT16); + final JsonObject jsonAsType = gson.toJsonTree(asTypeCodec).getAsJsonObject(); + final JsonElement expected = gson.fromJson( + "{\"name\":\"astype\",\"configuration\":{\"dataType\":\"FLOAT64\",\"encodedType\":\"INT16\"}}", + JsonElement.class); + assertEquals("asType", expected, jsonAsType.getAsJsonObject()); + } + + @Test + public void testSerializeCodecArray() { + + final Codec[] codecs = new Codec[]{ + new IdentityCodec(), + new AsTypeCodec(DataType.FLOAT64, DataType.INT16) + }; + final JsonArray jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); + final JsonElement expected = gson.fromJson( + "[{\"name\":\"id\",\"configuration\":{}},{\"name\":\"astype\",\"configuration\":{\"dataType\":\"FLOAT64\",\"encodedType\":\"INT16\"}}]", + JsonElement.class); + + assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); + } + +} From 344aa462d0855019a5475eddcf46a288112d9c0a Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 12 Aug 2024 13:39:59 -0400 Subject: [PATCH 032/423] test: codec array with a compressor --- .../n5/serialization/CodecSerialization.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java index 148618ee3..c5d09a1da 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java @@ -3,6 +3,7 @@ import static org.junit.Assert.assertEquals; import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.GzipCompression; import org.janelia.saalfeldlab.n5.NameConfigAdapter; import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; import org.janelia.saalfeldlab.n5.codec.Codec; @@ -55,15 +56,25 @@ public void testSerializeAsType() { @Test public void testSerializeCodecArray() { - final Codec[] codecs = new Codec[]{ + Codec[] codecs = new Codec[]{ new IdentityCodec(), new AsTypeCodec(DataType.FLOAT64, DataType.INT16) }; - final JsonArray jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); - final JsonElement expected = gson.fromJson( + JsonArray jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); + JsonElement expected = gson.fromJson( "[{\"name\":\"id\",\"configuration\":{}},{\"name\":\"astype\",\"configuration\":{\"dataType\":\"FLOAT64\",\"encodedType\":\"INT16\"}}]", JsonElement.class); + assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); + + codecs = new Codec[]{ + new AsTypeCodec(DataType.FLOAT64, DataType.INT16), + new GzipCompression() + }; + jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); + expected = gson.fromJson( + "[{\"name\":\"astype\",\"configuration\":{\"dataType\":\"FLOAT64\",\"encodedType\":\"INT16\"}},{\"level\":-1,\"useZlib\":false}]", + JsonElement.class); assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); } From 6b1e9a188d71e87abd7a925e4fe358bd7d2717f0 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 12 Aug 2024 16:38:52 -0400 Subject: [PATCH 033/423] test: deserialization behavior --- .../n5/serialization/CodecSerialization.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java index c5d09a1da..1fc6031c4 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java @@ -25,12 +25,15 @@ public class CodecSerialization { @Before public void before() { - final GsonBuilder builder = new GsonBuilder(); - builder.registerTypeAdapter(IdentityCodec.class, NameConfigAdapter.getJsonAdapter(IdentityCodec.class)); - builder.registerTypeAdapter(AsTypeCodec.class, NameConfigAdapter.getJsonAdapter(AsTypeCodec.class)); - builder.registerTypeAdapter(FixedScaleOffsetCodec.class, + final GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.registerTypeAdapter(IdentityCodec.class, NameConfigAdapter.getJsonAdapter(IdentityCodec.class)); + gsonBuilder.registerTypeAdapter(AsTypeCodec.class, NameConfigAdapter.getJsonAdapter(AsTypeCodec.class)); + gsonBuilder.registerTypeAdapter(FixedScaleOffsetCodec.class, NameConfigAdapter.getJsonAdapter(FixedScaleOffsetCodec.class)); - gson = builder.create(); + // gsonBuilder.registerTypeAdapter(Codec.class, + // NameConfigAdapter.getJsonAdapter(Codec.class)); + + gson = gsonBuilder.create(); } @Test @@ -66,6 +69,8 @@ public void testSerializeCodecArray() { JsonElement.class); assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); + // final Codec[] codecsDeserialized = gson.fromJson(expected, Codec[].class); + // System.out.println(Arrays.toString(codecsDeserialized)); codecs = new Codec[]{ new AsTypeCodec(DataType.FLOAT64, DataType.INT16), From 664007b7cb5e9da951aa58901bd6a06a73a63fa5 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 14 Aug 2024 12:15:40 -0400 Subject: [PATCH 034/423] feat: more shard and codec work --- .../janelia/saalfeldlab/n5/CodecAdapter.java | 121 ------------------ .../saalfeldlab/n5/DatasetAttributes.java | 68 ++++++++++ .../saalfeldlab/n5/DefaultBlockReader.java | 4 +- .../n5/FileSystemKeyValueAccess.java | 4 +- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 10 +- .../saalfeldlab/n5/GzipCompression.java | 6 + .../saalfeldlab/n5/KeyValueAccess.java | 2 +- .../org/janelia/saalfeldlab/n5/N5Writer.java | 24 +++- .../saalfeldlab/n5/NameConfigAdapter.java | 25 +++- .../saalfeldlab/n5/RawCompression.java | 1 + .../n5/ShardedDatasetAttributes.java | 15 +-- .../saalfeldlab/n5/codec/BytesCodec.java | 15 +-- .../codec/checksum/Crc32cChecksumCodec.java | 5 +- .../n5/serialization/NameConfig.java | 1 + .../saalfeldlab/n5/shard/AbstractShard.java | 2 +- .../saalfeldlab/n5/shard/ShardIndex.java | 8 +- .../saalfeldlab/n5/shard/ShardReader.java | 21 ++- .../saalfeldlab/n5/shard/ShardWriter.java | 4 +- .../saalfeldlab/n5/shard/ShardingCodec.java | 90 ++++++++----- .../n5/shard/ShardingConfiguration.java | 92 ------------- .../saalfeldlab/n5/shard/VirtualShard.java | 68 +++++++++- .../n5/serialization/CodecSerialization.java | 13 +- .../saalfeldlab/n5/shard/ShardDemos.java | 42 +++--- 23 files changed, 308 insertions(+), 333 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingConfiguration.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java deleted file mode 100644 index 785380ad6..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/CodecAdapter.java +++ /dev/null @@ -1,121 +0,0 @@ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import java.lang.reflect.Type; - -import org.janelia.saalfeldlab.n5.codec.BytesCodec; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.FixedScaleOffsetCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration; - -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; - -public class CodecAdapter implements JsonDeserializer, JsonSerializer { - - @Override - public JsonElement serialize( - final Codec codec, - final Type typeOfSrc, - final JsonSerializationContext context) { - - if (codec.getType().equals(FixedScaleOffsetCodec.TYPE)) { - final FixedScaleOffsetCodec c = (FixedScaleOffsetCodec)codec; - final JsonObject obj = new JsonObject(); - obj.addProperty("name", c.getType()); - obj.addProperty("scale", c.getScale()); - obj.addProperty("offset", c.getOffset()); - obj.addProperty("type", c.getType().toString().toLowerCase()); - obj.addProperty("encodedType", c.getEncodedDataType().toString().toLowerCase()); - return obj; - } - else if (codec.getType().equals(ShardingCodec.TYPE)) { - final ShardingCodec sharding = (ShardingCodec)codec; - final JsonObject obj = new JsonObject(); - obj.addProperty("name", sharding.getType()); - obj.add("configuration", context.serialize(sharding.getConfiguration())); - return obj; - } - else if (codec.getType().equals(BytesCodec.TYPE)) { - final BytesCodec bytes = (BytesCodec)codec; - final JsonObject obj = new JsonObject(); - obj.addProperty("type", bytes.getType()); - - final JsonObject config = new JsonObject(); - config.addProperty("endian", bytes.getType()); - obj.add("configuration", config); - - return obj; - } - - return JsonNull.INSTANCE; - } - - @Override - public Codec deserialize( - final JsonElement json, - final Type typeOfT, - final JsonDeserializationContext context) throws JsonParseException { - - if (json == null) - return null; - else if (!json.isJsonObject()) - return null; - - final JsonObject jsonObject = json.getAsJsonObject(); - if (jsonObject.has("type")) { - - final String type = jsonObject.get("type").getAsString(); - if (type.equals(FixedScaleOffsetCodec.TYPE)) { - - return new FixedScaleOffsetCodec( - jsonObject.get("scale").getAsDouble(), - jsonObject.get("offset").getAsDouble(), - DataType.valueOf(jsonObject.get("type").getAsString().toUpperCase()), - DataType.valueOf(jsonObject.get("encodedType").getAsString().toUpperCase())); - } - else if (type.equals(ShardingCodec.TYPE)) { - return new ShardingCodec( - context.deserialize(jsonObject.get("configuration"), ShardingConfiguration.class)); - } else if (type.equals(BytesCodec.TYPE)) { - - // TODO implement - return new BytesCodec(); - } - } - - return null; - } - -} \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 3205312fe..95eec18aa 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -1,11 +1,20 @@ package org.janelia.saalfeldlab.n5; import java.io.Serializable; +import java.lang.reflect.Type; import java.util.Arrays; import java.util.HashMap; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.ComposedCodec; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; /** * Mandatory dataset attributes: @@ -172,4 +181,63 @@ static DatasetAttributes from( return new DatasetAttributes(dimensions, blockSize, dataType, compression, codecs); } + + private static DatasetAttributesAdapter adapter = null; + public static DatasetAttributesAdapter getJsonAdapter() { + if (adapter == null) { + adapter = new DatasetAttributesAdapter(); + } + return adapter; + } + + public static class DatasetAttributesAdapter implements JsonSerializer, JsonDeserializer { + + @Override public DatasetAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { + + final JsonObject obj = json.getAsJsonObject(); + if (!obj.has(DIMENSIONS_KEY) || !obj.has(BLOCK_SIZE_KEY) || !obj.has(DATA_TYPE_KEY) || !obj.has(COMPRESSION_KEY)) + return null; + + final long[] dimensions = context.deserialize(obj.get(DIMENSIONS_KEY), long[].class); + final int[] blockSize = context.deserialize(obj.get(BLOCK_SIZE_KEY), int[].class); + final DataType dataType = context.deserialize(obj.get(DATA_TYPE_KEY), DataType.class); + final Compression compression = context.deserialize(obj.get(COMPRESSION_KEY), Compression.class); + final Codec[] codecs; + if (obj.has(CODEC_KEY)) { + codecs = context.deserialize(obj.get(CODEC_KEY), Codec[].class); + } else codecs = new Codec[0]; + + for (Codec codec : codecs) { + if (codec instanceof ShardingCodec) { + ShardingCodec shardingCodec = (ShardingCodec)codec; + return new ShardedDatasetAttributes( + dimensions, + shardingCodec.getBlockSize(), + blockSize, + shardingCodec.getIndexLocation(), + dataType, + compression, + codecs + ); + } + } + return new DatasetAttributes(dimensions, blockSize, dataType, compression); + } + + @Override public JsonElement serialize(DatasetAttributes src, Type typeOfSrc, JsonSerializationContext context) { + + final JsonObject obj = new JsonObject(); + obj.add(DIMENSIONS_KEY, context.serialize(src.dimensions)); + obj.add(BLOCK_SIZE_KEY, context.serialize(src.blockSize)); + obj.add(DATA_TYPE_KEY, context.serialize(src.dataType)); + obj.add(COMPRESSION_KEY, CompressionAdapter.getJsonAdapter().serialize(src.compression, src.compression.getClass(), context)); + + //TODO Caleb: Per the zarr v3 spec, codecs is necessary and cannot be an empty list, since it always needs at least + // one array -> bytes codec. Even in the case of no compressor, there should always be at least the + // `bytes` codec it seems. Consider how we want to handle this in N5 + obj.add(CODEC_KEY, context.serialize(src.codecs)); + + return obj; + } + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 0f70baea7..9f9b97727 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -30,7 +30,7 @@ import java.io.InputStream; import java.nio.ByteBuffer; -import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; /** * Default implementation of {@link BlockReader}. @@ -149,7 +149,7 @@ public static > void readFromStream(final B dataBlock, dataBlock.readData(buffer); } - public static long getShardIndex(final ShardingConfiguration shardingConfiguration, final long[] gridPosition) { + public static long getShardIndex(final ShardingCodec shardingCodec, final long[] gridPosition) { // TODO implement return -1; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 967cb6e2b..bb2cf2db9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -218,10 +218,10 @@ public LockedFileChannel lockForWriting(final String normalPath) throws IOExcept } @Override - public LockedFileChannel lockForWriting(final String normalPath, final long startByte, final long endByte) + public LockedFileChannel lockForWriting(final String normalPath, final long startByte, final long size) throws IOException { - return new LockedFileChannel(normalPath, false, startByte, endByte); + return new LockedFileChannel(normalPath, false, startByte, size); } public LockedFileChannel lockForReading(final Path path) throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index c09d1e147..157c9bdd0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -30,12 +30,13 @@ import java.io.Writer; import java.lang.reflect.Array; import java.lang.reflect.Type; +import java.nio.ByteOrder; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; +import org.janelia.saalfeldlab.n5.codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -45,6 +46,7 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; /** * Utility class for working with JSON. @@ -56,9 +58,11 @@ public interface GsonUtils { static Gson registerGson(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(ShardingConfiguration.class, new ShardingConfiguration.ShardingConfigurationAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, DatasetAttributes.getJsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); + gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, BytesCodec.byteOrderAdapter); + gsonBuilder.registerTypeHierarchyAdapter(ShardingCodec.IndexLocation.class, ShardingCodec.indexLocationAdapter); gsonBuilder.disableHtmlEscaping(); return gsonBuilder.create(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index 0b6d734d8..ecaf2f0eb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -37,13 +37,19 @@ import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; import org.apache.commons.compress.compressors.gzip.GzipParameters; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("gzip") +@NameConfig.Name("gzip") public class GzipCompression implements DefaultBlockReader, DefaultBlockWriter, Compression { private static final long serialVersionUID = 8630847239813334263L; @CompressionParameter + @NameConfig.Parameter + //TODO Caleb: How to handle serialization of parameter-less constructor. + // For N5, default is -1, for zarr, range is 0-9 and is required. + // How to map -1 to some default (1?) when serializing to zarr? private final int level; @CompressionParameter diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index d199bf670..a4fa42b82 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -180,7 +180,7 @@ public LockedChannel lockForReading(String normalPath, final long startByte, fin */ public LockedChannel lockForWriting(final String normalPath) throws IOException; - public LockedChannel lockForWriting(String normalPath, final long startByte, final long endByte) + public LockedChannel lockForWriting(String normalPath, final long startByte, final long size) throws IOException; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index f4645a802..0c734e163 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -37,8 +37,7 @@ import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.shard.Shard; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration; -import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; /** * A simple structured container API for hierarchies of chunked @@ -141,7 +140,7 @@ default void setDatasetAttributes( final String datasetPath, final DatasetAttributes datasetAttributes) throws N5Exception { - setAttributes(datasetPath, datasetAttributes.asMap()); + setAttribute(datasetPath, "/", datasetAttributes); } /** @@ -220,8 +219,7 @@ default void createDataset( final DataType dataType, final Compression compression) throws N5Exception { - final Codec[] codecs = new Codec[]{new ShardingCodec( - new ShardingConfiguration(blockSize, null, null, IndexLocation.END))}; + final Codec[] codecs = new Codec[]{new ShardingCodec(blockSize, null, null, IndexLocation.END)}; createDataset(datasetPath, new DatasetAttributes(dimensions, shardSize, dataType, compression, codecs)); } @@ -284,6 +282,22 @@ void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception; + /** + * Write multiple data blocks, useful for request aggregation . + * + * @param datasetPath dataset path + * @param datasetAttributes the dataset attributes + * @param dataBlocks the data block + * @param the data block data type + * @throws N5Exception the exception + */ + default void writeBlocks( + final String datasetPath, + final DatasetAttributes datasetAttributes, + final DataBlock... dataBlocks) throws N5Exception { + //TODO Caleb: write this + } + /** * Writes a complete {@link Shard} to a dataset. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java index 85768d180..5081f8215 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java @@ -131,6 +131,7 @@ public JsonElement serialize( final JsonSerializationContext context) { final Class clazz = (Class)object.getClass(); + final String name = clazz.getAnnotation(NameConfig.Name.class).value(); final String prefix = type.getAnnotation(NameConfig.Prefix.class).value(); final String type = prefix + "." + name; @@ -138,7 +139,6 @@ public JsonElement serialize( final JsonObject json = new JsonObject(); json.addProperty("name", name); final JsonObject configuration = new JsonObject(); - json.add("configuration", configuration); final HashMap parameterTypes = parameters.get(type); final HashMap parameterNameMap = parameterNames.get(type); @@ -158,6 +158,8 @@ public JsonElement serialize( configuration.add(parameterNameMap.get(fieldName), serialized); } + if (!configuration.isEmpty()) + json.add("configuration", configuration); } catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) { e.printStackTrace(System.err); return null; @@ -176,14 +178,22 @@ public T deserialize( final JsonObject objectJson = json.getAsJsonObject(); final String name = objectJson.getAsJsonPrimitive("name").getAsString(); - if (name == null) - return null; - final JsonObject configuration = objectJson.getAsJsonObject("configuration"); - if (configuration == null) + if (name == null) { return null; + } final String type = prefix + "." + name; + final JsonObject configuration = objectJson.getAsJsonObject("configuration"); + /* It's ok to be null if all parameters are optional. + * Otherwise, return*/ + if (configuration == null) { + for (Field field : parameters.get(type).values()) { + if (!field.getAnnotation(NameConfig.Parameter.class).optional()) + return null; + } + } + final Constructor constructor = constructors.get(type); constructor.setAccessible(true); final T object; @@ -195,8 +205,8 @@ public T deserialize( final String fieldName = parameterType.getKey(); final String paramName = parameterNameMap.get(fieldName); final JsonElement paramJson = configuration.get(paramName); + final Field field = parameterType.getValue(); if (paramJson != null) { - final Field field = parameterType.getValue(); final Object parameter; if (field.getAnnotation(N5Annotations.ReverseArray.class) != null) { final JsonArray reversedArray = reverseJsonArray(paramJson); @@ -204,6 +214,9 @@ public T deserialize( } else parameter = context.deserialize(paramJson, field.getType()); ReflectionUtils.setFieldValue(object, fieldName, parameter); + } else if (!field.getAnnotation(NameConfig.Parameter.class).optional()) { + /* if param is null, and not optional, return null */ + return null; } } } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index 7d1327b0d..ebd58b38b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -30,6 +30,7 @@ import java.io.OutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("raw") public class RawCompression implements DefaultBlockReader, DefaultBlockWriter, Compression { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index 2825eff91..60244334a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -4,8 +4,7 @@ import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration; -import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; public class ShardedDatasetAttributes extends DatasetAttributes { @@ -24,7 +23,7 @@ public ShardedDatasetAttributes( final Compression compression, final Codec[] codecs) { - super(dimensions, blockSize, dataType, compression, codecs); + super(dimensions, shardSize, dataType, compression, codecs); this.shardSize = shardSize; this.indexLocation = shardIndexLocation; @@ -106,11 +105,11 @@ public long getNumBlocks() { public static int[] getBlockSize(Codec[] codecs) { - //TODO Caleb: Move this? - return Arrays.stream(codecs) - .filter(ShardingCodec::isShardingCodec) - .map(x -> ((ShardingCodec)x).getConfiguration()) - .map(ShardingConfiguration::getBlockSize).findFirst().orElse(null); + for (Codec codec : codecs) + if (codec instanceof ShardingCodec) + return ((ShardingCodec)codec).getBlockSize(); + + return null; } public IndexLocation getIndexLocation() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java index 5f0bb4abb..a16d0ec3b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java @@ -14,18 +14,16 @@ import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -@NameConfig.Name("bytes") +@NameConfig.Name(value = BytesCodec.TYPE) public class BytesCodec implements Codec { private static final long serialVersionUID = 3523505403978222360L; - public static String TYPE = "bytes"; + public static final String TYPE = "bytes"; - @NameConfig.Parameter("endian") + @NameConfig.Parameter(value = "endian", optional = true) protected final ByteOrder byteOrder; - protected transient final byte[] array; - public BytesCodec() { this(ByteOrder.LITTLE_ENDIAN); @@ -33,13 +31,8 @@ public BytesCodec() { public BytesCodec(final ByteOrder byteOrder) { - this(byteOrder, 256); - } - - public BytesCodec(final ByteOrder byteOrder, final int N) { - this.byteOrder = byteOrder; - this.array = new byte[N]; + } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java index 0b85ed5a9..f7c036083 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java @@ -1,13 +1,16 @@ package org.janelia.saalfeldlab.n5.codec.checksum; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + import java.nio.ByteBuffer; import java.util.zip.CRC32; +@NameConfig.Name(Crc32cChecksumCodec.TYPE) public class Crc32cChecksumCodec extends ChecksumCodec { private static final long serialVersionUID = 7424151868725442500L; - public static transient final String TYPE = "crc32c"; + public static final String TYPE = "crc32c"; public Crc32cChecksumCodec() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java b/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java index 229653826..2ccb122ea 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java @@ -31,6 +31,7 @@ public interface NameConfig extends Serializable { @Target(ElementType.FIELD) @interface Parameter { String value() default ""; + boolean optional() default false; } default String getType() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java index f642075a4..fc30eaa6c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java @@ -7,7 +7,7 @@ public abstract class AbstractShard implements Shard { protected final ShardedDatasetAttributes datasetAttributes; - protected final ShardIndex index; + protected ShardIndex index; private final long[] gridPosition; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 1d0da8a49..31f510492 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -20,7 +20,7 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; -import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; public class ShardIndex extends LongArrayDataBlock { @@ -83,8 +83,7 @@ private int getNumBytesIndex(long... gridPosition) { public static ShardIndex read(final KeyValueAccess keyValueAccess, final String key, final ShardedDatasetAttributes datasetAttributes) throws IOException { - return read(keyValueAccess, key, datasetAttributes.getShardBlockGridSize(), - datasetAttributes.getIndexLocation()); + return read(keyValueAccess, key, datasetAttributes.getShardBlockGridSize(), datasetAttributes.getIndexLocation()); } public static ShardIndex read( @@ -144,8 +143,7 @@ public static IndexByteBounds byteBounds(ShardedDatasetAttributes datasetAttribu return byteBounds(indexShape, datasetAttributes.getIndexLocation(), objectSize); } - public static IndexByteBounds byteBounds(final int[] indexShape, final IndexLocation indexLocation, - final long objectSize) { + public static IndexByteBounds byteBounds(final int[] indexShape, final IndexLocation indexLocation, final long objectSize) { final int indexSize = (int)Arrays.stream(indexShape).reduce(1, (x, y) -> x * y); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java index 0ab5d276a..4584d1dbd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java @@ -1,10 +1,5 @@ package org.janelia.saalfeldlab.n5.shard; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; - import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DefaultBlockReader; @@ -15,7 +10,12 @@ import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.IdentityCodec; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; public class ShardReader { @@ -79,11 +79,10 @@ private static ShardedDatasetAttributes buildTestAttributes() { final Codec[] codecs = new Codec[]{ new IdentityCodec(), new ShardingCodec( - new ShardingConfiguration( - new int[]{2, 2}, - new Codec[]{new RawCompression(), new IdentityCodec()}, - new Codec[]{new Crc32cChecksumCodec()}, - IndexLocation.END) + new int[]{2, 2}, + new Codec[]{new RawCompression(), new IdentityCodec()}, + new Codec[]{new Crc32cChecksumCodec()}, + IndexLocation.END ) }; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java index d1ade67f4..1c18e5643 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java @@ -54,7 +54,7 @@ public void write(final OutputStream out) throws IOException { // TODO need codecs // prepareForWriting(); - // if (datasetAttributes.getShardingConfiguration().areIndexesAtStart()) { + // if (datasetAttributes.getShardingConfiguration().getIndexLocation()) { // writeIndexes(out); // writeBlocks(out); // } else { @@ -63,7 +63,7 @@ public void write(final OutputStream out) throws IOException { // } prepareForWritingDataBlock(); - if (datasetAttributes.getIndexLocation() == ShardingConfiguration.IndexLocation.START) { + if (datasetAttributes.getIndexLocation() == ShardingCodec.IndexLocation.START) { writeIndexBlock(out); writeBlocks(out); } else { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index aa0f66846..7297aab5d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -1,32 +1,54 @@ package org.janelia.saalfeldlab.n5.shard; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.reflect.Type; - -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; - import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; -import com.google.gson.JsonObject; import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Type; + +@NameConfig.Name(ShardingCodec.TYPE) public class ShardingCodec implements Codec { private static final long serialVersionUID = -5879797314954717810L; public static final String TYPE = "sharding_indexed"; - private final ShardingConfiguration configuration; + public final static String CHUNK_SHAPE_KEY = "chunk_shape"; + public static final String INDEX_LOCATION_KEY = "index_location"; + public static final String CODECS_KEY = "codecs"; + public static final String INDEX_CODECS_KEY = "index_codecs"; + + public enum IndexLocation { + START, END + } + + @NameConfig.Parameter(CHUNK_SHAPE_KEY) + private final int[] blockSize; + + @NameConfig.Parameter(CODECS_KEY) + private final Codec[] codecs; - public ShardingCodec(ShardingConfiguration configuration) { + @NameConfig.Parameter(INDEX_CODECS_KEY) + private final Codec[] indexCodecs; - this.configuration = configuration; + @NameConfig.Parameter(INDEX_LOCATION_KEY) + private final IndexLocation indexLocation; + + private ShardingCodec() { + + blockSize = null; + codecs = null; + indexCodecs = null; + indexLocation = null; } public ShardingCodec( @@ -35,12 +57,20 @@ public ShardingCodec( final Codec[] indexCodecs, final IndexLocation indexLocation) { - this.configuration = new ShardingConfiguration(blockSize, codecs, indexCodecs, indexLocation); + this.blockSize = blockSize; + this.codecs = codecs; + this.indexCodecs = indexCodecs; + this.indexLocation = indexLocation; + } + + public int[] getBlockSize() { + + return blockSize; } - public ShardingConfiguration getConfiguration() { + public IndexLocation getIndexLocation() { - return configuration; + return indexLocation; } @Override @@ -64,33 +94,27 @@ public static boolean isShardingCodec(final Codec codec) { return codec instanceof ShardingCodec; } - // public static void TypeAd - public static class ShardingCodecAdapter implements JsonDeserializer, JsonSerializer { + @Override + public String getType() { - @Override - public JsonElement serialize(ShardingCodec src, Type typeOfSrc, JsonSerializationContext context) { + return TYPE; + } - final JsonObject jsonObj = new JsonObject(); + public static IndexLocationAdapter indexLocationAdapter = new IndexLocationAdapter(); - jsonObj.addProperty("name", ShardingCodec.TYPE); - // context.serialize(typeOfSrc); + public static class IndexLocationAdapter implements JsonSerializer, JsonDeserializer { - return jsonObj; - } + @Override public IndexLocation deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { - @Override - public ShardingCodec deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) - throws JsonParseException { + if (!json.isJsonPrimitive()) return null; - return null; + return IndexLocation.valueOf(json.getAsString().toUpperCase()); } - } - - @Override - public String getType() { + @Override public JsonElement serialize(IndexLocation src, Type typeOfSrc, JsonSerializationContext context) { - return TYPE; + return new JsonPrimitive(src.name().toLowerCase()); + } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingConfiguration.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingConfiguration.java deleted file mode 100644 index ce520700d..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingConfiguration.java +++ /dev/null @@ -1,92 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import java.lang.reflect.Type; -import java.util.Arrays; - -import org.janelia.saalfeldlab.n5.codec.Codec; - -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; - -public class ShardingConfiguration { - - public static final String CHUNK_SHAPE_KEY = "chunk_shape"; - public static final String INDEX_LOCATION_KEY = "index_location"; - public static final String CODECS_KEY = "codecs"; - public static final String INDEX_CODECS_KEY = "index_codecs"; - - public enum IndexLocation { - START, END - } - - protected int[] blockSize; - protected Codec[] codecs; - protected Codec[] indexCodecs; - protected IndexLocation indexLocation; - - public ShardingConfiguration( - final int[] blockSize, - final Codec[] codecs, - final Codec[] indexCodecs, - final IndexLocation indexLocation) { - - this.blockSize = blockSize; - this.codecs = codecs; - this.indexCodecs = indexCodecs; - this.indexLocation = indexLocation; - } - - public int[] getBlockSize() { - - return blockSize; - } - - public boolean areIndexesAtStart() { - - return indexLocation == IndexLocation.START; - } - - public static class ShardingConfigurationAdapter - implements JsonDeserializer, JsonSerializer { - - @Override - public JsonElement serialize(ShardingConfiguration src, Type typeOfSrc, JsonSerializationContext context) { - - if( anyShardingCodecs(src.codecs) || anyShardingCodecs(src.indexCodecs)) - return JsonNull.INSTANCE; - - final JsonObject jsonObj = new JsonObject(); - jsonObj.add(CHUNK_SHAPE_KEY, context.serialize(src.blockSize)); - jsonObj.add(INDEX_LOCATION_KEY, context.serialize(src.indexLocation.toString())); - jsonObj.add(CODECS_KEY, context.serialize(src.codecs)); - jsonObj.add(INDEX_CODECS_KEY, context.serialize(src.indexCodecs)); - - return jsonObj; - } - - @Override - public ShardingConfiguration deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) - throws JsonParseException { - - return null; - } - - public boolean anyShardingCodecs(final Codec[] codecs) { - - if (codecs == null) - return false; - - return Arrays.stream(codecs).anyMatch(c -> { - return (c instanceof ShardingCodec); - }); - } - - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 699c45bf0..d1dce0e90 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -1,7 +1,9 @@ package org.janelia.saalfeldlab.n5.shard; import java.io.IOException; +import java.io.OutputStream; import java.io.UncheckedIOException; +import java.nio.file.NoSuchFileException; import java.util.Arrays; import org.janelia.saalfeldlab.n5.DataBlock; @@ -63,18 +65,21 @@ public void writeBlock(final DataBlock block) { throw new N5IOException("Attempted to write block in the wrong shard."); final ShardIndex idx = getIndex(); - final long startByte = idx.getOffset(relativePosition); - final long endByte = startByte + idx.getNumBytes(relativePosition); + final long startByte = idx.getOffset(relativePosition) == Shard.EMPTY_INDEX_NBYTES ? 0 : idx.getOffset(relativePosition); + final long size = idx.getNumBytes(relativePosition) == Shard.EMPTY_INDEX_NBYTES ? Long.MAX_VALUE : idx.getNumBytes(relativePosition); // TODO this assumes that the block exists in the shard and // that the available space is sufficient. Should generalize - try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(path, startByte, endByte)) { + try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(path, startByte, size)) { // TODO codecs - datasetAttributes.getCompression().getWriter().write(block, lockedChannel.newOutputStream()); + final CountingOutputStream out = new CountingOutputStream(lockedChannel.newOutputStream()); + datasetAttributes.getCompression().getWriter().write(block, out); // TODO update index when we know how many bytes were written - + idx.set(startByte, out.getNumBytes(), relativePosition); + out.write(index.toByteBuffer().array()); + out.realClose(); } catch (final IOException | UncheckedIOException e) { throw new N5IOException("Failed to read block from " + path, e); } @@ -95,7 +100,8 @@ private static int numBlockElements(DatasetAttributes datasetAttributes) { public ShardIndex createIndex() { // Empty index of the correct size - return new ShardIndex(datasetAttributes.getShardBlockGridSize()); + index = new ShardIndex(datasetAttributes.getShardBlockGridSize()); + return index; } @Override @@ -104,9 +110,57 @@ public ShardIndex getIndex() { try { final ShardIndex result = ShardIndex.read(keyValueAccess, path, datasetAttributes); return result == null ? createIndex() : result; - } catch (final IOException e) { + } catch (final NoSuchFileException e) { + return createIndex(); + } catch (IOException e) { throw new N5IOException("Failed to read index at " + path, e); } } + + static class CountingOutputStream extends OutputStream { + private final OutputStream out; + private long numBytes; + + public CountingOutputStream(OutputStream out) { + this.out = out; + this.numBytes = 0; + } + + @Override + public void write(int b) throws IOException { + out.write(b); + numBytes++; + } + + @Override + public void write(byte[] b) throws IOException { + out.write(b); + numBytes += b.length; + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + numBytes += len; + } + + @Override + public void flush() throws IOException { + out.flush(); + } + + @Override + public void close() throws IOException { + + } + + private void realClose() throws IOException { + out.close(); + } + + public long getNumBytes() { + return numBytes; + } + } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java index 1fc6031c4..94a07c7f3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java @@ -18,6 +18,8 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import java.util.Arrays; + public class CodecSerialization { private Gson gson; @@ -28,10 +30,9 @@ public void before() { final GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.registerTypeAdapter(IdentityCodec.class, NameConfigAdapter.getJsonAdapter(IdentityCodec.class)); gsonBuilder.registerTypeAdapter(AsTypeCodec.class, NameConfigAdapter.getJsonAdapter(AsTypeCodec.class)); - gsonBuilder.registerTypeAdapter(FixedScaleOffsetCodec.class, - NameConfigAdapter.getJsonAdapter(FixedScaleOffsetCodec.class)); - // gsonBuilder.registerTypeAdapter(Codec.class, - // NameConfigAdapter.getJsonAdapter(Codec.class)); + gsonBuilder.registerTypeAdapter(FixedScaleOffsetCodec.class, NameConfigAdapter.getJsonAdapter(FixedScaleOffsetCodec.class)); + gsonBuilder.registerTypeAdapter(Codec.class, + NameConfigAdapter.getJsonAdapter(Codec.class)); gson = gsonBuilder.create(); } @@ -69,8 +70,8 @@ public void testSerializeCodecArray() { JsonElement.class); assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); - // final Codec[] codecsDeserialized = gson.fromJson(expected, Codec[].class); - // System.out.println(Arrays.toString(codecsDeserialized)); + final Codec[] codecsDeserialized = gson.fromJson(expected, Codec[].class); + System.out.println(Arrays.toString(codecsDeserialized)); codecs = new Codec[]{ new AsTypeCodec(DataType.FLOAT64, DataType.INT16), diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java index 32574d4b5..8814231a3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -1,24 +1,25 @@ package org.janelia.saalfeldlab.n5.shard; -import java.net.MalformedURLException; -import java.nio.file.FileSystems; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; - +import com.google.gson.GsonBuilder; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; +import org.janelia.saalfeldlab.n5.GzipCompression; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.IdentityCodec; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingConfiguration.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.junit.Test; +import java.net.MalformedURLException; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; public class ShardDemos { @@ -27,7 +28,6 @@ public static void main(String[] args) throws MalformedURLException { final Path p = Paths.get("src/test/resources/shardExamples/test.zarr/mid_sharded/c/0/0"); System.out.println(p); - final String key = p.toString(); final ShardedDatasetAttributes dsetAttrs = new ShardedDatasetAttributes(new long[]{6, 4}, new int[]{6, 4}, new int[]{3, 2}, IndexLocation.END, DataType.UINT8, new RawCompression(), null); @@ -55,7 +55,13 @@ public static void main(String[] args) throws MalformedURLException { @Test public void writeReadBlockTest() { - final N5Writer writer = N5Factory.createWriter("src/test/resources/shardExamples/test.n5"); + final N5Factory factory = new N5Factory(); + final GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.setPrettyPrinting(); + factory.gsonBuilder(gsonBuilder); + factory.cacheAttributes(false); + + final N5Writer writer = factory.openWriter("src/test/resources/shardExamples/test.n5"); final ShardedDatasetAttributes datasetAttributes = new ShardedDatasetAttributes( new long[]{8, 8}, @@ -67,19 +73,23 @@ public void writeReadBlockTest() { new Codec[]{ new IdentityCodec(), new ShardingCodec( - new ShardingConfiguration( - new int[]{2, 2}, - new Codec[]{new RawCompression(), new IdentityCodec()}, - new Codec[]{new Crc32cChecksumCodec()}, - IndexLocation.END) + new int[]{2, 2}, + new Codec[]{new GzipCompression(4), new IdentityCodec()}, + new Codec[]{new Crc32cChecksumCodec()}, + IndexLocation.END ) } ); writer.createDataset("shard", datasetAttributes); - final DataBlock dataBlock = datasetAttributes.getDataType().createDataBlock(datasetAttributes.getBlockSize(), new long[]{0, 0}, 2 * 2); + final DataBlock dataBlock = datasetAttributes.getDataType().createDataBlock(datasetAttributes.getBlockSize(), new long[]{0, 0}, 2 * 2); + byte[] data = (byte[])dataBlock.getData(); + for (int i = 0; i < data.length; i++) { + data[i] = (byte)i; + } + writer.deleteBlock("shard", 0,0 ); writer.writeBlock("shard", datasetAttributes, dataBlock); - writer.readBlock("shard", datasetAttributes, 0,0); + writer.readBlock("shard", datasetAttributes, 0, 0); } } From 21ab72c34b63a8ddf66ef162b829521d4d7e1412 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 14 Aug 2024 13:07:47 -0400 Subject: [PATCH 035/423] test: minor updates --- .../saalfeldlab/n5/codec/BytesTests.java | 11 ++++---- .../n5/serialization/CodecSerialization.java | 25 +++++++++++++------ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java index 8eeb83455..e8bf64b04 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java @@ -1,6 +1,10 @@ package org.janelia.saalfeldlab.n5.codec; -import com.google.gson.GsonBuilder; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteOrder; + import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.N5Writer; @@ -9,10 +13,7 @@ import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.junit.Test; -import java.nio.ByteOrder; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import com.google.gson.GsonBuilder; public class BytesTests { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java index 94a07c7f3..f12150b6c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java @@ -1,6 +1,7 @@ package org.janelia.saalfeldlab.n5.serialization; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.GzipCompression; @@ -18,8 +19,6 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import java.util.Arrays; - public class CodecSerialization { private Gson gson; @@ -30,9 +29,12 @@ public void before() { final GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.registerTypeAdapter(IdentityCodec.class, NameConfigAdapter.getJsonAdapter(IdentityCodec.class)); gsonBuilder.registerTypeAdapter(AsTypeCodec.class, NameConfigAdapter.getJsonAdapter(AsTypeCodec.class)); - gsonBuilder.registerTypeAdapter(FixedScaleOffsetCodec.class, NameConfigAdapter.getJsonAdapter(FixedScaleOffsetCodec.class)); - gsonBuilder.registerTypeAdapter(Codec.class, - NameConfigAdapter.getJsonAdapter(Codec.class)); + gsonBuilder.registerTypeAdapter(FixedScaleOffsetCodec.class, + NameConfigAdapter.getJsonAdapter(FixedScaleOffsetCodec.class)); + gsonBuilder.registerTypeAdapter(GzipCompression.class, + NameConfigAdapter.getJsonAdapter(GzipCompression.class)); + gsonBuilder.registerTypeAdapter(Codec.class, + NameConfigAdapter.getJsonAdapter(Codec.class)); gson = gsonBuilder.create(); } @@ -70,8 +72,10 @@ public void testSerializeCodecArray() { JsonElement.class); assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); - final Codec[] codecsDeserialized = gson.fromJson(expected, Codec[].class); - System.out.println(Arrays.toString(codecsDeserialized)); + Codec[] codecsDeserialized = gson.fromJson(expected, Codec[].class); + assertEquals("codecs length not 2", 2, codecsDeserialized.length); + assertTrue("first codec not identity", codecsDeserialized[0] instanceof IdentityCodec); + assertTrue("second codec not asType", codecsDeserialized[1] instanceof AsTypeCodec); codecs = new Codec[]{ new AsTypeCodec(DataType.FLOAT64, DataType.INT16), @@ -79,9 +83,14 @@ public void testSerializeCodecArray() { }; jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); expected = gson.fromJson( - "[{\"name\":\"astype\",\"configuration\":{\"dataType\":\"FLOAT64\",\"encodedType\":\"INT16\"}},{\"level\":-1,\"useZlib\":false}]", + "[{\"name\":\"astype\",\"configuration\":{\"dataType\":\"FLOAT64\",\"encodedType\":\"INT16\"}},{\"name\":\"gzip\",\"configuration\":{\"level\":-1,\"use_z_lib\":false}}]", JsonElement.class); assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); + + codecsDeserialized = gson.fromJson(expected, Codec[].class); + assertEquals("codecs length not 2", 2, codecsDeserialized.length); + assertTrue("first codec not asType", codecsDeserialized[0] instanceof AsTypeCodec); + assertTrue("second codec not gzip", codecsDeserialized[1] instanceof GzipCompression); } } From 314b1996df72b528fcc7798ddaecd9e9c78ebb1b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 14 Aug 2024 13:08:05 -0400 Subject: [PATCH 036/423] perf: BlockReader have read call static method --- .../java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 9f9b97727..3d8319d25 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -47,13 +47,9 @@ public default > void read( final B dataBlock, final InputStream in) throws IOException { - final ByteBuffer buffer = dataBlock.toByteBuffer(); - // do not try with this input stream because subsequent block reads may happen if the stream points to a shard final InputStream inflater = getInputStream(in); - final DataInputStream dis = new DataInputStream(inflater); - dis.readFully(buffer.array()); - dataBlock.readData(buffer); + readFromStream(dataBlock, inflater); } /** From 5ad00fd58d55a3c03e53a5bdf3c2eed30d2c14c9 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 16 Aug 2024 16:58:21 -0400 Subject: [PATCH 037/423] feat: WIP initial read/write blocks through composed codecs implemention analogous to zarr array->bytes/ bytes->bytes codecs. --- .../saalfeldlab/n5/Bzip2Compression.java | 2 + .../n5/CachedGsonKeyValueN5Reader.java | 3 +- .../janelia/saalfeldlab/n5/Compression.java | 2 +- .../saalfeldlab/n5/DatasetAttributes.java | 84 +++++++------- .../saalfeldlab/n5/DefaultBlockReader.java | 70 ++---------- .../saalfeldlab/n5/DefaultBlockWriter.java | 79 +++---------- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 2 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 3 +- .../janelia/saalfeldlab/n5/GsonN5Reader.java | 30 ++--- .../saalfeldlab/n5/Lz4Compression.java | 2 + .../saalfeldlab/n5/ShortArrayDataBlock.java | 12 ++ .../janelia/saalfeldlab/n5/XzCompression.java | 2 + .../saalfeldlab/n5/codec/AsTypeCodec.java | 13 +-- .../saalfeldlab/n5/codec/BytesCodec.java | 104 ++++++++++++++++-- .../janelia/saalfeldlab/n5/codec/Codec.java | 74 ++++++++++--- .../saalfeldlab/n5/codec/ComposedCodec.java | 16 +-- .../n5/codec/DeterministicSizeCodec.java | 2 +- .../saalfeldlab/n5/codec/IdentityCodec.java | 2 +- .../n5/codec/checksum/ChecksumCodec.java | 20 ++-- .../saalfeldlab/n5/shard/ShardingCodec.java | 2 +- .../saalfeldlab/n5/AbstractN5Test.java | 93 ++++++++++------ .../saalfeldlab/n5/codec/AsTypeTests.java | 6 +- .../saalfeldlab/n5/shard/ShardDemos.java | 6 +- 23 files changed, 339 insertions(+), 290 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 0c40d01a8..49a333f3e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -32,8 +32,10 @@ import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("bzip2") +@NameConfig.Name("bzip2") public class Bzip2Compression implements DefaultBlockReader, DefaultBlockWriter, Compression { private static final long serialVersionUID = -4873117458390529118L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index 324d242e8..d2e8418ed 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -27,6 +27,7 @@ import java.lang.reflect.Type; +import com.google.gson.reflect.TypeToken; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; import org.janelia.saalfeldlab.n5.cache.N5JsonCacheableContainer; @@ -82,7 +83,7 @@ default DatasetAttributes normalGetDatasetAttributes(final String pathName) thro final String normalPath = N5URI.normalizeGroupPath(pathName); final JsonElement attributes = GsonKeyValueN5Reader.super.getAttributes(normalPath); - return createDatasetAttributes(attributes); + return getGson().fromJson(attributes, DatasetAttributes.class); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index ba78cecf7..eac6a0ac8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -43,7 +43,7 @@ * * @author Stephan Saalfeld */ -public interface Compression extends Serializable, Codec { +public interface Compression extends Serializable, Codec.BytesToBytes { /** * Annotation for runtime discovery of compression schemes. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 95eec18aa..ff420fea5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -12,10 +12,12 @@ import com.google.gson.JsonParseException; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import org.janelia.saalfeldlab.n5.codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.ComposedCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; +import javax.xml.crypto.Data; + /** * Mandatory dataset attributes: * @@ -62,13 +64,19 @@ public DatasetAttributes( final int[] blockSize, final DataType dataType, final Compression compression, - final Codec[] codecs ) { + final Codec[] codecs) { this.dimensions = dimensions; this.blockSize = blockSize; this.dataType = dataType; - this.codecs = codecs; this.compression = compression; + if (codecs == null && !(compression instanceof RawCompression)) { + this.codecs = new Codec[]{new BytesCodec(), compression}; + } else if (codecs == null) { + this.codecs = new Codec[]{new BytesCodec()}; + } else { + this.codecs = codecs; + } } public DatasetAttributes( @@ -110,22 +118,6 @@ public Codec[] getCodecs() { return codecs; } - public Codec collectCodecs() { - - if (codecs == null || codecs.length == 0) - return compression; - else if (codecs.length == 1) - return new ComposedCodec(codecs[0], compression); - else { - final Codec[] codecsAndCompresor = new Codec[codecs.length + 1]; - for (int i = 0; i < codecs.length; i++) - codecsAndCompresor[i] = codecs[i]; - - codecsAndCompresor[codecs.length] = compression; - return new ComposedCodec(codecsAndCompresor); - } - } - public HashMap asMap() { final HashMap map = new HashMap<>(); @@ -160,28 +152,29 @@ static DatasetAttributes from( /* version 0 */ if (compression == null) { - switch (compressionVersion0Name) { - case "raw": - compression = new RawCompression(); - break; - case "gzip": - compression = new GzipCompression(); - break; - case "bzip2": - compression = new Bzip2Compression(); - break; - case "lz4": - compression = new Lz4Compression(); - break; - case "xz": - compression = new XzCompression(); - break; - } + compression = getCompressionVersion0(compressionVersion0Name); } return new DatasetAttributes(dimensions, blockSize, dataType, compression, codecs); } + private static Compression getCompressionVersion0(final String compressionVersion0Name) { + + switch (compressionVersion0Name) { + case "raw": + return new RawCompression(); + case "gzip": + return new GzipCompression(); + case "bzip2": + return new Bzip2Compression(); + case "lz4": + return new Lz4Compression(); + case "xz": + return new XzCompression(); + } + return null; + } + private static DatasetAttributesAdapter adapter = null; public static DatasetAttributesAdapter getJsonAdapter() { if (adapter == null) { @@ -201,11 +194,20 @@ public static class DatasetAttributesAdapter implements JsonSerializer bytes codec. Even in the case of no compressor, there should always be at least the - // `bytes` codec it seems. Consider how we want to handle this in N5 obj.add(CODEC_KEY, context.serialize(src.codecs)); return obj; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 3d8319d25..bb3f2639a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -30,6 +30,7 @@ import java.io.InputStream; import java.nio.ByteBuffer; +import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; /** @@ -70,70 +71,19 @@ public static DataBlock readBlock( final DatasetAttributes datasetAttributes, final long[] gridPosition) throws IOException { - final DataInputStream dis = new DataInputStream(in); - final short mode = dis.readShort(); - final int numElements; - final DataBlock dataBlock; - if (mode != 2) { - final int nDim = dis.readShort(); - final int[] blockSize = new int[nDim]; - for (int d = 0; d < nDim; ++d) - blockSize[d] = dis.readInt(); - if (mode == 0) { - numElements = DataBlock.getNumElements(blockSize); - } else { - numElements = dis.readInt(); + Codec.DataBlockInputStream dataBlockStream = null; + InputStream stream = in; + for (Codec codec : datasetAttributes.getCodecs()) { + if (codec instanceof Codec.ArrayToBytes) { + stream = dataBlockStream = ((Codec.ArrayToBytes)codec).decode(datasetAttributes, gridPosition, stream); } - dataBlock = datasetAttributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); - } else { - numElements = dis.readInt(); - dataBlock = datasetAttributes.getDataType().createDataBlock(null, gridPosition, numElements); - } - - final BlockReader reader = datasetAttributes.getCompression().getReader(); - reader.read(dataBlock, in); - return dataBlock; - } - - /** - * Reads a {@link DataBlock} from an {@link InputStream}. - * - * @param in - * the input stream - * @param datasetAttributes - * the dataset attributes - * @param gridPosition - * the grid position - * @return the block - * @throws IOException - * the exception - */ - public static DataBlock readBlockWithCodecs( - final InputStream in, - final DatasetAttributes datasetAttributes, - final long[] gridPosition) throws IOException { - - final DataInputStream dis = new DataInputStream(in); - final short mode = dis.readShort(); - final int numElements; - final DataBlock dataBlock; - if (mode != 2) { - final int nDim = dis.readShort(); - final int[] blockSize = new int[nDim]; - for (int d = 0; d < nDim; ++d) - blockSize[d] = dis.readInt(); - if (mode == 0) { - numElements = DataBlock.getNumElements(blockSize); - } else { - numElements = dis.readInt(); + if (codec instanceof Codec.BytesToBytes) { + stream = ((Codec.BytesToBytes)codec).decode(stream); } - dataBlock = datasetAttributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); - } else { - numElements = dis.readInt(); - dataBlock = datasetAttributes.getDataType().createDataBlock(null, gridPosition, numElements); } - readFromStream(dataBlock, datasetAttributes.collectCodecs().decode(in)); + final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); + readFromStream(dataBlock, stream); return dataBlock; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index c5d985f92..6b0e988d8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -25,7 +25,7 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataOutputStream; +import org.janelia.saalfeldlab.n5.codec.Codec; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; @@ -70,75 +70,22 @@ public static void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - final DataOutputStream dos = new DataOutputStream(out); - final int mode; - if (datasetAttributes.getDataType() == DataType.OBJECT || dataBlock.getSize() == null) - mode = 2; - else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSize())) - mode = 0; - else - mode = 1; - dos.writeShort(mode); - - if (mode != 2) { - dos.writeShort(datasetAttributes.getNumDimensions()); - for (final int size : dataBlock.getSize()) - dos.writeInt(size); - } - - if (mode != 0) - dos.writeInt(dataBlock.getNumElements()); - - dos.flush(); - - final BlockWriter writer = datasetAttributes.getCompression().getWriter(); - writer.write(dataBlock, out); - } - - /** - * Writes a {@link DataBlock} into an {@link OutputStream}. - * - * @param - * the type of data - * @param out - * the output stream - * @param datasetAttributes - * the dataset attributes - * @param dataBlock - * the data block the block data type - * @throws IOException - * the exception - */ - public static void writeBlockWithCodecs( - final OutputStream out, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws IOException { - - final DataOutputStream dos = new DataOutputStream(out); - - final int mode; - if (datasetAttributes.getDataType() == DataType.OBJECT || dataBlock.getSize() == null) - mode = 2; - else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSize())) - mode = 0; - else - mode = 1; - dos.writeShort(mode); - - if (mode != 2) { - dos.writeShort(datasetAttributes.getNumDimensions()); - for (final int size : dataBlock.getSize()) - dos.writeInt(size); + OutputStream stream = out; + final Codec[] codecs = datasetAttributes.getCodecs(); + for (Codec codec : codecs) { + if (codec instanceof Codec.BytesToBytes) + stream = ((Codec.BytesToBytes)codec).encode(stream); + else if (codec instanceof Codec.ArrayToBytes) + stream = ((Codec.ArrayToBytes)codec).encode(datasetAttributes, dataBlock, stream); } - if (mode != 0) - dos.writeInt(dataBlock.getNumElements()); + writeFromStream(dataBlock, stream); + stream.flush(); - try (final OutputStream encodedStream = datasetAttributes.collectCodecs().encode(out)) { - writeFromStream(dataBlock, encodedStream); - out.flush(); - } + //FIXME Caleb: The stream must be closed BUT it shouldn't be `writeBlock`'s responsibility. + // Whoever opens the stream should close it + stream.close(); } public static void writeFromStream(final DataBlock dataBlock, final OutputStream out) throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 003820b1c..e6a3429f1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -113,7 +113,7 @@ default DataBlock readBlock( final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), gridPosition); try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(path)) { - return DefaultBlockReader.readBlockWithCodecs(lockedChannel.newInputStream(), datasetAttributes, gridPosition); + return DefaultBlockReader.readBlock(lockedChannel.newInputStream(), datasetAttributes, gridPosition); } catch (final N5Exception.N5NoSuchKeyException e) { return null; } catch (final IOException | UncheckedIOException e) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 69e77fc21..2545853ee 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -230,8 +230,7 @@ default void writeBlock( final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition()); try (final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath)) { - DefaultBlockWriter.writeBlockWithCodecs(lock.newOutputStream(), datasetAttributes, dataBlock); - // DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); + DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); } catch (final IOException | UncheckedIOException e) { throw new N5IOException( "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index c611a9e7d..0ba185e30 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -28,6 +28,8 @@ import java.lang.reflect.Type; import java.util.Map; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonParseException; import org.janelia.saalfeldlab.n5.codec.Codec; import com.google.gson.Gson; @@ -60,31 +62,15 @@ default DatasetAttributes getDatasetAttributes(final String pathName) throws N5E default DatasetAttributes createDatasetAttributes(final JsonElement attributes) { - try { - final long[] dimensions = GsonUtils.readAttribute(attributes, DatasetAttributes.DIMENSIONS_KEY, long[].class, getGson()); - if (dimensions == null) { - return null; - } - - final DataType dataType = GsonUtils.readAttribute(attributes, DatasetAttributes.DATA_TYPE_KEY, DataType.class, getGson()); - if (dataType == null) { - return null; - } + final JsonDeserializationContext context = new JsonDeserializationContext() { - final int[] blockSize = GsonUtils.readAttribute(attributes, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, getGson()); - final Compression compression = GsonUtils.readAttribute(attributes, DatasetAttributes.COMPRESSION_KEY, Compression.class, getGson()); - final Codec[] codecs = GsonUtils.readAttribute(attributes, DatasetAttributes.CODEC_KEY, Codec[].class, getGson()); + @Override public T deserialize(JsonElement json, Type typeOfT) throws JsonParseException { - /* version 0 */ - final String compressionVersion0Name = compression == null - ? GsonUtils.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, getGson()) - : null; + return getGson().fromJson(json, typeOfT); + } + }; - return DatasetAttributes.from(dimensions, dataType, blockSize, compression, compressionVersion0Name, codecs); - } catch (JsonSyntaxException | NumberFormatException | ClassCastException e) { - /* We cannot create a dataset, so return null. */ - return null; - } + return DatasetAttributes.getJsonAdapter().deserialize(attributes, DatasetAttributes.class, context); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index 0ba88e126..654ca4b56 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -33,8 +33,10 @@ import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("lz4") +@NameConfig.Name("lz4") public class Lz4Compression implements DefaultBlockReader, DefaultBlockWriter, Compression { private static final long serialVersionUID = -9071316415067427256L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 2dbf6b176..68aa4b351 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -25,6 +25,9 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; public class ShortArrayDataBlock extends AbstractDataBlock { @@ -48,6 +51,15 @@ public void readData(final ByteBuffer buffer) { buffer.asShortBuffer().get(data); } + + public void readData(final InputStream stream) throws IOException { + + final DataInputStream dis = new DataInputStream(stream); + for (int i = 0; i < data.length; i++) { + data[i] = dis.readShort(); + } + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index ed73b7d53..d2c1ce3dc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -32,8 +32,10 @@ import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("xz") +@NameConfig.Name("xz") public class XzCompression implements DefaultBlockReader, DefaultBlockWriter, Compression { private static final long serialVersionUID = -7272153943564743774L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java index 3a437ffa2..c3f8e69bf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java @@ -10,8 +10,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @NameConfig.Name(AsTypeCodec.TYPE) -@NameConfig.Prefix("codec") -public class AsTypeCodec implements Codec { +public class AsTypeCodec implements Codec.BytesToBytes { private static final long serialVersionUID = 1031322606191894484L; @@ -141,15 +140,15 @@ else if (to == DataType.FLOAT64) else if (to == DataType.INT16) return AsTypeCodec::INT_TO_SHORT; if (to == DataType.INT8) - return AsTypeCodec::DOUBLE_TO_BYTE; + return AsTypeCodec::INT_TO_BYTE; else if (to == DataType.INT16) - return AsTypeCodec::DOUBLE_TO_SHORT; + return AsTypeCodec::INT_TO_SHORT; else if (to == DataType.INT32) - return AsTypeCodec::DOUBLE_TO_INT; + return AsTypeCodec::IDENTITY; else if (to == DataType.INT64) - return AsTypeCodec::DOUBLE_TO_LONG; + return AsTypeCodec::INT_TO_LONG; else if (to == DataType.FLOAT32) - return AsTypeCodec::DOUBLE_TO_FLOAT; + return AsTypeCodec::INT_TO_FLOAT; else if (to == DataType.INT64) return AsTypeCodec::INT_TO_LONG; else if (to == DataType.FLOAT32) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java index a16d0ec3b..3fa2c844a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java @@ -1,8 +1,11 @@ package org.janelia.saalfeldlab.n5.codec; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.nio.ByteOrder; import com.google.gson.JsonDeserializationContext; @@ -12,10 +15,14 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import org.apache.commons.io.output.ProxyOutputStream; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @NameConfig.Name(value = BytesCodec.TYPE) -public class BytesCodec implements Codec { +public class BytesCodec implements Codec.ArrayToBytes { private static final long serialVersionUID = 3523505403978222360L; @@ -35,18 +42,93 @@ public BytesCodec(final ByteOrder byteOrder) { } - @Override - public InputStream decode(InputStream in) throws IOException { - - // TODO not applicable for array -> bytes - return in; + @Override public DataBlockInputStream decode(final DatasetAttributes attributes, final long[] gridPosition, InputStream in) throws IOException { + + return new DataBlockInputStream(in) { + + private short mode = -1; + private int[] blockSize = null; + private int numElements = -1; + + private boolean start = true; + + @Override protected void beforeRead(int n) throws IOException { + + if (start) { + readHeader(); + start = false; + } + } + + @Override + public DataBlock allocateDataBlock() throws IOException { + if (start) { + readHeader(); + start = false; + } + if (mode != 2) { + return attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); + } else { + return attributes.getDataType().createDataBlock(null, gridPosition, numElements); + } + } + + private void readHeader() throws IOException { + final DataInputStream dis = new DataInputStream(in); + mode = dis.readShort(); + if (mode != 2) { + final int nDim = dis.readShort(); + blockSize = new int[nDim]; + for (int d = 0; d < nDim; ++d) + blockSize[d] = dis.readInt(); + if (mode == 0) { + numElements = DataBlock.getNumElements(blockSize); + } else { + numElements = dis.readInt(); + } + + } else { + numElements = dis.readInt(); + } + } + }; } - @Override - public OutputStream encode(OutputStream out) throws IOException { - - // TODO not applicable for array -> bytes - return out; + @Override public OutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, final OutputStream out) throws IOException { + + return new ProxyOutputStream(out) { + + boolean start = true; + + @Override protected void beforeWrite(int n) throws IOException { + if (start) { + writeHeader(); + start = false; + } + } + + private void writeHeader() throws IOException { + final DataOutputStream dos = new DataOutputStream(out); + + final int mode; + if (attributes.getDataType() == DataType.OBJECT || dataBlock.getSize() == null) + mode = 2; + else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSize())) + mode = 0; + else + mode = 1; + dos.writeShort(mode); + + if (mode != 2) { + dos.writeShort(attributes.getNumDimensions()); + for (final int size : dataBlock.getSize()) + dos.writeInt(size); + } + + if (mode != 0) + dos.writeInt(dataBlock.getNumElements()); + } + }; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index 2c76438ca..fdd3a0376 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -1,7 +1,11 @@ package org.janelia.saalfeldlab.n5.codec; +import org.apache.commons.io.input.ProxyInputStream; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.serialization.NameConfig; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -17,24 +21,58 @@ @NameConfig.Prefix("codec") public interface Codec extends Serializable { - /** - * Decode an {@link InputStream}. - * - * @param in - * input stream - * @return the decoded input stream - */ - public InputStream decode(InputStream in) throws IOException; - - /** - * Encode an {@link OutputStream}. - * - * @param out - * the output stream - * @return the encoded output stream - */ - public OutputStream encode(OutputStream out) throws IOException; + public interface BytesToBytes extends Codec { - public String getType(); + /** + * Decode an {@link InputStream}. + * + * @param in + * input stream + * @return the decoded input stream + */ + public InputStream decode(final InputStream in) throws IOException; + + /** + * Encode an {@link OutputStream}. + * + * @param out + * the output stream + * @return the encoded output stream + */ + public OutputStream encode(final OutputStream out) throws IOException; + } + + interface ArrayToBytes extends Codec { + + /** + * Decode an {@link InputStream}. + * + * @param in + * input stream + * @return the DataBlock corresponding to the input stream + */ + public DataBlockInputStream decode(final DatasetAttributes attributes, final long[] gridPosition, final InputStream in) throws IOException; + + /** + * Encode a {@link DataBlock}. + * + * @param datablock the datablock to encode + */ + public OutputStream encode(final DatasetAttributes attributes, final DataBlock datablock, final OutputStream out) throws IOException; + + } + + public abstract class DataBlockInputStream extends ProxyInputStream { + + protected DataBlockInputStream(InputStream in) { + + super(in); + } + + public abstract DataBlock allocateDataBlock() throws IOException; + } + + public String getType(); } + diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java index 3b07ad2b6..de720bbc5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java @@ -7,17 +7,17 @@ /** * A {@link Codec} that is composition of a collection of codecs. */ -public class ComposedCodec implements Codec { +public class ComposedCodec implements Codec.BytesToBytes { //TODO Caleb: Remove? private static final long serialVersionUID = 5068349140842235924L; protected static final String TYPE = "composed"; - private final Codec[] filters; + private final Codec[] codecs; - public ComposedCodec(final Codec... filters) { + public ComposedCodec(final Codec... codec) { - this.filters = filters; + this.codecs = codec; } @Override @@ -31,8 +31,8 @@ public InputStream decode(InputStream in) throws IOException { // note that decoding is in reverse order InputStream decoded = in; - for (int i = filters.length - 1; i >= 0; i--) - decoded = filters[i].decode(decoded); + for (int i = codecs.length - 1; i >= 0; i--){} +// decoded = codecs[i].decode(decoded); return decoded; } @@ -41,8 +41,8 @@ public InputStream decode(InputStream in) throws IOException { public OutputStream encode(OutputStream out) throws IOException { OutputStream encoded = out; - for (int i = 0; i < filters.length; i++) - encoded = filters[i].encode(encoded); + for (int i = 0; i < codecs.length; i++){} +// encoded = codecs[i].encode(encoded); return encoded; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java index 9ac0a1fe0..b0288adfa 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java @@ -4,7 +4,7 @@ * A {@link Codec} that can deterministically determine the size of encoded data from the size of the raw data and vice versa from the data length alone (i.e. encoding is data * independent). */ -public interface DeterministicSizeCodec extends Codec { +public interface DeterministicSizeCodec extends Codec.BytesToBytes { public abstract long encodedSize(long size); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java index 41fd10f12..b308d31b7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -8,7 +8,7 @@ @NameConfig.Name(IdentityCodec.TYPE) @NameConfig.Prefix("codec") -public class IdentityCodec implements Codec { +public class IdentityCodec implements Codec.BytesToBytes { private static final long serialVersionUID = 8354269325800855621L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java index 84555dd38..5cf83cf05 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java @@ -37,16 +37,6 @@ public int numChecksumBytes() { return numChecksumBytes; } - @Override - public CheckedInputStream decode(final InputStream in) throws IOException { - - // TODO get the correct expected checksum - // TODO write a test with nested checksum codecs - - // has to know the number of it needs to read? - return new CheckedInputStream(in, getChecksum()); - } - @Override public CheckedOutputStream encode(final OutputStream out) throws IOException { @@ -61,6 +51,16 @@ public void encode(final OutputStream out, ByteBuffer buffer) throws IOException writeChecksum(out); } + @Override + public CheckedInputStream decode(final InputStream in) throws IOException { + + // TODO get the correct expected checksum + // TODO write a test with nested checksum codecs + + // has to know the number of it needs to read? + return new CheckedInputStream(in, getChecksum()); + } + public ByteBuffer decodeAndValidate(final InputStream in, int numBytes) throws IOException, ChecksumException { final CheckedInputStream cin = decode(in); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 7297aab5d..dc5392d4a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -16,7 +16,7 @@ import java.lang.reflect.Type; @NameConfig.Name(ShardingCodec.TYPE) -public class ShardingCodec implements Codec { +public class ShardingCodec implements Codec.BytesToBytes { //TODO Caleb: should be ArrayToBytes private static final long serialVersionUID = -5879797314954717810L; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 6e2e9e8f2..076ae2461 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -25,13 +25,21 @@ */ package org.janelia.saalfeldlab.n5; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.reflect.TypeToken; +import org.janelia.saalfeldlab.n5.N5Exception.N5ClassCastException; +import org.janelia.saalfeldlab.n5.N5Reader.Version; +import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; +import org.janelia.saalfeldlab.n5.codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; import java.io.IOException; import java.net.URISyntaxException; @@ -48,19 +56,13 @@ import java.util.concurrent.Executors; import java.util.function.Predicate; -import org.janelia.saalfeldlab.n5.N5Exception.N5ClassCastException; -import org.janelia.saalfeldlab.n5.N5Reader.Version; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.reflect.TypeToken; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; /** * Abstract base class for testing N5 functionality. @@ -117,6 +119,7 @@ protected final N5Writer createTempN5Writer(String location, GsonBuilder gson) { @After public void removeTempWriters() { + synchronized (tempWriters) { for (final N5Writer writer : tempWriters) { try { @@ -208,7 +211,7 @@ public void testSetAttributeDoesntCreateGroup() { } @Test - public void testCreateDataset() { + public void testCreateDataset() { final DatasetAttributes info; try (N5Writer writer = createTempN5Writer()) { @@ -246,6 +249,33 @@ public void testWriteReadByteBlock() { } } + @Test + public void testWriteReadByteBlockMultipleCompressors() { + + try (final N5Writer n5 = createTempN5Writer()) { + final Codec[] codecs = { + new BytesCodec(), + new AsTypeCodec(DataType.INT32, DataType.INT8), + new AsTypeCodec(DataType.INT64, DataType.INT32), + }; + final long[] longBlock1 = new long[]{1,2,3,4,5,6,7,8}; + final long[] dimensions1 = new long[]{2,2,2}; + final int[] blockSize1 = new int[]{2,2,2}; + n5.createDataset(datasetName, dimensions1, blockSize1, DataType.INT8, new RawCompression(), codecs); +// n5.createDataset(datasetName, dimensions, blockSize, DataType.INT64, new RawCompression(), codecs); + final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); +// final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize, new long[]{0, 0, 0}, longBlock); + final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize1, new long[]{0, 0, 0}, longBlock1); + n5.writeBlock(datasetName, attributes, dataBlock); + + final DatasetAttributes fakeAttributes = new DatasetAttributes(dimensions1, blockSize1, DataType.INT64, new RawCompression(), codecs); + final DataBlock loadedDataBlock = n5.readBlock(datasetName, fakeAttributes, 0, 0, 0); + assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); + assertTrue(n5.remove(datasetName)); + + } + } + @Test public void testWriteReadStringBlock() { @@ -465,7 +495,7 @@ public void testOverwriteBlock() { } @Test - public void testAttributeParsingPrimitive() { + public void testAttributeParsingPrimitive() { try (final N5Writer n5 = createTempN5Writer()) { @@ -541,7 +571,7 @@ public void testAttributeParsingPrimitive() { } @Test - public void testAttributes() { + public void testAttributes() { try (final N5Writer n5 = createTempN5Writer()) { assertNull(n5.getAttribute(groupName, "test", String.class)); @@ -607,7 +637,6 @@ public void testAttributes() { } } - @Test public void testNullAttributes() throws URISyntaxException, IOException { @@ -831,7 +860,7 @@ public void testUri() throws IOException, URISyntaxException { } @Test - public void testRemoveGroup() { + public void testRemoveGroup() { try (final N5Writer n5 = createTempN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression()); @@ -982,7 +1011,7 @@ public void testDeepList() throws ExecutionException, InterruptedException { } @Test - public void testExists() { + public void testExists() { final String groupName2 = groupName + "-2"; final String datasetName2 = datasetName + "-2"; @@ -1003,7 +1032,7 @@ public void testExists() { } @Test - public void testListAttributes() { + public void testListAttributes() { try (N5Writer n5 = createTempN5Writer()) { final String groupName2 = groupName + "-2"; @@ -1106,7 +1135,7 @@ public void testReaderCreation() throws IOException, URISyntaxException { writer.setAttribute("/", N5Reader.VERSION_KEY, invalidVersion); assertThrows("Incompatible version throws error", N5Exception.class, () -> { try (final N5Reader ignored = createN5Reader(location)) { - /*Only try with resource to ensure `close()` is called.*/ + /*Only try with resource to ensure `close()` is called.*/ } }); } finally { @@ -1123,7 +1152,7 @@ public void testReaderCreation() throws IOException, URISyntaxException { } @Test - public void testDelete() { + public void testDelete() { try (N5Writer n5 = createTempN5Writer()) { final String datasetName = AbstractN5Test.datasetName + "-test-delete"; @@ -1209,7 +1238,7 @@ protected static void runTests(final N5Writer writer, final ArrayList Date: Fri, 23 Aug 2024 10:02:50 -0400 Subject: [PATCH 038/423] fix: isDataset caching --- .../org/janelia/saalfeldlab/n5/DatasetAttributes.java | 3 +-- .../java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index ff420fea5..3eb2b8d5d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -16,8 +16,6 @@ import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; -import javax.xml.crypto.Data; - /** * Mandatory dataset attributes: * @@ -187,6 +185,7 @@ public static class DatasetAttributesAdapter implements JsonSerializer loadedDataBlock = n5.readBlock(datasetName, fakeAttributes, 0, 0, 0); - assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); + assertArrayEquals(longBlock1, (long[])loadedDataBlock.getData()); assertTrue(n5.remove(datasetName)); } From 9155ec5e4faa710568424c98d774771c387ffa64 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 26 Aug 2024 10:52:31 -0400 Subject: [PATCH 039/423] test: fix FixedScaleOffsetTests --- .../org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java | 4 ++-- .../janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java | 4 ++++ .../janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index 60244334a..b74002f81 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -23,7 +23,7 @@ public ShardedDatasetAttributes( final Compression compression, final Codec[] codecs) { - super(dimensions, shardSize, dataType, compression, codecs); + super(dimensions, blockSize, dataType, compression, codecs); this.shardSize = shardSize; this.indexLocation = shardIndexLocation; @@ -105,7 +105,7 @@ public long getNumBlocks() { public static int[] getBlockSize(Codec[] codecs) { - for (Codec codec : codecs) + for (final Codec codec : codecs) if (codec instanceof ShardingCodec) return ((ShardingCodec)codec).getBlockSize(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java index 55613b573..e6c831635 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java @@ -98,12 +98,16 @@ public String getType() { @Override public InputStream decode(InputStream in) throws IOException { + numBytes = bytes(dataType); + numEncodedBytes = bytes(encodedType); return new FixedLengthConvertedInputStream(numEncodedBytes, numBytes, this.decoder, in); } @Override public OutputStream encode(OutputStream out) throws IOException { + numBytes = bytes(dataType); + numEncodedBytes = bytes(encodedType); return new FixedLengthConvertedOutputStream(numBytes, numEncodedBytes, this.encoder, out); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java index 6de676901..135a7f2ba 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java @@ -35,7 +35,7 @@ public void testDouble2Byte() throws IOException { public void testLong2Short() throws IOException { final int N = 16; - final ByteBuffer encodedLongs = ByteBuffer.allocate(Double.BYTES * N); + final ByteBuffer encodedLongs = ByteBuffer.allocate(Long.BYTES * N); final ByteBuffer encodedShorts = ByteBuffer.allocate(Short.BYTES * N); final long scale = 2; From e8cfefd382a007a5ce7865cb4106e2c7f227fc00 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 26 Aug 2024 11:18:46 -0400 Subject: [PATCH 040/423] fix: LockedFileChannel locking --- .../org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index bb2cf2db9..03dfde816 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -75,7 +75,7 @@ protected class LockedFileChannel implements LockedChannel { protected LockedFileChannel(final String path, final boolean readOnly) throws IOException { - this(fileSystem.getPath(path), readOnly, 0, 0); + this(fileSystem.getPath(path), readOnly, 0, Long.MAX_VALUE); } protected LockedFileChannel(final String path, final boolean readOnly, final long startByte, final long size) throws IOException { @@ -85,7 +85,7 @@ protected LockedFileChannel(final String path, final boolean readOnly, final lon protected LockedFileChannel(final Path path, final boolean readOnly) throws IOException { - this(path, readOnly, 0, 0); + this(path, readOnly, 0, Long.MAX_VALUE); } protected LockedFileChannel(final Path path, final boolean readOnly, final long startByte, final long size) From 17ef0fef07ad0e25992ba5b088c179e52a2efeb0 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 26 Aug 2024 11:45:19 -0400 Subject: [PATCH 041/423] fix: LockedFileChannel truncation logic --- .../org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 03dfde816..54aacf809 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -93,10 +93,10 @@ protected LockedFileChannel(final Path path, final boolean readOnly, final long final long start = startByte < 0 ? 0L : startByte; - final long len = size < 0 ? 0L : size; + final long len = size < 0 ? Long.MAX_VALUE : size; //TODO Caleb: How does this handle if manually overwriting the entire file? (e.g. len > file size) - truncate = (start == 0 && len == 0); + truncate = (start == 0 && len == Long.MAX_VALUE); final OpenOption[] options; if (readOnly) { From 2e18288dbf647e93e39b110fefaf9c685a0b5a16 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 26 Aug 2024 14:56:26 -0400 Subject: [PATCH 042/423] fix: NameConfigAdapter avoid NPE * configuration may be null when all parameters are optional --- .../java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java index 5081f8215..2cb72463b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java @@ -188,7 +188,7 @@ public T deserialize( /* It's ok to be null if all parameters are optional. * Otherwise, return*/ if (configuration == null) { - for (Field field : parameters.get(type).values()) { + for (final Field field : parameters.get(type).values()) { if (!field.getAnnotation(NameConfig.Parameter.class).optional()) return null; } @@ -204,7 +204,7 @@ public T deserialize( for (final Entry parameterType : parameterTypes.entrySet()) { final String fieldName = parameterType.getKey(); final String paramName = parameterNameMap.get(fieldName); - final JsonElement paramJson = configuration.get(paramName); + final JsonElement paramJson = configuration == null ? null : configuration.get(paramName); final Field field = parameterType.getValue(); if (paramJson != null) { final Object parameter; From 71802905336096f9db27cfe3556993479a5986fd Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 26 Aug 2024 14:57:35 -0400 Subject: [PATCH 043/423] pref: normalGetDatasetAttributes should call createDatasetAttributes * so zarr only needs to override createDatasetAttributes --- .../org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index d2e8418ed..324d242e8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -27,7 +27,6 @@ import java.lang.reflect.Type; -import com.google.gson.reflect.TypeToken; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; import org.janelia.saalfeldlab.n5.cache.N5JsonCacheableContainer; @@ -83,7 +82,7 @@ default DatasetAttributes normalGetDatasetAttributes(final String pathName) thro final String normalPath = N5URI.normalizeGroupPath(pathName); final JsonElement attributes = GsonKeyValueN5Reader.super.getAttributes(normalPath); - return getGson().fromJson(attributes, DatasetAttributes.class); + return createDatasetAttributes(attributes); } @Override From 764d05ad63305447dc0f757a46f4175400af03c5 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 3 Sep 2024 16:48:17 -0400 Subject: [PATCH 044/423] feat: DataBlock methods to read/write directly from DataInput/Output --- .../saalfeldlab/n5/AbstractDataBlock.java | 21 +++++++++++++++++++ .../saalfeldlab/n5/ByteArrayDataBlock.java | 8 +++++++ .../org/janelia/saalfeldlab/n5/DataBlock.java | 7 +++++++ .../saalfeldlab/n5/DoubleArrayDataBlock.java | 9 ++++++++ .../saalfeldlab/n5/FloatArrayDataBlock.java | 9 ++++++++ .../saalfeldlab/n5/IntArrayDataBlock.java | 19 ++++++++++++++++- .../saalfeldlab/n5/LongArrayDataBlock.java | 17 +++++++++++++++ .../saalfeldlab/n5/ShortArrayDataBlock.java | 13 +++++------- 8 files changed, 94 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index f1cbc3529..59208fcfe 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -25,6 +25,11 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.ByteBuffer; + /** * Abstract base class for {@link DataBlock} implementations. * @@ -63,4 +68,20 @@ public T getData() { return data; } + + @Override + public void readData(final DataInput input) throws IOException { + + final ByteBuffer buffer = toByteBuffer(); + input.readFully(buffer.array()); + readData(buffer); + } + + @Override + public void writeData(final DataOutput output) throws IOException { + + final ByteBuffer buffer = toByteBuffer(); + output.write(buffer.array()); + } + } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index 5717ad2ef..4610811d7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -25,6 +25,8 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.IOException; import java.nio.ByteBuffer; public class ByteArrayDataBlock extends AbstractDataBlock { @@ -47,6 +49,12 @@ public void readData(final ByteBuffer buffer) { buffer.get(getData()); } + @Override + public void readData(final DataInput inputStream) throws IOException { + + inputStream.readFully(data); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 3d9dc92a1..5ccdbbaf9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -25,6 +25,9 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.nio.ByteBuffer; /** @@ -94,6 +97,10 @@ public interface DataBlock { */ public void readData(final ByteBuffer buffer); + public void readData(final DataInput inputStream) throws IOException; + + public void writeData(final DataOutput output) throws IOException; + /** * Returns the number of elements in this {@link DataBlock}. This number is * not necessarily equal {@link #getNumElements(int[]) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 426c7944d..8cbb15117 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -25,6 +25,8 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.IOException; import java.nio.ByteBuffer; public class DoubleArrayDataBlock extends AbstractDataBlock { @@ -48,6 +50,13 @@ public void readData(final ByteBuffer buffer) { buffer.asDoubleBuffer().get(data); } + @Override + public void readData(final DataInput inputStream) throws IOException { + + for (int i = 0; i < data.length; i++) + data[i] = inputStream.readDouble(); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index b8d309998..aa97ce3f5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -25,6 +25,8 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.IOException; import java.nio.ByteBuffer; public class FloatArrayDataBlock extends AbstractDataBlock { @@ -48,6 +50,13 @@ public void readData(final ByteBuffer buffer) { buffer.asFloatBuffer().get(data); } + @Override + public void readData(final DataInput inputStream) throws IOException { + + for (int i = 0; i < data.length; i++) + data[i] = inputStream.readFloat(); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 98c5577d4..4d3383328 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -25,10 +25,13 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.nio.ByteBuffer; -public class IntArrayDataBlock extends AbstractDataBlock { +public class IntArrayDataBlock extends AbstractDataBlock { public IntArrayDataBlock(final int[] size, final long[] gridPosition, final int[] data) { super(size, gridPosition, data); @@ -48,6 +51,20 @@ public void readData(final ByteBuffer buffer) { buffer.asIntBuffer().get(data); } + @Override + public void readData(final DataInput input) throws IOException { + + for (int i = 0; i < data.length; i++) + data[i] = input.readInt(); + } + + @Override + public void writeData(final DataOutput output) throws IOException { + + for (int i = 0; i < data.length; i++) + output.writeInt(data[i]); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index d3f3fc9c5..be435c4f9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -25,6 +25,9 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.nio.ByteBuffer; public class LongArrayDataBlock extends AbstractDataBlock { @@ -48,6 +51,20 @@ public void readData(final ByteBuffer buffer) { buffer.asLongBuffer().get(data); } + @Override + public void readData(final DataInput inputStream) throws IOException { + + for (int i = 0; i < data.length; i++) + data[i] = inputStream.readLong(); + } + + @Override + public void writeData(final DataOutput output) throws IOException { + + for (int i = 0; i < data.length; i++) + output.writeLong(data[i]); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 68aa4b351..34c5a883d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -25,9 +25,8 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInputStream; +import java.io.DataInput; import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; public class ShortArrayDataBlock extends AbstractDataBlock { @@ -51,13 +50,11 @@ public void readData(final ByteBuffer buffer) { buffer.asShortBuffer().get(data); } + @Override + public void readData(final DataInput dataInput) throws IOException { - public void readData(final InputStream stream) throws IOException { - - final DataInputStream dis = new DataInputStream(stream); - for (int i = 0; i < data.length; i++) { - data[i] = dis.readShort(); - } + for (int i = 0; i < data.length; i++) + data[i] = dataInput.readShort(); } @Override From dbd7b158930734cb85af2ecf5bbad40eddd60b62 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 3 Sep 2024 20:13:59 -0400 Subject: [PATCH 045/423] refactor: create N5BytesCodec, BytesCodec is simple zarr approach --- .../saalfeldlab/n5/DatasetAttributes.java | 49 ++++-- .../saalfeldlab/n5/codec/BytesCodec.java | 114 +++++-------- .../saalfeldlab/n5/codec/N5BytesCodec.java | 153 ++++++++++++++++++ .../saalfeldlab/n5/AbstractN5Test.java | 2 +- .../saalfeldlab/n5/shard/ShardDemos.java | 6 +- 5 files changed, 231 insertions(+), 93 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 3eb2b8d5d..d12feee16 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -5,6 +5,10 @@ import java.util.Arrays; import java.util.HashMap; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.N5BytesCodec; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; + import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; @@ -12,9 +16,6 @@ import com.google.gson.JsonParseException; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec; /** * Mandatory dataset attributes: @@ -55,7 +56,8 @@ public class DatasetAttributes implements Serializable { private final int[] blockSize; private final DataType dataType; private final Compression compression; - private final Codec[] codecs; + private final Codec.ArrayToBytes arrayToBytesCodec; + private final Codec[] byteByteCodecs; public DatasetAttributes( final long[] dimensions, @@ -69,11 +71,19 @@ public DatasetAttributes( this.dataType = dataType; this.compression = compression; if (codecs == null && !(compression instanceof RawCompression)) { - this.codecs = new Codec[]{new BytesCodec(), compression}; + byteByteCodecs = new Codec[]{compression}; + arrayToBytesCodec = new N5BytesCodec(); } else if (codecs == null) { - this.codecs = new Codec[]{new BytesCodec()}; + byteByteCodecs = new Codec[]{}; + arrayToBytesCodec = new N5BytesCodec(); } else { - this.codecs = codecs; + if (!(codecs[0] instanceof Codec.ArrayToBytes)) + throw new N5Exception("Expected first element of codecs to be ArrayToBytes, but was: " + codecs[0]); + + arrayToBytesCodec = (Codec.ArrayToBytes)codecs[0]; + byteByteCodecs = new Codec[codecs.length - 1]; + for (int i = 0; i < byteByteCodecs.length; i++) + byteByteCodecs[i] = codecs[i + 1]; } } @@ -111,9 +121,14 @@ public DataType getDataType() { return dataType; } + public Codec.ArrayToBytes getArrayToBytesCodec() { + + return arrayToBytesCodec; + } + public Codec[] getCodecs() { - return codecs; + return byteByteCodecs; } public HashMap asMap() { @@ -123,7 +138,7 @@ public HashMap asMap() { map.put(BLOCK_SIZE_KEY, blockSize); map.put(DATA_TYPE_KEY, dataType); map.put(COMPRESSION_KEY, compression); - map.put(CODEC_KEY, codecs); // TODO : consider not adding to map when null + map.put(CODEC_KEY, concatenateCodecs()); // TODO : consider not adding to map when null return map; } @@ -173,6 +188,16 @@ private static Compression getCompressionVersion0(final String compressionVersio return null; } + private Codec[] concatenateCodecs() { + + final Codec[] allCodecs = new Codec[byteByteCodecs.length + 1]; + allCodecs[0] = arrayToBytesCodec; + for (int i = 0; i < byteByteCodecs.length; i++) + allCodecs[i + 1] = byteByteCodecs[i]; + + return allCodecs; + } + private static DatasetAttributesAdapter adapter = null; public static DatasetAttributesAdapter getJsonAdapter() { if (adapter == null) { @@ -208,9 +233,9 @@ public static class DatasetAttributesAdapter implements JsonSerializer allocateDataBlock() throws IOException { - if (start) { - readHeader(); - start = false; - } - if (mode != 2) { - return attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); - } else { - return attributes.getDataType().createDataBlock(null, gridPosition, numElements); - } + final int[] blockSize = attributes.getBlockSize(); + final int numElements = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y); + return attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); } - private void readHeader() throws IOException { - final DataInputStream dis = new DataInputStream(in); - mode = dis.readShort(); - if (mode != 2) { - final int nDim = dis.readShort(); - blockSize = new int[nDim]; - for (int d = 0; d < nDim; ++d) - blockSize[d] = dis.readInt(); - if (mode == 0) { - numElements = DataBlock.getNumElements(blockSize); - } else { - numElements = dis.readInt(); - } - - } else { - numElements = dis.readInt(); - } + @Override + public DataInput getDataInput() { + + if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) + return new DataInputStream(super.in); + else + return new LittleEndianDataInputStream(super.in); } }; - } - @Override public OutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, final OutputStream out) throws IOException { + } - return new ProxyOutputStream(out) { + @Override + public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, + final OutputStream out) + throws IOException { - boolean start = true; + return new DataBlockOutputStream(out) { - @Override protected void beforeWrite(int n) throws IOException { - if (start) { - writeHeader(); - start = false; - } - } + @Override + public void beforeWrite(OutputStream rawOut) throws IOException {} - private void writeHeader() throws IOException { - final DataOutputStream dos = new DataOutputStream(out); + @Override + public DataOutput getDataOutput() { - final int mode; - if (attributes.getDataType() == DataType.OBJECT || dataBlock.getSize() == null) - mode = 2; - else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSize())) - mode = 0; + if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) + return new DataOutputStream(out); else - mode = 1; - dos.writeShort(mode); - - if (mode != 2) { - dos.writeShort(attributes.getNumDimensions()); - for (final int size : dataBlock.getSize()) - dos.writeInt(size); - } - - if (mode != 0) - dos.writeInt(dataBlock.getNumElements()); + return new LittleEndianDataOutputStream(out); } }; } @@ -164,4 +123,5 @@ public ByteOrder deserialize(JsonElement json, java.lang.reflect.Type typeOfT, } } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java new file mode 100644 index 000000000..fb57a4c6f --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java @@ -0,0 +1,153 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteOrder; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; + +@NameConfig.Name(value = N5BytesCodec.TYPE) +public class N5BytesCodec implements Codec.ArrayToBytes { + + private static final long serialVersionUID = 3523505403978222360L; + + public static final String TYPE = "n5bytes"; + + @NameConfig.Parameter(value = "endian", optional = true) + protected final ByteOrder byteOrder; + + public N5BytesCodec() { + + this(ByteOrder.BIG_ENDIAN); + } + + public N5BytesCodec(final ByteOrder byteOrder) { + + this.byteOrder = byteOrder; + } + + @Override public DataBlockInputStream decode(final DatasetAttributes attributes, final long[] gridPosition, InputStream in) throws IOException { + + return new DataBlockInputStream(in) { + + private short mode = -1; + private int[] blockSize = null; + private int numElements = -1; + + private boolean start = true; + + @Override protected void beforeRead(int n) throws IOException { + + if (start) { + readHeader(); + start = false; + } + } + + @Override + public DataBlock allocateDataBlock() throws IOException { + if (start) { + readHeader(); + start = false; + } + if (mode != 2) { + return attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); + } else { + return attributes.getDataType().createDataBlock(null, gridPosition, numElements); + } + } + + private void readHeader() throws IOException { + final DataInputStream dis = new DataInputStream(in); + mode = dis.readShort(); + if (mode != 2) { + final int nDim = dis.readShort(); + blockSize = new int[nDim]; + for (int d = 0; d < nDim; ++d) + blockSize[d] = dis.readInt(); + if (mode == 0) { + numElements = DataBlock.getNumElements(blockSize); + } else { + numElements = dis.readInt(); + } + + } else { + numElements = dis.readInt(); + } + } + + @Override + public DataInput getDataInput() { + + if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) + return new DataInputStream(super.in); + else + return new LittleEndianDataInputStream(super.in); + } + }; + } + + @Override + public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, + final OutputStream out) + throws IOException { + + return new DataBlockOutputStream(out) { + + @Override + public void beforeWrite(OutputStream rawOut) throws IOException { + + writeHeader(rawOut); + } + + private void writeHeader(OutputStream out) throws IOException { + final DataOutputStream dos = new DataOutputStream(out); + + final int mode; + if (attributes.getDataType() == DataType.OBJECT || dataBlock.getSize() == null) + mode = 2; + else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSize())) + mode = 0; + else + mode = 1; + dos.writeShort(mode); + + if (mode != 2) { + dos.writeShort(attributes.getNumDimensions()); + for (final int size : dataBlock.getSize()) + dos.writeInt(size); + } + + if (mode != 0) + dos.writeInt(dataBlock.getNumElements()); + } + + @Override + public DataOutput getDataOutput() { + + if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) + return new DataOutputStream(out); + else + return new LittleEndianDataOutputStream(out); + } + }; + } + + @Override + public String getType() { + + return TYPE; + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 0d0d01238..4372ec2f6 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -256,7 +256,7 @@ public void testWriteReadByteBlockMultipleCodecs() { * maybe is not the behavior we actually want*/ try (final N5Writer n5 = createTempN5Writer()) { final Codec[] codecs = { - new BytesCodec(), + new N5BytesCodec(), new AsTypeCodec(DataType.INT32, DataType.INT8), new AsTypeCodec(DataType.INT64, DataType.INT32), }; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java index 3479efe5f..50981dc69 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -8,7 +8,7 @@ import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.N5BytesCodec; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.IdentityCodec; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; @@ -72,10 +72,10 @@ public void writeReadBlockTest() { DataType.UINT8, new RawCompression(), new Codec[]{ - new BytesCodec(), + new N5BytesCodec(), new ShardingCodec( new int[]{2, 2}, - new Codec[]{new BytesCodec(), new GzipCompression(4)}, + new Codec[]{new N5BytesCodec(), new GzipCompression(4)}, new Codec[]{new Crc32cChecksumCodec()}, IndexLocation.END ) From 786e4d5ea38f7a739f3dc827bfb8e22d6c5639c9 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 4 Sep 2024 09:17:14 -0400 Subject: [PATCH 046/423] chore(pom): depend on guava --- pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pom.xml b/pom.xml index be2913baf..9ea763f0d 100644 --- a/pom.xml +++ b/pom.xml @@ -169,6 +169,10 @@ org.apache.commons commons-compress + + com.google.guava + guava + From 2a7327a306aaa4906ba4eba7496b124dfe6502b6 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 4 Sep 2024 10:38:48 -0400 Subject: [PATCH 047/423] wip: undo ArrayToBytes codec changes --- .../saalfeldlab/n5/codec/BytesCodec.java | 43 ++++++------------- .../saalfeldlab/n5/codec/N5BytesCodec.java | 42 ++++++------------ 2 files changed, 26 insertions(+), 59 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java index 4d19ec868..42304a9ee 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java @@ -1,21 +1,16 @@ package org.janelia.saalfeldlab.n5.codec; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteOrder; import java.util.Arrays; +import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import com.google.common.io.LittleEndianDataInputStream; -import com.google.common.io.LittleEndianDataOutputStream; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; @@ -50,43 +45,31 @@ public DataBlockInputStream decode(final DatasetAttributes attributes, final lon return new DataBlockInputStream(in) { + private int[] blockSize = attributes.getBlockSize(); + private int numElements = Arrays.stream(blockSize).reduce(1, (x, y) -> { + return x * y; + }); + @Override - public DataBlock allocateDataBlock() throws IOException { - final int[] blockSize = attributes.getBlockSize(); - final int numElements = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y); - return attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); - } + protected void beforeRead(int n) throws IOException {} @Override - public DataInput getDataInput() { + public DataBlock allocateDataBlock() throws IOException { - if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) - return new DataInputStream(super.in); - else - return new LittleEndianDataInputStream(super.in); + return attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); } - }; + }; } @Override - public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, - final OutputStream out) + public OutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, final OutputStream out) throws IOException { - return new DataBlockOutputStream(out) { - - @Override - public void beforeWrite(OutputStream rawOut) throws IOException {} + return new ProxyOutputStream(out) { @Override - public DataOutput getDataOutput() { - - if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) - return new DataOutputStream(out); - else - return new LittleEndianDataOutputStream(out); - } + protected void beforeWrite(int n) throws IOException {} }; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java index fb57a4c6f..813d3523e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java @@ -1,22 +1,18 @@ package org.janelia.saalfeldlab.n5.codec; -import java.io.DataInput; import java.io.DataInputStream; -import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteOrder; +import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import com.google.common.io.LittleEndianDataInputStream; -import com.google.common.io.LittleEndianDataOutputStream; - @NameConfig.Name(value = N5BytesCodec.TYPE) public class N5BytesCodec implements Codec.ArrayToBytes { @@ -37,6 +33,7 @@ public N5BytesCodec(final ByteOrder byteOrder) { this.byteOrder = byteOrder; } + @Override public DataBlockInputStream decode(final DatasetAttributes attributes, final long[] gridPosition, InputStream in) throws IOException { return new DataBlockInputStream(in) { @@ -86,32 +83,28 @@ private void readHeader() throws IOException { numElements = dis.readInt(); } } - - @Override - public DataInput getDataInput() { - - if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) - return new DataInputStream(super.in); - else - return new LittleEndianDataInputStream(super.in); - } }; } + @Override - public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, - final OutputStream out) + public OutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, final OutputStream out) throws IOException { - return new DataBlockOutputStream(out) { + return new ProxyOutputStream(out) { + + boolean start = true; @Override - public void beforeWrite(OutputStream rawOut) throws IOException { + protected void beforeWrite(int n) throws IOException { - writeHeader(rawOut); + if (start) { + writeHeader(); + start = false; + } } - private void writeHeader(OutputStream out) throws IOException { + private void writeHeader() throws IOException { final DataOutputStream dos = new DataOutputStream(out); final int mode; @@ -132,15 +125,6 @@ else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSiz if (mode != 0) dos.writeInt(dataBlock.getNumElements()); } - - @Override - public DataOutput getDataOutput() { - - if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) - return new DataOutputStream(out); - else - return new LittleEndianDataOutputStream(out); - } }; } From c24605d807cfafde2b8a3ca99c238cf70848184a Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 4 Sep 2024 10:47:11 -0400 Subject: [PATCH 048/423] fix(wip): DefaultBlockReader/Writer can get ArrayToBytesCodec directly --- .../janelia/saalfeldlab/n5/DefaultBlockReader.java | 11 ++++------- .../janelia/saalfeldlab/n5/DefaultBlockWriter.java | 12 +++++------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index bb3f2639a..6890cbc24 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -73,13 +73,10 @@ public static DataBlock readBlock( Codec.DataBlockInputStream dataBlockStream = null; InputStream stream = in; - for (Codec codec : datasetAttributes.getCodecs()) { - if (codec instanceof Codec.ArrayToBytes) { - stream = dataBlockStream = ((Codec.ArrayToBytes)codec).decode(datasetAttributes, gridPosition, stream); - } - if (codec instanceof Codec.BytesToBytes) { - stream = ((Codec.BytesToBytes)codec).decode(stream); - } + stream = dataBlockStream = datasetAttributes.getArrayToBytesCodec().decode(datasetAttributes, gridPosition, + stream); + for (final Codec codec : datasetAttributes.getCodecs()) { + stream = ((Codec.BytesToBytes)codec).decode(stream); } final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index 6b0e988d8..df584f834 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -25,11 +25,12 @@ */ package org.janelia.saalfeldlab.n5; -import org.janelia.saalfeldlab.n5.codec.Codec; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import org.janelia.saalfeldlab.n5.codec.Codec; + /** * Default implementation of {@link BlockWriter}. * @@ -70,14 +71,11 @@ public static void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - OutputStream stream = out; final Codec[] codecs = datasetAttributes.getCodecs(); - for (Codec codec : codecs) { - if (codec instanceof Codec.BytesToBytes) - stream = ((Codec.BytesToBytes)codec).encode(stream); - else if (codec instanceof Codec.ArrayToBytes) - stream = ((Codec.ArrayToBytes)codec).encode(datasetAttributes, dataBlock, stream); + stream = datasetAttributes.getArrayToBytesCodec().encode(datasetAttributes, dataBlock, stream); + for (final Codec codec : codecs) { + stream = ((Codec.BytesToBytes)codec).encode(stream); } writeFromStream(dataBlock, stream); From 5f8fa624e29df82867b774644e583932cbe1bf5a Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 4 Sep 2024 11:48:35 -0400 Subject: [PATCH 049/423] test/fix: BytesTests now works with refactor --- .../janelia/saalfeldlab/n5/codec/BytesTests.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java index e8bf64b04..fa407163e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java @@ -29,7 +29,7 @@ public void testSerialization() { final N5Writer reader = factory.openWriter("n5:src/test/resources/shardExamples/test.zarr"); final Codec bytes = reader.getAttribute("mid_sharded", "codecs[0]/configuration/codecs[0]", Codec.class); - assertTrue("as BytesCodec", bytes instanceof BytesCodec); + assertTrue("as BytesCodec", bytes instanceof N5BytesCodec); final N5Writer writer = factory.openWriter("n5:src/test/resources/shardExamples/test.n5"); @@ -39,16 +39,17 @@ public void testSerialization() { DataType.UINT8, new RawCompression(), new Codec[]{ - new IdentityCodec(), - new BytesCodec(ByteOrder.LITTLE_ENDIAN) + new N5BytesCodec(ByteOrder.LITTLE_ENDIAN), + new IdentityCodec() } ); writer.setAttribute("shard", "/", datasetAttributes); final DatasetAttributes deserialized = writer.getAttribute("shard", "/", DatasetAttributes.class); - assertEquals("2 codecs", 2, deserialized.getCodecs().length); + assertEquals("1 codecs", 1, deserialized.getCodecs().length); assertTrue("Identity", deserialized.getCodecs()[0] instanceof IdentityCodec); - assertTrue("Bytes", deserialized.getCodecs()[1] instanceof BytesCodec); - assertEquals("LittleEndian",ByteOrder.LITTLE_ENDIAN, ((BytesCodec)deserialized.getCodecs()[1]).byteOrder); + assertTrue("Bytes", deserialized.getArrayToBytesCodec() instanceof N5BytesCodec); + assertEquals("LittleEndian", ByteOrder.LITTLE_ENDIAN, + ((N5BytesCodec)deserialized.getArrayToBytesCodec()).byteOrder); } } From 784a4a2ca6228d8293d970fffb56da2642345971 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 4 Sep 2024 13:37:18 -0400 Subject: [PATCH 050/423] wip: toward supporting endianness --- .../saalfeldlab/n5/DefaultBlockReader.java | 2 +- .../saalfeldlab/n5/DefaultBlockWriter.java | 8 +++-- .../saalfeldlab/n5/codec/BytesCodec.java | 30 +++++++++++++++++-- .../janelia/saalfeldlab/n5/codec/Codec.java | 29 +++++++++++++----- .../saalfeldlab/n5/codec/N5BytesCodec.java | 29 ++++++++++++++++-- 5 files changed, 82 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 6890cbc24..5df1fc019 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -80,7 +80,7 @@ public static DataBlock readBlock( } final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); - readFromStream(dataBlock, stream); + dataBlock.readData(dataBlockStream.getDataInput(stream)); return dataBlock; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index df584f834..4c9a18e95 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -30,6 +30,8 @@ import java.nio.ByteBuffer; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.Codec.ArrayToBytes; +import org.janelia.saalfeldlab.n5.codec.Codec.DataBlockOutputStream; /** * Default implementation of {@link BlockWriter}. @@ -73,12 +75,14 @@ public static void writeBlock( OutputStream stream = out; final Codec[] codecs = datasetAttributes.getCodecs(); - stream = datasetAttributes.getArrayToBytesCodec().encode(datasetAttributes, dataBlock, stream); + final ArrayToBytes arrayToBytes = datasetAttributes.getArrayToBytesCodec(); + final DataBlockOutputStream dataBlockOutput; + stream = dataBlockOutput = arrayToBytes.encode(datasetAttributes, dataBlock, stream); for (final Codec codec : codecs) { stream = ((Codec.BytesToBytes)codec).encode(stream); } - writeFromStream(dataBlock, stream); + dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); stream.flush(); //FIXME Caleb: The stream must be closed BUT it shouldn't be `writeBlock`'s responsibility. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java index 42304a9ee..8cb8e116c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java @@ -1,16 +1,21 @@ package org.janelia.saalfeldlab.n5.codec; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteOrder; import java.util.Arrays; -import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.serialization.NameConfig; +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; @@ -59,17 +64,36 @@ public DataBlock allocateDataBlock() throws IOException { return attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); } + @Override + public DataInput getDataInput(final InputStream inputStream) { + + if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) + return new DataInputStream(inputStream); + else + return new LittleEndianDataInputStream(inputStream); + } + }; } @Override - public OutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, final OutputStream out) + public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, + final OutputStream out) throws IOException { - return new ProxyOutputStream(out) { + return new DataBlockOutputStream(out) { @Override protected void beforeWrite(int n) throws IOException {} + + @Override + public DataOutput getDataOutput(OutputStream outputStream) { + + if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) + return new DataOutputStream(outputStream); + else + return new LittleEndianDataOutputStream(outputStream); + } }; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index fdd3a0376..a8fdd9996 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -1,16 +1,18 @@ package org.janelia.saalfeldlab.n5.codec; -import org.apache.commons.io.input.ProxyInputStream; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.serialization.NameConfig; - -import java.io.FilterInputStream; +import java.io.DataInput; +import java.io.DataOutput; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Serializable; +import org.apache.commons.io.input.ProxyInputStream; +import org.apache.commons.io.output.ProxyOutputStream; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + /** * Interface representing a filter can encode a {@link OutputStream}s when writing data, and decode * the {@link InputStream}s when reading data. @@ -58,7 +60,8 @@ interface ArrayToBytes extends Codec { * * @param datablock the datablock to encode */ - public OutputStream encode(final DatasetAttributes attributes, final DataBlock datablock, final OutputStream out) throws IOException; + public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock datablock, + final OutputStream out) throws IOException; } @@ -71,6 +74,18 @@ protected DataBlockInputStream(InputStream in) { } public abstract DataBlock allocateDataBlock() throws IOException; + + public abstract DataInput getDataInput(final InputStream inputStream); + } + + public abstract class DataBlockOutputStream extends ProxyOutputStream { + + protected DataBlockOutputStream(final OutputStream out) { + + super(out); + } + + public abstract DataOutput getDataOutput(final OutputStream outputStream); } public String getType(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java index 813d3523e..bdc712256 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java @@ -1,18 +1,22 @@ package org.janelia.saalfeldlab.n5.codec; +import java.io.DataInput; import java.io.DataInputStream; +import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteOrder; -import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.serialization.NameConfig; +import com.google.common.io.LittleEndianDataInputStream; +import com.google.common.io.LittleEndianDataOutputStream; + @NameConfig.Name(value = N5BytesCodec.TYPE) public class N5BytesCodec implements Codec.ArrayToBytes { @@ -83,15 +87,25 @@ private void readHeader() throws IOException { numElements = dis.readInt(); } } + + @Override + public DataInput getDataInput(final InputStream inputStream) { + + if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) + return new DataInputStream(inputStream); + else + return new LittleEndianDataInputStream(inputStream); + } }; } @Override - public OutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, final OutputStream out) + public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, + final OutputStream out) throws IOException { - return new ProxyOutputStream(out) { + return new DataBlockOutputStream(out) { boolean start = true; @@ -125,6 +139,15 @@ else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSiz if (mode != 0) dos.writeInt(dataBlock.getNumElements()); } + + @Override + public DataOutput getDataOutput(final OutputStream outputStream) { + + if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) + return new DataOutputStream(outputStream); + else + return new LittleEndianDataOutputStream(outputStream); + } }; } From 87456a42e11072cdd62d65d6c7ede21e7fb452f2 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 4 Sep 2024 13:37:37 -0400 Subject: [PATCH 051/423] wip: DatasetAttributes allow empty codecs --- src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index d12feee16..f853b20cb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -73,7 +73,7 @@ public DatasetAttributes( if (codecs == null && !(compression instanceof RawCompression)) { byteByteCodecs = new Codec[]{compression}; arrayToBytesCodec = new N5BytesCodec(); - } else if (codecs == null) { + } else if (codecs == null || codecs.length == 0) { byteByteCodecs = new Codec[]{}; arrayToBytesCodec = new N5BytesCodec(); } else { From 7ef4297a1d32e7082b12ad497e15c1f1530b5b67 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 4 Sep 2024 14:40:46 -0400 Subject: [PATCH 052/423] style: import order --- .../saalfeldlab/n5/AbstractN5Test.java | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 4372ec2f6..939b99254 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -25,21 +25,13 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.reflect.TypeToken; -import org.janelia.saalfeldlab.n5.N5Exception.N5ClassCastException; -import org.janelia.saalfeldlab.n5.N5Reader.Version; -import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.net.URISyntaxException; @@ -56,13 +48,22 @@ import java.util.concurrent.Executors; import java.util.function.Predicate; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import org.janelia.saalfeldlab.n5.N5Exception.N5ClassCastException; +import org.janelia.saalfeldlab.n5.N5Reader.Version; +import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.N5BytesCodec; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.reflect.TypeToken; /** * Abstract base class for testing N5 functionality. From b80fef09617c76347bd8d28dedabe57c847b3b11 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 4 Sep 2024 15:08:50 -0400 Subject: [PATCH 053/423] fix: BytesTest N5BytesCodec has name "n5bytes" --- .../shardExamples/test.zarr/mid_sharded/attributes.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/shardExamples/test.zarr/mid_sharded/attributes.json b/src/test/resources/shardExamples/test.zarr/mid_sharded/attributes.json index a80cb9d96..920dff928 100644 --- a/src/test/resources/shardExamples/test.zarr/mid_sharded/attributes.json +++ b/src/test/resources/shardExamples/test.zarr/mid_sharded/attributes.json @@ -32,7 +32,7 @@ ], "codecs": [ { - "name": "bytes", + "name": "n5bytes", "configuration": { "endian": "little" } From ae1f347c9d0713aadebea000912af2792a90b8aa Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 16 Sep 2024 15:05:24 -0400 Subject: [PATCH 054/423] feat: wip shard/codec support Basic working examples with file based shards writing new shards, and reading existing shards (with index at the end) --- .../janelia/saalfeldlab/n5/Compression.java | 2 +- .../saalfeldlab/n5/DatasetAttributes.java | 86 +++++++++------- .../saalfeldlab/n5/DefaultBlockReader.java | 19 ++-- .../saalfeldlab/n5/DefaultBlockWriter.java | 23 ++--- .../n5/FileSystemKeyValueAccess.java | 12 ++- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 7 +- .../saalfeldlab/n5/GzipCompression.java | 4 +- .../saalfeldlab/n5/KeyValueAccess.java | 2 +- .../saalfeldlab/n5/NameConfigAdapter.java | 4 +- .../n5/ShardedDatasetAttributes.java | 97 +++++++++++++++---- .../saalfeldlab/n5/codec/AsTypeCodec.java | 2 +- .../saalfeldlab/n5/codec/BytesCodec.java | 2 +- .../janelia/saalfeldlab/n5/codec/Codec.java | 13 ++- .../saalfeldlab/n5/codec/ComposedCodec.java | 50 ---------- .../n5/codec/DeterministicSizeCodec.java | 2 +- .../saalfeldlab/n5/codec/IdentityCodec.java | 3 +- .../{N5BytesCodec.java => N5BlockCodec.java} | 8 +- .../n5/codec/checksum/ChecksumCodec.java | 24 +++-- .../janelia/saalfeldlab/n5/shard/Shard.java | 20 +--- .../saalfeldlab/n5/shard/ShardIndex.java | 66 +++++++++---- .../saalfeldlab/n5/shard/ShardReader.java | 24 +++-- .../saalfeldlab/n5/shard/ShardWriter.java | 41 ++++---- .../saalfeldlab/n5/shard/ShardingCodec.java | 44 +++++---- .../saalfeldlab/n5/shard/VirtualShard.java | 71 +++++++------- .../saalfeldlab/n5/AbstractN5Test.java | 4 +- .../saalfeldlab/n5/codec/AsTypeTests.java | 6 +- .../saalfeldlab/n5/codec/BytesTests.java | 8 +- .../n5/serialization/CodecSerialization.java | 21 ++-- .../saalfeldlab/n5/shard/ShardDemos.java | 65 ++++++++----- 29 files changed, 409 insertions(+), 321 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java rename src/main/java/org/janelia/saalfeldlab/n5/codec/{N5BytesCodec.java => N5BlockCodec.java} (95%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index eac6a0ac8..ac0c49b55 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -43,7 +43,7 @@ * * @author Stephan Saalfeld */ -public interface Compression extends Serializable, Codec.BytesToBytes { +public interface Compression extends Serializable, Codec.BytesCodec { /** * Annotation for runtime discovery of compression schemes. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index f853b20cb..1ada17086 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -6,7 +6,9 @@ import java.util.HashMap; import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.N5BytesCodec; +import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; +import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import com.google.gson.JsonDeserializationContext; @@ -56,8 +58,8 @@ public class DatasetAttributes implements Serializable { private final int[] blockSize; private final DataType dataType; private final Compression compression; - private final Codec.ArrayToBytes arrayToBytesCodec; - private final Codec[] byteByteCodecs; + private final ArrayCodec arrayCodec; + private final BytesCodec[] byteCodecs; public DatasetAttributes( final long[] dimensions, @@ -69,22 +71,37 @@ public DatasetAttributes( this.dimensions = dimensions; this.blockSize = blockSize; this.dataType = dataType; - this.compression = compression; if (codecs == null && !(compression instanceof RawCompression)) { - byteByteCodecs = new Codec[]{compression}; - arrayToBytesCodec = new N5BytesCodec(); + byteCodecs = new BytesCodec[]{compression}; + arrayCodec = new N5BlockCodec(); } else if (codecs == null || codecs.length == 0) { - byteByteCodecs = new Codec[]{}; - arrayToBytesCodec = new N5BytesCodec(); + byteCodecs = new BytesCodec[]{}; + arrayCodec = new N5BlockCodec(); } else { - if (!(codecs[0] instanceof Codec.ArrayToBytes)) - throw new N5Exception("Expected first element of codecs to be ArrayToBytes, but was: " + codecs[0]); + if (!(codecs[0] instanceof ArrayCodec)) + throw new N5Exception("Expected first element of codecs to be ArrayCodec, but was: " + codecs[0]); - arrayToBytesCodec = (Codec.ArrayToBytes)codecs[0]; - byteByteCodecs = new Codec[codecs.length - 1]; - for (int i = 0; i < byteByteCodecs.length; i++) - byteByteCodecs[i] = codecs[i + 1]; + arrayCodec = (ArrayCodec)codecs[0]; + byteCodecs = new BytesCodec[codecs.length - 1]; + for (int i = 0; i < byteCodecs.length; i++) + byteCodecs[i] = (BytesCodec)codecs[i + 1]; } + + //TODO Caleb: Do we want to do this? + this.compression = Arrays.stream(byteCodecs) + .filter(codec -> codec instanceof Compression) + .map(codec -> (Compression)codec) + .findFirst() + .orElse(compression == null ? new RawCompression() : compression); + + } + + public DatasetAttributes( + final long[] dimensions, + final int[] blockSize, + final DataType dataType, + final Codec[] codecs) { + this(dimensions, blockSize, dataType, null, codecs); } public DatasetAttributes( @@ -121,14 +138,14 @@ public DataType getDataType() { return dataType; } - public Codec.ArrayToBytes getArrayToBytesCodec() { + public ArrayCodec getArrayCodec() { - return arrayToBytesCodec; + return arrayCodec; } - public Codec[] getCodecs() { + public BytesCodec[] getCodecs() { - return byteByteCodecs; + return byteCodecs; } public HashMap asMap() { @@ -190,10 +207,10 @@ private static Compression getCompressionVersion0(final String compressionVersio private Codec[] concatenateCodecs() { - final Codec[] allCodecs = new Codec[byteByteCodecs.length + 1]; - allCodecs[0] = arrayToBytesCodec; - for (int i = 0; i < byteByteCodecs.length; i++) - allCodecs[i + 1] = byteByteCodecs[i]; + final Codec[] allCodecs = new Codec[byteCodecs.length + 1]; + allCodecs[0] = arrayCodec; + for (int i = 0; i < byteCodecs.length; i++) + allCodecs[i + 1] = byteCodecs[i]; return allCodecs; } @@ -233,19 +250,18 @@ public static class DatasetAttributesAdapter implements JsonSerializer blocksPerShard[i] * blockSize[i]); + return new ShardedDatasetAttributes( + dimensions, + shardSize, + blockSize, + dataType, + shardingCodec + ); } return new DatasetAttributes(dimensions, blockSize, dataType, compression, codecs); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 5df1fc019..f881aea96 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -30,7 +30,9 @@ import java.io.InputStream; import java.nio.ByteBuffer; -import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; +import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.Codec.DataBlockInputStream; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; /** @@ -71,16 +73,19 @@ public static DataBlock readBlock( final DatasetAttributes datasetAttributes, final long[] gridPosition) throws IOException { - Codec.DataBlockInputStream dataBlockStream = null; - InputStream stream = in; - stream = dataBlockStream = datasetAttributes.getArrayToBytesCodec().decode(datasetAttributes, gridPosition, - stream); - for (final Codec codec : datasetAttributes.getCodecs()) { - stream = ((Codec.BytesToBytes)codec).decode(stream); + final BytesCodec[] codecs = datasetAttributes.getCodecs(); + final ArrayCodec arrayCodec = datasetAttributes.getArrayCodec(); + final DataBlockInputStream dataBlockStream = arrayCodec.decode(datasetAttributes, gridPosition, in); + + InputStream stream = dataBlockStream; + for (final BytesCodec codec : codecs) { + stream = codec.decode(stream); } final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); dataBlock.readData(dataBlockStream.getDataInput(stream)); + stream.close(); + return dataBlock; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index 4c9a18e95..fd7450ef2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -29,8 +29,8 @@ import java.io.OutputStream; import java.nio.ByteBuffer; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.Codec.ArrayToBytes; +import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; +import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.Codec.DataBlockOutputStream; /** @@ -73,20 +73,15 @@ public static void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - OutputStream stream = out; - final Codec[] codecs = datasetAttributes.getCodecs(); - final ArrayToBytes arrayToBytes = datasetAttributes.getArrayToBytesCodec(); - final DataBlockOutputStream dataBlockOutput; - stream = dataBlockOutput = arrayToBytes.encode(datasetAttributes, dataBlock, stream); - for (final Codec codec : codecs) { - stream = ((Codec.BytesToBytes)codec).encode(stream); - } + final BytesCodec[] codecs = datasetAttributes.getCodecs(); + final ArrayCodec arrayCodec = datasetAttributes.getArrayCodec(); + final DataBlockOutputStream dataBlockOutput = arrayCodec.encode(datasetAttributes, dataBlock, out); - dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); - stream.flush(); + OutputStream stream = dataBlockOutput; + for (final BytesCodec codec : codecs) + stream = codec.encode(stream); - //FIXME Caleb: The stream must be closed BUT it shouldn't be `writeBlock`'s responsibility. - // Whoever opens the stream should close it + dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); stream.close(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 54aacf809..ffed4c1e7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -25,6 +25,8 @@ */ package org.janelia.saalfeldlab.n5; +import org.apache.commons.io.input.BoundedInputStream; + import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -73,6 +75,8 @@ protected class LockedFileChannel implements LockedChannel { protected final boolean truncate; + protected long len; + protected LockedFileChannel(final String path, final boolean readOnly) throws IOException { this(fileSystem.getPath(path), readOnly, 0, Long.MAX_VALUE); @@ -93,7 +97,7 @@ protected LockedFileChannel(final Path path, final boolean readOnly, final long final long start = startByte < 0 ? 0L : startByte; - final long len = size < 0 ? Long.MAX_VALUE : size; + len = size < 0 ? Long.MAX_VALUE : size; //TODO Caleb: How does this handle if manually overwriting the entire file? (e.g. len > file size) truncate = (start == 0 && len == Long.MAX_VALUE); @@ -159,7 +163,7 @@ public Writer newWriter() throws IOException { @Override public InputStream newInputStream() throws IOException { - return Channels.newInputStream(channel); + return new BoundedInputStream(Channels.newInputStream(channel), len); } @Override @@ -201,11 +205,11 @@ public LockedFileChannel lockForReading(final String normalPath) throws IOExcept } @Override - public LockedFileChannel lockForReading(final String normalPath, final long startByte, final long endByte) + public LockedFileChannel lockForReading(final String normalPath, final long startByte, final long size) throws IOException { try { - return new LockedFileChannel(normalPath, true, startByte, endByte); + return new LockedFileChannel(normalPath, true, startByte, size); } catch (final NoSuchFileException e) { throw new N5Exception.N5NoSuchKeyException("No such file", e); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 2545853ee..60ce92997 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; +import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.Arrays; import java.util.List; @@ -222,7 +223,7 @@ default void writeBlock( if (datasetAttributes instanceof ShardedDatasetAttributes) { ShardedDatasetAttributes shardDatasetAttrs = (ShardedDatasetAttributes)datasetAttributes; final long[] shardPos = shardDatasetAttrs.getShardPositionForBlock(dataBlock.getGridPosition()); - final String shardPath = absoluteShardPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition()); + final String shardPath = absoluteShardPath(N5URI.normalizeGroupPath(path), shardPos); final VirtualShard shard = new VirtualShard<>(shardDatasetAttrs, shardPos, getKeyValueAccess(), shardPath); shard.writeBlock(dataBlock); return; @@ -230,7 +231,9 @@ default void writeBlock( final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition()); try (final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath)) { - DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); + try ( final OutputStream out = lock.newOutputStream()) { + DefaultBlockWriter.writeBlock(out, datasetAttributes, dataBlock); + } } catch (final IOException | UncheckedIOException e) { throw new N5IOException( "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index ecaf2f0eb..3091ad284 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -48,11 +48,13 @@ public class GzipCompression implements DefaultBlockReader, DefaultBlockWriter, @CompressionParameter @NameConfig.Parameter //TODO Caleb: How to handle serialization of parameter-less constructor. - // For N5, default is -1, for zarr, range is 0-9 and is required. + // For N5 the default is -1. + // For zarr the range is 0-9 and is required. // How to map -1 to some default (1?) when serializing to zarr? private final int level; @CompressionParameter + @NameConfig.Parameter private final boolean useZlib; private final transient GzipParameters parameters = new GzipParameters(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index a4fa42b82..138bb73c7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -157,7 +157,7 @@ public default String compose(final URI uri, final String... components) { */ public LockedChannel lockForReading(final String normalPath) throws IOException; - public LockedChannel lockForReading(String normalPath, final long startByte, final long endByte) + public LockedChannel lockForReading(String normalPath, final long startByte, final long size) throws IOException; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java index 2cb72463b..30e45d80d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java @@ -58,7 +58,7 @@ public class NameConfigAdapter implements JsonDeserializer, JsonSerializer private static void registerAdapter(Class cls) { - adapters.put(cls, new NameConfigAdapter(cls)); + adapters.put(cls, new NameConfigAdapter<>(cls)); update(adapters.get(cls)); } private final HashMap> constructors = new HashMap<>(); @@ -77,6 +77,7 @@ private static ArrayList getDeclaredFields(Class clazz) { @SuppressWarnings("unchecked") public static synchronized void update(final NameConfigAdapter adapter) { + final String prefix = adapter.type.getAnnotation(NameConfig.Prefix.class).value(); final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); final Index annotationIndex = Index.load(NameConfig.Name.class, classLoader); for (final IndexItem item : annotationIndex) { @@ -84,7 +85,6 @@ public static synchronized void update(final NameConfigAdapter adapter) { try { clazz = (Class)Class.forName(item.className()); final String name = clazz.getAnnotation(NameConfig.Name.class).value(); - final String prefix = adapter.type.getAnnotation(NameConfig.Prefix.class).value(); final String type = prefix + "." + name; final Constructor constructor = clazz.getDeclaredConstructor(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index b74002f81..c72bad299 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -3,6 +3,10 @@ import java.util.Arrays; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; +import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.shard.ShardIndex; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; @@ -12,22 +16,51 @@ public class ShardedDatasetAttributes extends DatasetAttributes { private final int[] shardSize; - private final IndexLocation indexLocation; + private final ShardingCodec shardingCodec; - public ShardedDatasetAttributes( + + public ShardedDatasetAttributes ( final long[] dimensions, - final int[] shardSize, - final int[] blockSize, - final IndexLocation shardIndexLocation, + final int[] shardSize, //in pixels + final int[] blockSize, //in pixels final DataType dataType, - final Compression compression, - final Codec[] codecs) { + final Codec[] blocksCodecs, + final DeterministicSizeCodec[] indexCodecs, + final IndexLocation indexLocation + ) { + super(dimensions, blockSize, dataType, null, blocksCodecs); + this.shardSize = shardSize; + this.shardingCodec = new ShardingCodec( + blockSize, + blocksCodecs, + indexCodecs, + indexLocation + ); + } - super(dimensions, blockSize, dataType, compression, codecs); + public ShardedDatasetAttributes( + final long[] dimensions, + final int[] shardSize, //in pixels + final int[] blockSize, //in pixels + final DataType dataType, + final ShardingCodec codec) { + super(dimensions, blockSize, dataType, null, codec.getCodecs()); this.shardSize = shardSize; - this.indexLocation = shardIndexLocation; + this.shardingCodec = codec; + } - // TODO figure out codecs + public ShardingCodec getShardingCodec() { + return shardingCodec; + } + + @Override public ArrayCodec getArrayCodec() { + + return shardingCodec.getArrayCodec(); + } + + @Override public BytesCodec[] getCodecs() { + + return shardingCodec.getCodecs(); } public int[] getShardSize() { @@ -36,9 +69,9 @@ public int[] getShardSize() { } /** - * Returns the number of blocks a shard contains along all dimensions. + * Returns the number of shards per dimension for the dataset. * - * @return the size of the block grid of a shard + * @return the size of the shard grid of a dataset */ public int[] getShardBlockGridSize() { @@ -51,6 +84,22 @@ public int[] getShardBlockGridSize() { return shardBlockGridSize; } + /** + * Returns the number of blocks per dimension for a shard. + * + * @return the size of the block grid of a shard + */ + public int[] getBlocksPerShard() { + + final int nd = getNumDimensions(); + final int[] blocksPerShard = new int[nd]; + final int[] blockSize = getBlockSize(); + for (int i = 0; i < nd; i++) + blocksPerShard[i] = getShardSize()[i] / blockSize[i]; + + return blocksPerShard; + } + /** * Given a block's position relative to the array, returns the position of the shard containing that block relative to the shard grid. * @@ -61,10 +110,10 @@ public int[] getShardBlockGridSize() { public long[] getShardPositionForBlock(final long... blockGridPosition) { // TODO have this return a shard - final int[] shardBlockDimensions = getShardBlockGridSize(); + final int[] blocksPerShard = getBlocksPerShard(); final long[] shardGridPosition = new long[blockGridPosition.length]; for (int i = 0; i < shardGridPosition.length; i++) { - shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / shardBlockDimensions[i]); + shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / blocksPerShard[i]); } return shardGridPosition; @@ -75,7 +124,7 @@ public long[] getShardPositionForBlock(final long... blockGridPosition) { * * @return the shard position */ - public int[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { + public long[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { final long[] shardPos = getShardPositionForBlock(blockPosition); if (!Arrays.equals(shardPosition, shardPos)) @@ -85,7 +134,7 @@ public int[] getBlockPositionInShard(final long[] shardPosition, final long[] bl final int[] blkSize = getBlockSize(); final int[] blkGridSize = getShardBlockGridSize(); - final int[] blockShardPos = new int[shardSize.length]; + final long[] blockShardPos = new long[shardSize.length]; for (int i = 0; i < shardSize.length; i++) { final long shardP = shardPos[i] * shardSize[i]; final long blockP = blockPosition[i] * blkSize[i]; @@ -100,7 +149,7 @@ public int[] getBlockPositionInShard(final long[] shardPosition, final long[] bl */ public long getNumBlocks() { - return Arrays.stream(getShardBlockGridSize()).reduce(1, (x, y) -> x * y); + return Arrays.stream(getBlocksPerShard()).reduce(1, (x, y) -> x * y); } public static int[] getBlockSize(Codec[] codecs) { @@ -114,6 +163,18 @@ public static int[] getBlockSize(Codec[] codecs) { public IndexLocation getIndexLocation() { - return indexLocation; + return getShardingCodec().getIndexLocation(); + } + + public ShardIndex createIndex() { + return new ShardIndex(getBlocksPerShard(), getShardingCodec().getIndexCodecs()); + } + + public DatasetAttributes getIndexAttributes() { + return createShardIndexAttributes(getShardingCodec().getIndexCodecs()); + } + + private static DatasetAttributes createShardIndexAttributes(Codec[] indexCodecs) { + return new DatasetAttributes(null, null, null, null, indexCodecs); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java index c3f8e69bf..e8883c752 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java @@ -10,7 +10,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @NameConfig.Name(AsTypeCodec.TYPE) -public class AsTypeCodec implements Codec.BytesToBytes { +public class AsTypeCodec implements Codec.BytesCodec { private static final long serialVersionUID = 1031322606191894484L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java index 8cb8e116c..76b015cf1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java @@ -25,7 +25,7 @@ import com.google.gson.JsonSerializer; @NameConfig.Name(value = BytesCodec.TYPE) -public class BytesCodec implements Codec.ArrayToBytes { +public class BytesCodec implements Codec.ArrayCodec { private static final long serialVersionUID = 3282569607795127005L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index a8fdd9996..a78df016a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -23,7 +23,7 @@ @NameConfig.Prefix("codec") public interface Codec extends Serializable { - public interface BytesToBytes extends Codec { + public interface BytesCodec extends Codec { /** * Decode an {@link InputStream}. @@ -44,7 +44,7 @@ public interface BytesToBytes extends Codec { public OutputStream encode(final OutputStream out) throws IOException; } - interface ArrayToBytes extends Codec { + interface ArrayCodec extends DeterministicSizeCodec { /** * Decode an {@link InputStream}. @@ -63,6 +63,15 @@ interface ArrayToBytes extends Codec { public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock datablock, final OutputStream out) throws IOException; + @Override default long encodedSize(long size) { + + return size; + } + + @Override default long decodedSize(long size) { + + return size; + } } public abstract class DataBlockInputStream extends ProxyInputStream { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java deleted file mode 100644 index de720bbc5..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/ComposedCodec.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -/** - * A {@link Codec} that is composition of a collection of codecs. - */ -public class ComposedCodec implements Codec.BytesToBytes { //TODO Caleb: Remove? - - private static final long serialVersionUID = 5068349140842235924L; - - protected static final String TYPE = "composed"; - - private final Codec[] codecs; - - public ComposedCodec(final Codec... codec) { - - this.codecs = codec; - } - - @Override - public String getType() { - - return TYPE; - } - - @Override - public InputStream decode(InputStream in) throws IOException { - - // note that decoding is in reverse order - InputStream decoded = in; - for (int i = codecs.length - 1; i >= 0; i--){} -// decoded = codecs[i].decode(decoded); - - return decoded; - } - - @Override - public OutputStream encode(OutputStream out) throws IOException { - - OutputStream encoded = out; - for (int i = 0; i < codecs.length; i++){} -// encoded = codecs[i].encode(encoded); - - return encoded; - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java index b0288adfa..9ac0a1fe0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java @@ -4,7 +4,7 @@ * A {@link Codec} that can deterministically determine the size of encoded data from the size of the raw data and vice versa from the data length alone (i.e. encoding is data * independent). */ -public interface DeterministicSizeCodec extends Codec.BytesToBytes { +public interface DeterministicSizeCodec extends Codec { public abstract long encodedSize(long size); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java index b308d31b7..93a384ddf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -7,8 +7,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @NameConfig.Name(IdentityCodec.TYPE) -@NameConfig.Prefix("codec") -public class IdentityCodec implements Codec.BytesToBytes { +public class IdentityCodec implements Codec.BytesCodec { private static final long serialVersionUID = 8354269325800855621L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java similarity index 95% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java index bdc712256..c4866fa19 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java @@ -17,8 +17,8 @@ import com.google.common.io.LittleEndianDataInputStream; import com.google.common.io.LittleEndianDataOutputStream; -@NameConfig.Name(value = N5BytesCodec.TYPE) -public class N5BytesCodec implements Codec.ArrayToBytes { +@NameConfig.Name(value = N5BlockCodec.TYPE) +public class N5BlockCodec implements Codec.ArrayCodec { private static final long serialVersionUID = 3523505403978222360L; @@ -27,12 +27,12 @@ public class N5BytesCodec implements Codec.ArrayToBytes { @NameConfig.Parameter(value = "endian", optional = true) protected final ByteOrder byteOrder; - public N5BytesCodec() { + public N5BlockCodec() { this(ByteOrder.BIG_ENDIAN); } - public N5BytesCodec(final ByteOrder byteOrder) { + public N5BlockCodec(final ByteOrder byteOrder) { this.byteOrder = byteOrder; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java index 5cf83cf05..7d7a58fbc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java @@ -8,12 +8,14 @@ import java.util.zip.CheckedOutputStream; import java.util.zip.Checksum; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; /** * A {@link Codec} that appends a checksum to data when encoding and can validate against that checksum when decoding. */ -public abstract class ChecksumCodec implements DeterministicSizeCodec { +public abstract class ChecksumCodec implements BytesCodec, DeterministicSizeCodec { private static final long serialVersionUID = 3141427377277375077L; @@ -41,14 +43,18 @@ public int numChecksumBytes() { public CheckedOutputStream encode(final OutputStream out) throws IOException { // when do we validate? - return new CheckedOutputStream(out, getChecksum()); - } - - public void encode(final OutputStream out, ByteBuffer buffer) throws IOException { - - final CheckedOutputStream cout = new CheckedOutputStream(out, getChecksum()); - cout.write(buffer.array()); - writeChecksum(out); + return new CheckedOutputStream(out, getChecksum()) { + + private boolean closed = false; + @Override public void close() throws IOException { + + if (!closed) { + writeChecksum(out); + closed = true; + out.close(); + } + } + }; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 43a90ca6f..d6d009dd9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -60,22 +60,8 @@ default int[] getBlockGridSize() { */ default long[] getBlockPosition(long... blockPosition) { - final long[] shardPos = getShard(blockPosition); - if (!Arrays.equals(getGridPosition(), shardPos)) - return null; - - final int[] shardSize = getSize(); - final int[] blkSize = getBlockSize(); - final int[] blkGridSize = getBlockGridSize(); - - final long[] blockShardPos = new long[shardSize.length]; - for (int i = 0; i < shardSize.length; i++) { - final long shardP = shardPos[i] * shardSize[i]; - final long blockP = blockPosition[i] * blkSize[i]; - blockShardPos[i] = (int)((blockP - shardP) / blkGridSize[i]); - } - - return blockShardPos; + final long[] shardPos = getDatasetAttributes().getShardPositionForBlock(blockPosition); + return getDatasetAttributes().getBlockPositionInShard(shardPos, blockPosition); } /** @@ -111,7 +97,7 @@ public static Shard createEmpty(final ShardedDatasetAttributes attributes final long[] emptyIndex = new long[(int)(2 * attributes.getNumBlocks())]; Arrays.fill(emptyIndex, EMPTY_INDEX_NBYTES); - final ShardIndex shardIndex = new ShardIndex(attributes.getShardBlockGridSize(), emptyIndex); + final ShardIndex shardIndex = new ShardIndex(attributes.getBlocksPerShard(), emptyIndex); return new InMemoryShard(attributes, shardPosition, shardIndex); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 31f510492..36ad5cd76 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -20,6 +20,8 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; public class ShardIndex extends LongArrayDataBlock { @@ -30,14 +32,20 @@ public class ShardIndex extends LongArrayDataBlock { private static final long[] DUMMY_GRID_POSITION = null; - public ShardIndex(int[] shardBlockGridSize, long[] data) { + private long byteOffset = -1; + + private final DeterministicSizeCodec[] codecs; + + public ShardIndex(int[] shardBlockGridSize, long[] data, final DeterministicSizeCodec... codecs) { super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, data); + this.codecs = codecs; } - public ShardIndex(int[] shardBlockGridSize) { + public ShardIndex(int[] shardBlockGridSize, final DeterministicSizeCodec... codecs) { super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, emptyIndexData(shardBlockGridSize)); + this.codecs = codecs; } public boolean exists(long... gridPosition) { @@ -77,28 +85,46 @@ private int getOffsetIndex(long... gridPosition) { private int getNumBytesIndex(long... gridPosition) { - return getOffsetIndex() + 1; + return getOffsetIndex(gridPosition) + 1; } - public static ShardIndex read(final KeyValueAccess keyValueAccess, final String key, - final ShardedDatasetAttributes datasetAttributes) throws IOException { + public static ShardIndex read( + final KeyValueAccess keyValueAccess, + final String key, + final ShardedDatasetAttributes datasetAttributes + ) throws IOException { - return read(keyValueAccess, key, datasetAttributes.getShardBlockGridSize(), datasetAttributes.getIndexLocation()); + final IndexLocation indexLocation = datasetAttributes.getIndexLocation(); + return read(keyValueAccess, key, datasetAttributes.createIndex(), indexLocation); + } + + public long numBytes() { + + final int numEntries = Arrays.stream(getSize()).reduce(1, (x, y) -> x * y); + final int numBytesFromBlocks = numEntries * BYTES_PER_LONG; + long totalNumBytes = numBytesFromBlocks; + for (Codec codec : codecs) { + if (codec instanceof DeterministicSizeCodec) { + totalNumBytes = ((DeterministicSizeCodec)codec).encodedSize(totalNumBytes); + } + } + return totalNumBytes; } public static ShardIndex read( final KeyValueAccess keyValueAccess, final String key, - final int[] shardBlockGridSize, - final IndexLocation indexLocation) throws IOException { + final ShardIndex idx, + final IndexLocation indexLocation + ) throws IOException { - final ShardIndex idx = new ShardIndex(shardBlockGridSize); - final IndexByteBounds byteBounds = byteBounds(idx.getSize(), indexLocation, keyValueAccess.size(key)); + final IndexByteBounds byteBounds = byteBounds(idx.numBytes(), indexLocation, keyValueAccess.size(key)); + idx.byteOffset = byteBounds.start; try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(key, byteBounds.start, byteBounds.end)) { final byte[] bytes = new byte[idx.getNumElements() * ShardIndex.BYTES_PER_LONG]; lockedChannel.newInputStream().read(bytes); - idx.readData(ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)); // TODO generalize byte order + idx.readData(ByteBuffer.wrap(bytes).order(ByteOrder.BIG_ENDIAN)); // TODO generalize byte order return idx; } catch (final N5Exception.N5NoSuchKeyException e) { @@ -114,7 +140,7 @@ public static void write(ShardIndex index, final int[] shardBlockGridSize, final IndexLocation indexLocation) throws IOException { - final IndexByteBounds byteBounds = byteBounds(index.getSize(), indexLocation, keyValueAccess.size(key)); + final IndexByteBounds byteBounds = byteBounds(index.numBytes(), indexLocation, keyValueAccess.size(key)); try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(key, byteBounds.start, byteBounds.end)) { final OutputStream os = lockedChannel.newOutputStream(); @@ -139,21 +165,23 @@ public static DatasetAttributes indexDatasetAttributes(final int[] indexBlockSiz public static IndexByteBounds byteBounds(ShardedDatasetAttributes datasetAttributes, final long objectSize) { - final int[] indexShape = prepend(2, datasetAttributes.getShardBlockGridSize()); - return byteBounds(indexShape, datasetAttributes.getIndexLocation(), objectSize); + final long indexSize = datasetAttributes.createIndex().numBytes(); + return byteBounds(indexSize, datasetAttributes.getIndexLocation(), objectSize); } - public static IndexByteBounds byteBounds(final int[] indexShape, final IndexLocation indexLocation, final long objectSize) { - - final int indexSize = (int)Arrays.stream(indexShape).reduce(1, (x, y) -> x * y); + public static IndexByteBounds byteBounds(final long indexSize, final IndexLocation indexLocation, final long objectSize) { if (indexLocation == IndexLocation.START) { return new IndexByteBounds(0L, indexSize); } else { - return new IndexByteBounds(objectSize - (BYTES_PER_LONG * indexSize), objectSize - 1); + return new IndexByteBounds(objectSize - indexSize, objectSize - 1); } } + public long getByteOffset() { + return byteOffset; + } + private static class IndexByteBounds { private final long start; @@ -170,7 +198,7 @@ public static ShardIndex read(FileChannel channel, ShardedDatasetAttributes data // TODO need codecs // TODO FileChannel is too specific - generalize - final int[] indexShape = prepend(2, datasetAttributes.getShardBlockGridSize()); + final int[] indexShape = prepend(2, datasetAttributes.getBlocksPerShard()); final int indexSize = (int)Arrays.stream(indexShape).reduce(1, (x, y) -> x * y); final int indexBytes = BYTES_PER_LONG * indexSize; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java index 4584d1dbd..edab4ec0d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java @@ -5,10 +5,11 @@ import org.janelia.saalfeldlab.n5.DefaultBlockReader; import org.janelia.saalfeldlab.n5.N5FSReader; import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.codec.IdentityCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; @@ -50,7 +51,7 @@ public DataBlock readBlock( private long getIndexIndex(long... shardPosition) { - final int[] indexDimensions = datasetAttributes.getShardBlockGridSize(); + final int[] indexDimensions = datasetAttributes.getBlocksPerShard(); long idx = 0; for (int i = 0; i < indexDimensions.length; i++) { idx += shardPosition[i] * indexDimensions[i]; @@ -76,17 +77,14 @@ public static void main(String[] args) { private static ShardedDatasetAttributes buildTestAttributes() { - final Codec[] codecs = new Codec[]{ - new IdentityCodec(), - new ShardingCodec( - new int[]{2, 2}, - new Codec[]{new RawCompression(), new IdentityCodec()}, - new Codec[]{new Crc32cChecksumCodec()}, - IndexLocation.END - ) - }; - - return new ShardedDatasetAttributes(new long[]{4, 4}, new int[]{2, 2}, new int[]{2, 2}, IndexLocation.END, DataType.INT32, new RawCompression(), codecs); + return new ShardedDatasetAttributes( + new long[]{4, 4}, + new int[]{2, 2}, + new int[]{2, 2}, + DataType.INT32, + new Codec[]{new N5BlockCodec(), new IdentityCodec()}, + new DeterministicSizeCodec[]{new Crc32cChecksumCodec()}, + IndexLocation.END); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java index 1c18e5643..4a288878a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java @@ -1,5 +1,9 @@ package org.janelia.saalfeldlab.n5.shard; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DefaultBlockWriter; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; + import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; @@ -9,10 +13,6 @@ import java.util.Arrays; import java.util.List; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DefaultBlockWriter; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; - public class ShardWriter { private static final int BYTES_PER_LONG = 8; @@ -77,21 +77,22 @@ private void prepareForWritingDataBlock() throws IOException { // final ShardingProperties shardProps = new ShardingProperties(datasetAttributes); // indexData = new ShardIndexDataBlock(shardProps.getIndexDimensions()); - indexData = new ShardIndex(new int[]{blocks.size()}); + indexData = datasetAttributes.createIndex(); blockBytes = new ArrayList<>(); long cumulativeBytes = 0; final long[] shardPosition = new long[1]; for (int i = 0; i < blocks.size(); i++) { - final ByteArrayOutputStream blockOut = new ByteArrayOutputStream(); - DefaultBlockWriter.writeBlock(blockOut, datasetAttributes, blocks.get(i)); - System.out.println(String.format("block %d is %d bytes", i, blockOut.size())); + try (final ByteArrayOutputStream blockOut = new ByteArrayOutputStream()) { + DefaultBlockWriter.writeBlock(blockOut, datasetAttributes, blocks.get(i)); + System.out.println(String.format("block %d is %d bytes", i, blockOut.size())); - shardPosition[0] = i; - indexData.set(cumulativeBytes, blockOut.size(), shardPosition); - cumulativeBytes += blockOut.size(); + shardPosition[0] = i; + indexData.set(cumulativeBytes, blockOut.size(), shardPosition); + cumulativeBytes += blockOut.size(); - blockBytes.add(blockOut.toByteArray()); + blockBytes.add(blockOut.toByteArray()); + } } System.out.println(Arrays.toString(indexData.getData())); @@ -105,15 +106,17 @@ private void prepareForWriting() throws IOException { long cumulativeBytes = 0; for (int i = 0; i < blocks.size(); i++) { - final ByteArrayOutputStream blockOut = new ByteArrayOutputStream(); - DefaultBlockWriter.writeBlock(blockOut, datasetAttributes, blocks.get(i)); - System.out.println(String.format("block %d is %d bytes", i, blockOut.size())); + try (final ByteArrayOutputStream blockOut = new ByteArrayOutputStream()) { + + DefaultBlockWriter.writeBlock(blockOut, datasetAttributes, blocks.get(i)); + System.out.println(String.format("block %d is %d bytes", i, blockOut.size())); - blockIndexes.putLong(cumulativeBytes); - blockSizes.putLong(blockOut.size()); - cumulativeBytes += blockOut.size(); + blockIndexes.putLong(cumulativeBytes); + blockSizes.putLong(blockOut.size()); + cumulativeBytes += blockOut.size(); - blockBytes.add(blockOut.toByteArray()); + blockBytes.add(blockOut.toByteArray()); + } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index dc5392d4a..af06ec0e3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -7,7 +7,10 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.serialization.NameConfig; import java.io.IOException; @@ -16,7 +19,7 @@ import java.lang.reflect.Type; @NameConfig.Name(ShardingCodec.TYPE) -public class ShardingCodec implements Codec.BytesToBytes { //TODO Caleb: should be ArrayToBytes +public class ShardingCodec implements Codec.ArrayCodec { private static final long serialVersionUID = -5879797314954717810L; @@ -28,7 +31,7 @@ public class ShardingCodec implements Codec.BytesToBytes { //TODO Caleb: should public static final String INDEX_CODECS_KEY = "index_codecs"; public enum IndexLocation { - START, END + START, END; } @NameConfig.Parameter(CHUNK_SHAPE_KEY) @@ -38,7 +41,7 @@ public enum IndexLocation { private final Codec[] codecs; @NameConfig.Parameter(INDEX_CODECS_KEY) - private final Codec[] indexCodecs; + private final DeterministicSizeCodec[] indexCodecs; @NameConfig.Parameter(INDEX_LOCATION_KEY) private final IndexLocation indexLocation; @@ -54,7 +57,7 @@ private ShardingCodec() { public ShardingCodec( final int[] blockSize, final Codec[] codecs, - final Codec[] indexCodecs, + final DeterministicSizeCodec[] indexCodecs, final IndexLocation indexLocation) { this.blockSize = blockSize; @@ -73,25 +76,31 @@ public IndexLocation getIndexLocation() { return indexLocation; } - @Override - public InputStream decode(InputStream in) throws IOException { + public ArrayCodec getArrayCodec() { - // TODO Auto-generated method stub - // This method actually makes no sense for a sharding codec - return in; + return (Codec.ArrayCodec)codecs[0]; } - @Override - public OutputStream encode(OutputStream out) throws IOException { + public BytesCodec[] getCodecs() { + + final BytesCodec[] bytesCodecs = new BytesCodec[codecs.length - 1]; + System.arraycopy(codecs, 1, bytesCodecs, 0, bytesCodecs.length); + return bytesCodecs; + } + + public DeterministicSizeCodec[] getIndexCodecs() { + + return indexCodecs; + } + + @Override public DataBlockInputStream decode(DatasetAttributes attributes, long[] gridPosition, InputStream in) throws IOException { - // TODO Auto-generated method stub - // This method actually makes no sense for a sharding codec - return out; + return getArrayCodec().decode(attributes, gridPosition, in); } - public static boolean isShardingCodec(final Codec codec) { + @Override public DataBlockOutputStream encode(DatasetAttributes attributes, DataBlock datablock, OutputStream out) throws IOException { - return codec instanceof ShardingCodec; + return getArrayCodec().encode(attributes, datablock, out); } @Override @@ -106,7 +115,8 @@ public static class IndexLocationAdapter implements JsonSerializer extends AbstractShard { - private KeyValueAccess keyValueAccess; - private String path; + final private KeyValueAccess keyValueAccess; + final private String path; public VirtualShard(final ShardedDatasetAttributes datasetAttributes, long[] gridPosition, final KeyValueAccess keyValueAccess, final String path) { @@ -38,18 +40,15 @@ public DataBlock getBlock(long... blockGridPosition) { final ShardIndex idx = getIndex(); final long startByte = idx.getOffset(relativePosition); - final long endByte = startByte + idx.getNumBytes(relativePosition); - try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path, startByte, endByte)) { - // TODO add codecs, generalize to use any BlockReader - final DataBlock dataBlock = (DataBlock)datasetAttributes.getDataType().createDataBlock( - datasetAttributes.getBlockSize(), - blockGridPosition, - numBlockElements(datasetAttributes)); - - DefaultBlockReader.readFromStream(dataBlock, lockedChannel.newInputStream()); - return dataBlock; + if (startByte == Shard.EMPTY_INDEX_NBYTES ) + return null; + final long size = idx.getNumBytes(relativePosition); + try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path, startByte, size)) { + try ( final InputStream channelIn = lockedChannel.newInputStream()) { + return (DataBlock)DefaultBlockReader.readBlock(channelIn, datasetAttributes, blockGridPosition); + } } catch (final N5Exception.N5NoSuchKeyException e) { return null; } catch (final IOException | UncheckedIOException e) { @@ -65,21 +64,31 @@ public void writeBlock(final DataBlock block) { throw new N5IOException("Attempted to write block in the wrong shard."); final ShardIndex idx = getIndex(); - final long startByte = idx.getOffset(relativePosition) == Shard.EMPTY_INDEX_NBYTES ? 0 : idx.getOffset(relativePosition); - final long size = idx.getNumBytes(relativePosition) == Shard.EMPTY_INDEX_NBYTES ? Long.MAX_VALUE : idx.getNumBytes(relativePosition); - // TODO this assumes that the block exists in the shard and - // that the available space is sufficient. Should generalize - try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(path, startByte, size)) { - // TODO codecs - final CountingOutputStream out = new CountingOutputStream(lockedChannel.newOutputStream()); - datasetAttributes.getCompression().getWriter().write(block, out); + //TODO Caleb: reusing the offset of a prior block write is only safe when writing the same amount, or less, data. + // This is not generally guaranteed, since we compress the data. + // Either need to known the compressed size before writing, append only, or only overwrite when not compressing + final long getBlockOffset = idx.getOffset(relativePosition); + final long startByte; + if (getBlockOffset == Shard.EMPTY_INDEX_NBYTES) { + final long indexByteOffset = idx.getByteOffset(); + startByte = indexByteOffset == -1 ? 0 : idx.getByteOffset(); + } else { + startByte = getBlockOffset; + } + final long size = Long.MAX_VALUE - startByte; - // TODO update index when we know how many bytes were written - idx.set(startByte, out.getNumBytes(), relativePosition); - out.write(index.toByteBuffer().array()); - out.realClose(); + try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(path, startByte, size)) { + try ( final OutputStream channelOut = lockedChannel.newOutputStream()) { + try (final CountingOutputStream out = new CountingOutputStream(channelOut)) { + DefaultBlockWriter.writeBlock(out, datasetAttributes, block); + + /* Update and write the index to the shard*/ + idx.set(startByte, out.getNumBytes(), relativePosition); + DefaultBlockWriter.writeBlock(out, datasetAttributes.getIndexAttributes(), idx); + } + } } catch (final IOException | UncheckedIOException e) { throw new N5IOException("Failed to read block from " + path, e); } @@ -100,21 +109,21 @@ private static int numBlockElements(DatasetAttributes datasetAttributes) { public ShardIndex createIndex() { // Empty index of the correct size - index = new ShardIndex(datasetAttributes.getShardBlockGridSize()); - return index; + return datasetAttributes.createIndex(); } @Override public ShardIndex getIndex() { try { - final ShardIndex result = ShardIndex.read(keyValueAccess, path, datasetAttributes); - return result == null ? createIndex() : result; + final ShardIndex readIndex = ShardIndex.read(keyValueAccess, path, datasetAttributes); + index = readIndex == null ? createIndex() : readIndex; } catch (final NoSuchFileException e) { - return createIndex(); + index = createIndex(); } catch (IOException e) { throw new N5IOException("Failed to read index at " + path, e); } + return index; } @@ -155,10 +164,6 @@ public void close() throws IOException { } - private void realClose() throws IOException { - out.close(); - } - public long getNumBytes() { return numBytes; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 939b99254..d4d3591ba 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -52,7 +52,7 @@ import org.janelia.saalfeldlab.n5.N5Reader.Version; import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.N5BytesCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -257,7 +257,7 @@ public void testWriteReadByteBlockMultipleCodecs() { * maybe is not the behavior we actually want*/ try (final N5Writer n5 = createTempN5Writer()) { final Codec[] codecs = { - new N5BytesCodec(), + new N5BlockCodec(), new AsTypeCodec(DataType.INT32, DataType.INT8), new AsTypeCodec(DataType.INT64, DataType.INT32), }; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java index ee36a51d9..59aa32984 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java @@ -47,20 +47,20 @@ public void testDouble2Byte() throws IOException { testEncodingAndDecoding(new AsTypeCodec(DataType.INT8, DataType.FLOAT64), decodedDoubles, encodedBytes); } - public static void testEncodingAndDecoding(Codec.BytesToBytes codec, byte[] encodedBytes, byte[] decodedBytes) throws IOException { + public static void testEncodingAndDecoding(Codec.BytesCodec codec, byte[] encodedBytes, byte[] decodedBytes) throws IOException { testEncoding(codec, encodedBytes, decodedBytes); testDecoding(codec, decodedBytes, encodedBytes); } - public static void testDecoding(final Codec.BytesToBytes codec, final byte[] expected, final byte[] input) throws IOException { + public static void testDecoding(final Codec.BytesCodec codec, final byte[] expected, final byte[] input) throws IOException { final InputStream result = codec.decode(new ByteArrayInputStream(input)); for (int i = 0; i < expected.length; i++) assertEquals(expected[i], (byte)result.read()); } - public static void testEncoding(final Codec.BytesToBytes codec, final byte[] expected, final byte[] data) throws IOException { + public static void testEncoding(final Codec.BytesCodec codec, final byte[] expected, final byte[] data) throws IOException { final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(expected.length); final OutputStream encodedStream = codec.encode(outputStream); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java index fa407163e..f911ec6d2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java @@ -29,7 +29,7 @@ public void testSerialization() { final N5Writer reader = factory.openWriter("n5:src/test/resources/shardExamples/test.zarr"); final Codec bytes = reader.getAttribute("mid_sharded", "codecs[0]/configuration/codecs[0]", Codec.class); - assertTrue("as BytesCodec", bytes instanceof N5BytesCodec); + assertTrue("as BytesCodec", bytes instanceof N5BlockCodec); final N5Writer writer = factory.openWriter("n5:src/test/resources/shardExamples/test.n5"); @@ -39,7 +39,7 @@ public void testSerialization() { DataType.UINT8, new RawCompression(), new Codec[]{ - new N5BytesCodec(ByteOrder.LITTLE_ENDIAN), + new N5BlockCodec(ByteOrder.LITTLE_ENDIAN), new IdentityCodec() } ); @@ -48,8 +48,8 @@ public void testSerialization() { assertEquals("1 codecs", 1, deserialized.getCodecs().length); assertTrue("Identity", deserialized.getCodecs()[0] instanceof IdentityCodec); - assertTrue("Bytes", deserialized.getArrayToBytesCodec() instanceof N5BytesCodec); + assertTrue("Bytes", deserialized.getArrayCodec() instanceof N5BlockCodec); assertEquals("LittleEndian", ByteOrder.LITTLE_ENDIAN, - ((N5BytesCodec)deserialized.getArrayToBytesCodec()).byteOrder); + ((N5BlockCodec)deserialized.getArrayCodec()).byteOrder); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java index f12150b6c..0610c7c50 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java @@ -1,9 +1,11 @@ package org.janelia.saalfeldlab.n5.serialization; +import static org.janelia.saalfeldlab.n5.NameConfigAdapter.getJsonAdapter; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.GsonUtils; import org.janelia.saalfeldlab.n5.GzipCompression; import org.janelia.saalfeldlab.n5.NameConfigAdapter; import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; @@ -26,16 +28,9 @@ public class CodecSerialization { @Before public void before() { - final GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(IdentityCodec.class, NameConfigAdapter.getJsonAdapter(IdentityCodec.class)); - gsonBuilder.registerTypeAdapter(AsTypeCodec.class, NameConfigAdapter.getJsonAdapter(AsTypeCodec.class)); - gsonBuilder.registerTypeAdapter(FixedScaleOffsetCodec.class, - NameConfigAdapter.getJsonAdapter(FixedScaleOffsetCodec.class)); - gsonBuilder.registerTypeAdapter(GzipCompression.class, - NameConfigAdapter.getJsonAdapter(GzipCompression.class)); - gsonBuilder.registerTypeAdapter(Codec.class, - NameConfigAdapter.getJsonAdapter(Codec.class)); + final GsonBuilder gsonBuilder = new GsonBuilder(); + GsonUtils.registerGson(gsonBuilder); gson = gsonBuilder.create(); } @@ -44,7 +39,7 @@ public void testSerializeIdentity() { final IdentityCodec id = new IdentityCodec(); final JsonObject jsonId = gson.toJsonTree(id).getAsJsonObject(); - final JsonElement expected = gson.fromJson("{\"name\":\"id\", \"configuration\":{}}", JsonElement.class); + final JsonElement expected = gson.fromJson("{\"name\":\"id\"}", JsonElement.class); assertEquals("identity", expected, jsonId.getAsJsonObject()); } @@ -54,7 +49,7 @@ public void testSerializeAsType() { final AsTypeCodec asTypeCodec = new AsTypeCodec(DataType.FLOAT64, DataType.INT16); final JsonObject jsonAsType = gson.toJsonTree(asTypeCodec).getAsJsonObject(); final JsonElement expected = gson.fromJson( - "{\"name\":\"astype\",\"configuration\":{\"dataType\":\"FLOAT64\",\"encodedType\":\"INT16\"}}", + "{\"name\":\"astype\",\"configuration\":{\"dataType\":\"float64\",\"encodedType\":\"int16\"}}", JsonElement.class); assertEquals("asType", expected, jsonAsType.getAsJsonObject()); } @@ -68,7 +63,7 @@ public void testSerializeCodecArray() { }; JsonArray jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); JsonElement expected = gson.fromJson( - "[{\"name\":\"id\",\"configuration\":{}},{\"name\":\"astype\",\"configuration\":{\"dataType\":\"FLOAT64\",\"encodedType\":\"INT16\"}}]", + "[{\"name\":\"id\"},{\"name\":\"astype\",\"configuration\":{\"dataType\":\"float64\",\"encodedType\":\"int16\"}}]", JsonElement.class); assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); @@ -83,7 +78,7 @@ public void testSerializeCodecArray() { }; jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); expected = gson.fromJson( - "[{\"name\":\"astype\",\"configuration\":{\"dataType\":\"FLOAT64\",\"encodedType\":\"INT16\"}},{\"name\":\"gzip\",\"configuration\":{\"level\":-1,\"use_z_lib\":false}}]", + "[{\"name\":\"astype\",\"configuration\":{\"dataType\":\"float64\",\"encodedType\":\"int16\"}},{\"name\":\"gzip\",\"configuration\":{\"level\":-1,\"useZlib\":false}}]", JsonElement.class); assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java index 50981dc69..4bab3d45b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -6,17 +6,19 @@ import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.GzipCompression; import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; -import org.janelia.saalfeldlab.n5.codec.N5BytesCodec; +import org.janelia.saalfeldlab.n5.codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.IdentityCodec; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.junit.Assert; import org.junit.Test; import java.net.MalformedURLException; +import java.nio.ByteOrder; import java.nio.file.FileSystems; import java.nio.file.Path; import java.nio.file.Paths; @@ -30,15 +32,22 @@ public static void main(String[] args) throws MalformedURLException { System.out.println(p); final String key = p.toString(); - final ShardedDatasetAttributes dsetAttrs = new ShardedDatasetAttributes(new long[]{6, 4}, new int[]{6, 4}, - new int[]{3, 2}, IndexLocation.END, DataType.UINT8, new RawCompression(), null); + final ShardedDatasetAttributes dsetAttrs = new ShardedDatasetAttributes( + new long[]{6, 4}, + new int[]{6, 4}, + new int[]{3, 2}, + DataType.UINT8, + new Codec[]{new N5BlockCodec()}, + new DeterministicSizeCodec[]{new BytesCodec(), new Crc32cChecksumCodec()}, + IndexLocation.END + ); final FileSystemKeyValueAccess kva = new FileSystemKeyValueAccess(FileSystems.getDefault()); final VirtualShard shard = new VirtualShard<>(dsetAttrs, new long[]{0, 0}, kva, key); final DataBlock blk = shard.getBlock(0, 0); - final byte[] data = (byte[])blk.getData(); + final byte[] data = blk.getData(); System.out.println(Arrays.toString(data)); // fill the block with a weird value @@ -49,7 +58,7 @@ public static void main(String[] args) throws MalformedURLException { // re-read the block and check the data it contains final DataBlock blkReread = shard.getBlock(0, 0); - final byte[] dataReRead = (byte[])blkReread.getData(); + final byte[] dataReRead = blkReread.getData(); System.out.println(Arrays.toString(dataReRead)); } @@ -68,30 +77,34 @@ public void writeReadBlockTest() { new long[]{8, 8}, new int[]{4, 4}, new int[]{2, 2}, - IndexLocation.END, DataType.UINT8, - new RawCompression(), - new Codec[]{ - new N5BytesCodec(), - new ShardingCodec( - new int[]{2, 2}, - new Codec[]{new N5BytesCodec(), new GzipCompression(4)}, - new Codec[]{new Crc32cChecksumCodec()}, - IndexLocation.END - ) - } + new Codec[]{new N5BlockCodec(), new GzipCompression(4)}, + new DeterministicSizeCodec[]{new BytesCodec(ByteOrder.BIG_ENDIAN), new Crc32cChecksumCodec()}, + IndexLocation.END ); writer.createDataset("shard", datasetAttributes); + writer.deleteBlock("shard", 0, 0); - final DataBlock dataBlock = datasetAttributes.getDataType().createDataBlock(datasetAttributes.getBlockSize(), new long[]{0, 0}, 2 * 2); - byte[] data = (byte[])dataBlock.getData(); - for (int i = 0; i < data.length; i++) { - data[i] = (byte)i; - } + final int[] blockSize = datasetAttributes.getBlockSize(); + final DataType dataType = datasetAttributes.getDataType(); + final int numElements = 2 * 2; + + for (int idx1 = 1; idx1 >= 0; idx1--) { + for (int idx2 = 1; idx2 >= 0; idx2--) { - writer.deleteBlock("shard", 0,0 ); - writer.writeBlock("shard", datasetAttributes, dataBlock); - writer.readBlock("shard", datasetAttributes, 0, 0); + final long[] gridPosition = {idx1, idx2}; + final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(blockSize, gridPosition, numElements); + byte[] data = dataBlock.getData(); + for (int i = 0; i < data.length; i++) { + data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); + } + writer.writeBlock("shard", datasetAttributes, dataBlock); + + final DataBlock block = (DataBlock)writer.readBlock("shard", datasetAttributes, gridPosition); + + Assert.assertArrayEquals("Read from shard doesn't match", data, block.getData()); + } + } } } From 7a1bbcb33c358a1c78f52f0867b92b2fe9202380 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 17 Sep 2024 14:39:58 -0400 Subject: [PATCH 055/423] feat: rethrow NoSuchFile as NoSuchKey --- .../janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index ffed4c1e7..317c2c5de 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -266,8 +266,11 @@ public boolean exists(final String normalPath) { @Override public long size(final String normalPath) throws IOException { - final Path path = fileSystem.getPath(normalPath); - return Files.size(path); + try { + return Files.size(fileSystem.getPath(normalPath)); + } catch (NoSuchFileException e) { + throw new N5Exception.N5NoSuchKeyException("No such file", e); + } } @Override From 9fac328eb1f7fa88f5c3eccdf49ec2cc58680528 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 17 Sep 2024 14:41:50 -0400 Subject: [PATCH 056/423] feat: wip support for ShardIndex location and bytesorder This is very preliminary. It works in a way, but is not as performant or reasonable as it likely should be. Consider this an initial implementation proof of concept for implementing the rest of sharding, especially working towards multiple read/write block aggregations --- .../n5/ShardedDatasetAttributes.java | 18 ++- .../saalfeldlab/n5/codec/N5BlockCodec.java | 4 +- .../janelia/saalfeldlab/n5/shard/Shard.java | 2 +- .../saalfeldlab/n5/shard/ShardIndex.java | 131 +++++++++--------- .../saalfeldlab/n5/shard/VirtualShard.java | 38 ++--- .../saalfeldlab/n5/shard/ShardDemos.java | 2 +- 6 files changed, 98 insertions(+), 97 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index c72bad299..794b88611 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -132,7 +132,10 @@ public long[] getBlockPositionInShard(final long[] shardPosition, final long[] b final int[] shardSize = getShardSize(); final int[] blkSize = getBlockSize(); - final int[] blkGridSize = getShardBlockGridSize(); + final int[] blkGridSize = getBlocksPerShard(); +// final int[] shardSize = getSize(); +// final int[] blkSize = getBlockSize(); +// final int[] blkGridSize = getBlockGridSize(); final long[] blockShardPos = new long[shardSize.length]; for (int i = 0; i < shardSize.length; i++) { @@ -142,6 +145,9 @@ public long[] getBlockPositionInShard(final long[] shardPosition, final long[] b } return blockShardPos; + + + } /** @@ -167,14 +173,6 @@ public IndexLocation getIndexLocation() { } public ShardIndex createIndex() { - return new ShardIndex(getBlocksPerShard(), getShardingCodec().getIndexCodecs()); - } - - public DatasetAttributes getIndexAttributes() { - return createShardIndexAttributes(getShardingCodec().getIndexCodecs()); - } - - private static DatasetAttributes createShardIndexAttributes(Codec[] indexCodecs) { - return new DatasetAttributes(null, null, null, null, indexCodecs); + return new ShardIndex(getBlocksPerShard(), getIndexLocation(), getShardingCodec().getIndexCodecs()); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java index c4866fa19..82f118bb7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java @@ -70,7 +70,7 @@ public DataBlock allocateDataBlock() throws IOException { } private void readHeader() throws IOException { - final DataInputStream dis = new DataInputStream(in); + final DataInput dis = getDataInput(in); mode = dis.readShort(); if (mode != 2) { final int nDim = dis.readShort(); @@ -119,7 +119,7 @@ protected void beforeWrite(int n) throws IOException { } private void writeHeader() throws IOException { - final DataOutputStream dos = new DataOutputStream(out); + final DataOutput dos = getDataOutput(out); final int mode; if (attributes.getDataType() == DataType.OBJECT || dataBlock.getSize() == null) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index d6d009dd9..311343709 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -97,7 +97,7 @@ public static Shard createEmpty(final ShardedDatasetAttributes attributes final long[] emptyIndex = new long[(int)(2 * attributes.getNumBlocks())]; Arrays.fill(emptyIndex, EMPTY_INDEX_NBYTES); - final ShardIndex shardIndex = new ShardIndex(attributes.getBlocksPerShard(), emptyIndex); + final ShardIndex shardIndex = new ShardIndex(attributes.getBlocksPerShard(), emptyIndex, ShardingCodec.IndexLocation.END); return new InMemoryShard(attributes, shardPosition, shardIndex); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 36ad5cd76..1c840b1ea 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -1,29 +1,29 @@ package org.janelia.saalfeldlab.n5.shard; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UncheckedIOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; -import java.util.Arrays; - +import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.DefaultBlockReader; +import org.janelia.saalfeldlab.n5.DefaultBlockWriter; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.util.Arrays; + public class ShardIndex extends LongArrayDataBlock { private static final int BYTES_PER_LONG = 8; @@ -31,21 +31,25 @@ public class ShardIndex extends LongArrayDataBlock { private static final int LONGS_PER_BLOCK = 2; private static final long[] DUMMY_GRID_POSITION = null; - - private long byteOffset = -1; + private final IndexLocation location; private final DeterministicSizeCodec[] codecs; - public ShardIndex(int[] shardBlockGridSize, long[] data, final DeterministicSizeCodec... codecs) { + public ShardIndex(int[] shardBlockGridSize, long[] data, IndexLocation location, final DeterministicSizeCodec... codecs) { super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, data); this.codecs = codecs; + this.location = location; } - public ShardIndex(int[] shardBlockGridSize, final DeterministicSizeCodec... codecs) { + public ShardIndex(int[] shardBlockGridSize, IndexLocation location, DeterministicSizeCodec... codecs) { - super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, emptyIndexData(shardBlockGridSize)); - this.codecs = codecs; + this(shardBlockGridSize, emptyIndexData(shardBlockGridSize), location, codecs); + } + + public ShardIndex(int[] shardBlockGridSize, DeterministicSizeCodec... codecs) { + + this(shardBlockGridSize, emptyIndexData(shardBlockGridSize), IndexLocation.END, codecs); } public boolean exists(long... gridPosition) { @@ -54,6 +58,11 @@ public boolean exists(long... gridPosition) { getNumBytes(gridPosition) != Shard.EMPTY_INDEX_NBYTES; } + public IndexLocation getLocation() { + + return location; + } + public long getOffset(long... gridPosition) { return data[getOffsetIndex(gridPosition)]; @@ -88,16 +97,6 @@ private int getNumBytesIndex(long... gridPosition) { return getOffsetIndex(gridPosition) + 1; } - public static ShardIndex read( - final KeyValueAccess keyValueAccess, - final String key, - final ShardedDatasetAttributes datasetAttributes - ) throws IOException { - - final IndexLocation indexLocation = datasetAttributes.getIndexLocation(); - return read(keyValueAccess, key, datasetAttributes.createIndex(), indexLocation); - } - public long numBytes() { final int numEntries = Arrays.stream(getSize()).reduce(1, (x, y) -> x * y); @@ -114,53 +113,55 @@ public long numBytes() { public static ShardIndex read( final KeyValueAccess keyValueAccess, final String key, - final ShardIndex idx, - final IndexLocation indexLocation - ) throws IOException { + final ShardIndex index + ) throws IOException { - final IndexByteBounds byteBounds = byteBounds(idx.numBytes(), indexLocation, keyValueAccess.size(key)); - idx.byteOffset = byteBounds.start; + final IndexByteBounds byteBounds = byteBounds(index, keyValueAccess.size(key)); try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(key, byteBounds.start, byteBounds.end)) { - - final byte[] bytes = new byte[idx.getNumElements() * ShardIndex.BYTES_PER_LONG]; - lockedChannel.newInputStream().read(bytes); - idx.readData(ByteBuffer.wrap(bytes).order(ByteOrder.BIG_ENDIAN)); // TODO generalize byte order - return idx; - + final long[] indexData; + try (final InputStream in = lockedChannel.newInputStream()) { + final DataBlock indexBlock = (DataBlock)DefaultBlockReader.readBlock( + in, + index.getIndexAttributes(), + index.gridPosition); + indexData = indexBlock.getData(); + } + System.arraycopy(indexData, 0, index.data, 0, index.data.length); + return index; } catch (final N5Exception.N5NoSuchKeyException e) { return null; } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to read from " + key, e); + throw new N5IOException("Failed to read shard index from " + key, e); } } - public static void write(ShardIndex index, + public static void write( + final ShardIndex index, final KeyValueAccess keyValueAccess, - final String key, - final int[] shardBlockGridSize, - final IndexLocation indexLocation) throws IOException { - - final IndexByteBounds byteBounds = byteBounds(index.numBytes(), indexLocation, keyValueAccess.size(key)); - try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(key, byteBounds.start, byteBounds.end)) { - - final OutputStream os = lockedChannel.newOutputStream(); - os.write(index.toByteBuffer().array()); + final String key + ) throws IOException { + final long start = index.location == IndexLocation.START ? 0 : keyValueAccess.size(key); + try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(key, start, index.numBytes())) { + try (final OutputStream os = lockedChannel.newOutputStream()) { + DefaultBlockWriter.writeBlock(os, index.getIndexAttributes(), index); + } } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to read from " + key, e); + throw new N5IOException("Failed to write shard index to " + key, e); } } - public static DatasetAttributes indexDatasetAttributes(final int[] indexBlockSize) { + private DatasetAttributes getIndexAttributes() { - final int[] blkSize = new int[indexBlockSize.length]; - final long[] size = new long[indexBlockSize.length]; - for (int i = 0; i < blkSize.length; i++) { - blkSize[i] = (int)indexBlockSize[i]; - } - - // TODO codecs - return new DatasetAttributes(size, blkSize, DataType.UINT64, new RawCompression(), null); + final DatasetAttributes indexAttributes = + new DatasetAttributes( + Arrays.stream(getSize()).mapToLong(it -> it).toArray(), + getSize(), + DataType.UINT64, + null, + codecs + ); + return indexAttributes; } public static IndexByteBounds byteBounds(ShardedDatasetAttributes datasetAttributes, final long objectSize) { @@ -169,6 +170,10 @@ public static IndexByteBounds byteBounds(ShardedDatasetAttributes datasetAttribu return byteBounds(indexSize, datasetAttributes.getIndexLocation(), objectSize); } + public static IndexByteBounds byteBounds(final ShardIndex index, long objectSize) { + return byteBounds(index.numBytes(), index.location, objectSize); + } + public static IndexByteBounds byteBounds(final long indexSize, final IndexLocation indexLocation, final long objectSize) { if (indexLocation == IndexLocation.START) { @@ -178,10 +183,6 @@ public static IndexByteBounds byteBounds(final long indexSize, final IndexLocati } } - public long getByteOffset() { - return byteOffset; - } - private static class IndexByteBounds { private final long start; @@ -214,7 +215,7 @@ public static ShardIndex read(FileChannel channel, ShardedDatasetAttributes data indexes[i] = dis.readLong(); } - return new ShardIndex(indexShape, indexes); + return new ShardIndex(indexShape, indexes, IndexLocation.END); } private static long[] emptyIndexData(final int[] size) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 332f77b62..1dd63f34d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.shard; +import java.io.DataOutput; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -39,6 +40,8 @@ public DataBlock getBlock(long... blockGridPosition) { throw new N5IOException("Attempted to read a block from the wrong shard."); final ShardIndex idx = getIndex(); + + final long startByte = idx.getOffset(relativePosition); if (startByte == Shard.EMPTY_INDEX_NBYTES ) @@ -63,19 +66,14 @@ public void writeBlock(final DataBlock block) { if (relativePosition == null) throw new N5IOException("Attempted to write block in the wrong shard."); - final ShardIndex idx = getIndex(); - - - //TODO Caleb: reusing the offset of a prior block write is only safe when writing the same amount, or less, data. - // This is not generally guaranteed, since we compress the data. - // Either need to known the compressed size before writing, append only, or only overwrite when not compressing - final long getBlockOffset = idx.getOffset(relativePosition); - final long startByte; - if (getBlockOffset == Shard.EMPTY_INDEX_NBYTES) { - final long indexByteOffset = idx.getByteOffset(); - startByte = indexByteOffset == -1 ? 0 : idx.getByteOffset(); - } else { - startByte = getBlockOffset; + final ShardIndex index = getIndex(); + long startByte = 0; + try { + startByte = keyValueAccess.size(path); + } catch (N5Exception.N5NoSuchKeyException e) { + startByte = index.getLocation() == ShardingCodec.IndexLocation.START ? index.numBytes() : 0; + } catch (IOException e) { + throw new N5IOException(e); } final long size = Long.MAX_VALUE - startByte; @@ -85,14 +83,18 @@ public void writeBlock(final DataBlock block) { DefaultBlockWriter.writeBlock(out, datasetAttributes, block); /* Update and write the index to the shard*/ - idx.set(startByte, out.getNumBytes(), relativePosition); - DefaultBlockWriter.writeBlock(out, datasetAttributes.getIndexAttributes(), idx); + index.set(startByte, out.getNumBytes(), relativePosition); } } } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to read block from " + path, e); + throw new N5IOException("Failed to write block to shard " + path, e); } + try { + ShardIndex.write(index, keyValueAccess, path); + } catch (IOException e) { + throw new N5IOException("Failed to write index to shard " + path, e); + } } @Override @@ -116,9 +118,9 @@ public ShardIndex createIndex() { public ShardIndex getIndex() { try { - final ShardIndex readIndex = ShardIndex.read(keyValueAccess, path, datasetAttributes); + final ShardIndex readIndex = ShardIndex.read(keyValueAccess, path, datasetAttributes.createIndex()); index = readIndex == null ? createIndex() : readIndex; - } catch (final NoSuchFileException e) { + } catch (final N5Exception.N5NoSuchKeyException e) { index = createIndex(); } catch (IOException e) { throw new N5IOException("Failed to read index at " + path, e); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java index 4bab3d45b..42cc14748 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -78,7 +78,7 @@ public void writeReadBlockTest() { new int[]{4, 4}, new int[]{2, 2}, DataType.UINT8, - new Codec[]{new N5BlockCodec(), new GzipCompression(4)}, + new Codec[]{new N5BlockCodec(ByteOrder.LITTLE_ENDIAN), new GzipCompression(4)}, new DeterministicSizeCodec[]{new BytesCodec(ByteOrder.BIG_ENDIAN), new Crc32cChecksumCodec()}, IndexLocation.END ); From f667bb53e91038953a62863e1caec453abedd4e8 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 19 Sep 2024 15:39:06 -0400 Subject: [PATCH 057/423] fix: ShardingCodec indexLocation should default to END --- .../java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index af06ec0e3..ebea15e63 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -43,7 +43,7 @@ public enum IndexLocation { @NameConfig.Parameter(INDEX_CODECS_KEY) private final DeterministicSizeCodec[] indexCodecs; - @NameConfig.Parameter(INDEX_LOCATION_KEY) + @NameConfig.Parameter(value = INDEX_LOCATION_KEY, optional = true) private final IndexLocation indexLocation; private ShardingCodec() { @@ -51,7 +51,7 @@ private ShardingCodec() { blockSize = null; codecs = null; indexCodecs = null; - indexLocation = null; + indexLocation = IndexLocation.END; } public ShardingCodec( From 66351079e151cb42c44c1610663d0d70a2db195f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 19 Sep 2024 15:44:10 -0400 Subject: [PATCH 058/423] style: ShardingCodec --- .../java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index ebea15e63..af1f23e0e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -25,7 +25,7 @@ public class ShardingCodec implements Codec.ArrayCodec { public static final String TYPE = "sharding_indexed"; - public final static String CHUNK_SHAPE_KEY = "chunk_shape"; + public static final String CHUNK_SHAPE_KEY = "chunk_shape"; public static final String INDEX_LOCATION_KEY = "index_location"; public static final String CODECS_KEY = "codecs"; public static final String INDEX_CODECS_KEY = "index_codecs"; @@ -46,6 +46,7 @@ public enum IndexLocation { @NameConfig.Parameter(value = INDEX_LOCATION_KEY, optional = true) private final IndexLocation indexLocation; + @SuppressWarnings("unused") private ShardingCodec() { blockSize = null; From 4489116ce5e5a003d2108b37436df565b4a4bb54 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 20 Sep 2024 11:34:50 -0400 Subject: [PATCH 059/423] fix: getBlockPositionInShard --- .../saalfeldlab/n5/ShardedDatasetAttributes.java | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index 794b88611..ff69e4dc7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -18,7 +18,6 @@ public class ShardedDatasetAttributes extends DatasetAttributes { private final ShardingCodec shardingCodec; - public ShardedDatasetAttributes ( final long[] dimensions, final int[] shardSize, //in pixels @@ -44,7 +43,7 @@ public ShardedDatasetAttributes( final int[] blockSize, //in pixels final DataType dataType, final ShardingCodec codec) { - super(dimensions, blockSize, dataType, null, codec.getCodecs()); + super(dimensions, blockSize, dataType, null, null); this.shardSize = shardSize; this.shardingCodec = codec; } @@ -131,23 +130,12 @@ public long[] getBlockPositionInShard(final long[] shardPosition, final long[] b return null; final int[] shardSize = getShardSize(); - final int[] blkSize = getBlockSize(); - final int[] blkGridSize = getBlocksPerShard(); -// final int[] shardSize = getSize(); -// final int[] blkSize = getBlockSize(); -// final int[] blkGridSize = getBlockGridSize(); - final long[] blockShardPos = new long[shardSize.length]; for (int i = 0; i < shardSize.length; i++) { - final long shardP = shardPos[i] * shardSize[i]; - final long blockP = blockPosition[i] * blkSize[i]; - blockShardPos[i] = (int)((blockP - shardP) / blkGridSize[i]); + blockShardPos[i] = blockPosition[i] % shardSize[i]; } return blockShardPos; - - - } /** From eb6de85b51c70b96f02c35218de8d3795262c8ed Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 20 Sep 2024 11:35:55 -0400 Subject: [PATCH 060/423] fix/wip: sharding codec block sizes needs reversing in zarr --- .../java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index af1f23e0e..0847f51d2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -11,6 +11,7 @@ import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.serialization.N5Annotations; import org.janelia.saalfeldlab.n5.serialization.NameConfig; import java.io.IOException; @@ -34,6 +35,7 @@ public enum IndexLocation { START, END; } + @N5Annotations.ReverseArray // TODO need to reverse for zarr, not for n5 @NameConfig.Parameter(CHUNK_SHAPE_KEY) private final int[] blockSize; From a8df678ff3e3e048bbc1f378ee1eff655b2a38ec Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 20 Sep 2024 11:36:55 -0400 Subject: [PATCH 061/423] feat: add getShardAttributes method to DatasetAttributes * and minor clean up --- .../saalfeldlab/n5/DatasetAttributes.java | 16 ++++++++++++++++ .../saalfeldlab/n5/shard/ShardingCodec.java | 3 +++ .../saalfeldlab/n5/shard/VirtualShard.java | 2 -- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 1ada17086..1cabbc3ac 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -148,6 +148,22 @@ public BytesCodec[] getCodecs() { return byteCodecs; } + public ShardedDatasetAttributes getShardAttributes() { + + if (getArrayCodec() instanceof ShardingCodec) { + + final ShardingCodec shardingCodec = (ShardingCodec)getArrayCodec(); + return new ShardedDatasetAttributes( + dimensions, + blockSize, + shardingCodec.getBlockSize(), + getDataType(), + shardingCodec); + + } else + return null; + } + public HashMap asMap() { final HashMap map = new HashMap<>(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 0847f51d2..cb65f1a4d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -86,6 +86,9 @@ public ArrayCodec getArrayCodec() { public BytesCodec[] getCodecs() { + if (codecs.length == 1) + return new BytesCodec[]{}; + final BytesCodec[] bytesCodecs = new BytesCodec[codecs.length - 1]; System.arraycopy(codecs, 1, bytesCodecs, 0, bytesCodecs.length); return bytesCodecs; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 1dd63f34d..270994322 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -1,11 +1,9 @@ package org.janelia.saalfeldlab.n5.shard; -import java.io.DataOutput; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; -import java.nio.file.NoSuchFileException; import java.util.Arrays; import org.janelia.saalfeldlab.n5.DataBlock; From 865c861a6ff268e143d46a5f61d8d0213b311f31 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 19 Nov 2024 11:17:21 -0500 Subject: [PATCH 062/423] wip: n5 exception and InMemoryShard --- .../janelia/saalfeldlab/n5/N5Exception.java | 27 +++++ .../org/janelia/saalfeldlab/n5/N5Writer.java | 4 +- .../n5/ShardedDatasetAttributes.java | 2 +- .../saalfeldlab/n5/shard/InMemoryShard.java | 105 +++++++++++++++++- .../janelia/saalfeldlab/n5/shard/Shard.java | 16 +++ .../n5/shard/ShardIndexBuilder.java | 67 +++++++++++ 6 files changed, 216 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java index 345a7cd03..7fbe0135e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java @@ -121,4 +121,31 @@ protected N5NoSuchKeyException( super(message, cause, enableSuppression, writableStackTrace); } } + + public static class N5ShardException extends N5IOException { + + public N5ShardException(final String message) { + + super(message); + } + + public N5ShardException(final String message, final Throwable cause) { + + super(message, cause); + } + + public N5ShardException(final Throwable cause) { + + super(cause); + } + + protected N5ShardException( + final String message, + final Throwable cause, + final boolean enableSuppression, + final boolean writableStackTrace) { + + super(message, cause, enableSuppression, writableStackTrace); + } + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 0c734e163..93c86bcfe 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -304,7 +304,7 @@ default void writeBlocks( * @param datasetPath * dataset path * @param datasetAttributes - * the dataset attributes + * the sharded dataset attributes * @param dataBlock * the data block * @param @@ -314,7 +314,7 @@ default void writeBlocks( */ void writeShard( final String datasetPath, - final DatasetAttributes datasetAttributes, + final ShardedDatasetAttributes datasetAttributes, final Shard shard) throws N5Exception; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index ff69e4dc7..ab6e0f9b8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -119,7 +119,7 @@ public long[] getShardPositionForBlock(final long... blockGridPosition) { } /** - * Returns of the block at the given position relative to this shard, or null if this shard does not contain the given block. + * Returns the block at the given position relative to this shard, or null if this shard does not contain the given block. * * @return the shard position */ diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index baacc8c58..e79bf5aca 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -1,14 +1,28 @@ package org.janelia.saalfeldlab.n5.shard; +import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.List; +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.commons.io.output.CountingOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DefaultBlockWriter; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; public class InMemoryShard extends AbstractShard { private List> blocks; + + private ShardIndexBuilder indexBuilder; + + /* + * TODO: + * Use morton- or c-ording instead of writing blocks out in the order they're added? + * (later) + */ public InMemoryShard(final ShardedDatasetAttributes datasetAttributes, final long[] gridPosition, ShardIndex index) { @@ -19,16 +33,103 @@ public InMemoryShard(final ShardedDatasetAttributes datasetAttributes, final lon @Override public void writeBlock(DataBlock block) { + + addBlock(block); + } + + public void addBlock(DataBlock block) { + + blocks.add(block); + } - // TODO Auto-generated method stub + public int numBlocks() { + return blocks.size(); + } + + public DataBlock getBlock(int i) { + + return blocks.get(i); } @Override public void writeShard() { - // TODO Auto-generated method stub + } + + public static void writeShard( + final OutputStream out, + InMemoryShard shard ) throws IOException { + + final ShardedDatasetAttributes datasetAttributes = shard.getDatasetAttributes(); + + if( shard.getIndex().getLocation() == IndexLocation.END) + writeShardEnd( out, shard); + else + writeShardStart( out, shard); + + } + + protected static void writeShardEnd( + final OutputStream out, + InMemoryShard shard ) throws IOException { + + final ShardedDatasetAttributes datasetAttributes = shard.getDatasetAttributes(); + + final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); + indexBuilder.indexLocation(IndexLocation.END); + + final CountingOutputStream cout = new CountingOutputStream(out); + + long offset = 0; + for (int i = 0; i < shard.numBlocks(); i++) { + + final DataBlock block = shard.getBlock(i); + DefaultBlockWriter.writeBlock(cout, datasetAttributes, block); + + indexBuilder.addBLock( block.getGridPosition(), offset); + offset = cout.getByteCount(); + } + + final ShardIndex index = indexBuilder.build(); + DefaultBlockWriter.writeBlock(out, datasetAttributes, index); + } + + protected static void writeShardStart( + final OutputStream out, + InMemoryShard shard ) throws IOException { + + final ShardedDatasetAttributes datasetAttributes = shard.getDatasetAttributes(); + final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); + indexBuilder.indexLocation(IndexLocation.START); + + long offset = 0; + final List blockData = new ArrayList<>(shard.numBlocks()); + for (int i = 0; i < shard.numBlocks(); i++) { + + final DataBlock block = shard.getBlock(i); + + final ByteArrayOutputStream os = new ByteArrayOutputStream(); + DefaultBlockWriter.writeBlock(os, datasetAttributes, block); + final byte[] data = os.toByteArray(); + + blockData.add(data); + indexBuilder.addBLock( block.getGridPosition(), offset); + offset += data.length; + } + + final ShardIndex index = indexBuilder.build(); + try { + DefaultBlockWriter.writeBlock(out, datasetAttributes, index); + + for( byte[] data : blockData ) + out.write(data); + + } catch (Exception e) { + e.printStackTrace(); + } } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 311343709..704e6c981 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -111,6 +111,22 @@ public static long flatIndex(long[] gridPosition, int[] gridSize) { } return index; } + + /** + * + * @param + * the type + * @param dataBlocks + * an array + * @return a shard containing the given blocks + */ + public static Shard fromDataBlocks( + final ShardedDatasetAttributes attributes, + final DataBlock[] dataBlocks) { + + // TODO implement me + return null; + } /** * Say we want async datablock access diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java new file mode 100644 index 000000000..35073c665 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java @@ -0,0 +1,67 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; + +public class ShardIndexBuilder { + + private final Shard shard; + + private final ShardIndex temporaryIndex; + + private IndexLocation location = IndexLocation.END; + + private DeterministicSizeCodec[] codecs; + + private long currentOffset = 0; + + public ShardIndexBuilder(Shard shard) { + + this.shard = shard; + this.temporaryIndex = new ShardIndex(shard.getBlockGridSize(), location); + } + + public ShardIndex build() { + + return new ShardIndex( + shard.getBlockGridSize(), + temporaryIndex.getData(), + location, + codecs); + } + + public ShardIndexBuilder indexLocation(IndexLocation location) { + + this.location = location; + return this; + } + + public ShardIndexBuilder setCodecs(DeterministicSizeCodec... codecs) { + + this.codecs = codecs; + return this; + } + + public ShardIndexBuilder addBLock(long[] blockPosition, long numBytes) { + + final long[] blockPositionInShard = shard.getDatasetAttributes().getBlockPositionInShard( + shard.getGridPosition(), + blockPosition); + + if (blockPositionInShard == null) { + throw new IllegalArgumentException(String.format( + "The block at position %s is not contained in the shard at position : %s and size : %s )", + Arrays.toString(blockPosition), + Arrays.toString(shard.getGridPosition()), + Arrays.toString(shard.getSize()))); + } + + temporaryIndex.set(currentOffset, numBytes, blockPositionInShard); + currentOffset += numBytes; + + return this; + } + +} From fccdb9abe4149f1a5175305a006e90e9c1aa7609 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 19 Nov 2024 11:39:59 -0500 Subject: [PATCH 063/423] wip: dummy impl of writeShard --- .../org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 60ce92997..7cf3518bc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -244,13 +244,10 @@ default void writeBlock( @Override default void writeShard( final String path, - final DatasetAttributes datasetAttributes, + final ShardedDatasetAttributes datasetAttributes, final Shard shard) throws N5Exception { - if (!(datasetAttributes instanceof ShardedDatasetAttributes)) - throw new N5IOException("Can not write shard into non-sharded dataset " + path); - - // TODO implement me + throw new N5Exception("not implemented"); } @Override From 19b8618251284654752286f68221c7ec88105ba5 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 16 Dec 2024 16:07:25 -0500 Subject: [PATCH 064/423] doc: getShardSize --- .../org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index ab6e0f9b8..dfb01e6f2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -62,6 +62,11 @@ public ShardingCodec getShardingCodec() { return shardingCodec.getCodecs(); } + /** + * The size of the blocks in pixel units. + * + * @return the number of pixels per dimension for this shard. + */ public int[] getShardSize() { return shardSize; From 3229fdd6c1bcae0984b6f702b5145c5115dc5ad8 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 16 Dec 2024 16:07:51 -0500 Subject: [PATCH 065/423] fix: ShardedDatasetAttributes.getBlockPosition --- .../org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index dfb01e6f2..0c04c372e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -134,7 +134,7 @@ public long[] getBlockPositionInShard(final long[] shardPosition, final long[] b if (!Arrays.equals(shardPosition, shardPos)) return null; - final int[] shardSize = getShardSize(); + final int[] shardSize = getBlocksPerShard(); final long[] blockShardPos = new long[shardSize.length]; for (int i = 0; i < shardSize.length; i++) { blockShardPos[i] = blockPosition[i] % shardSize[i]; From b270ece22224cbe326f0ea1a7f0140a05b4b4d9f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 18 Dec 2024 13:17:36 -0500 Subject: [PATCH 066/423] perf: override writeData * (Short/Float/Double)DataBlock --- .../org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java | 8 ++++++++ .../org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java | 8 ++++++++ .../org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 8cbb15117..0240e6fae 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.io.DataInput; +import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; @@ -57,6 +58,13 @@ public void readData(final DataInput inputStream) throws IOException { data[i] = inputStream.readDouble(); } + @Override + public void writeData(final DataOutput output) throws IOException { + + for (int i = 0; i < data.length; i++) + output.writeDouble(data[i]); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index aa97ce3f5..a2bc2c696 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.io.DataInput; +import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; @@ -57,6 +58,13 @@ public void readData(final DataInput inputStream) throws IOException { data[i] = inputStream.readFloat(); } + @Override + public void writeData(final DataOutput output) throws IOException { + + for (int i = 0; i < data.length; i++) + output.writeFloat(data[i]); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 34c5a883d..c7d141f39 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.io.DataInput; +import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; @@ -57,6 +58,13 @@ public void readData(final DataInput dataInput) throws IOException { data[i] = dataInput.readShort(); } + @Override + public void writeData(final DataOutput output) throws IOException { + + for (int i = 0; i < data.length; i++) + output.writeShort(data[i]); + } + @Override public int getNumElements() { From f46aa524c230863194c3423235c31a4cc90f5f3b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 18 Dec 2024 13:20:21 -0500 Subject: [PATCH 067/423] chore: bump pom-scijva to 40.0.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9ea763f0d..cab565a7b 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.scijava pom-scijava - 38.0.1 + 40.0.0 From 362c74d21e944f26eff70418c9b710a43b4888f7 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 20 Dec 2024 10:27:51 -0500 Subject: [PATCH 068/423] perf: initialize cache only if using it --- .../java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index edbc6947e..dd4b9cc24 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -123,7 +123,11 @@ protected N5KeyValueReader( this.keyValueAccess = keyValueAccess; this.gson = GsonUtils.registerGson(gsonBuilder); this.cacheMeta = cacheMeta; - this.cache = newCache(); + + if (this.cacheMeta) + this.cache = newCache(); + else + this.cache = null; try { uri = keyValueAccess.uri(basePath); From edbdef6cda0a781e601d975ce72442b3583426f8 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 23 Dec 2024 08:34:56 -0500 Subject: [PATCH 069/423] fix: make removeAttribute methods' behavior more consistent * this change allows re-use of this method by zarr v3 --- .../java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 7cf3518bc..3039f79ae 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -196,7 +196,7 @@ default T removeAttribute(final String pathName, final String key, final Cla throw new N5Exception.N5ClassCastException(e); } if (obj != null) { - writeAttributes(normalPath, attributes); + setAttributes(normalPath, attributes); } return obj; } From 1b672ded40616286244bd772ae6d49a19cc9bc9b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 23 Dec 2024 09:04:27 -0500 Subject: [PATCH 070/423] fix: gzip make uzeZlib parameter optional * e.g. not specified in zarrs written by tensorstore --- src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index 3091ad284..b03a4d930 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -54,7 +54,7 @@ public class GzipCompression implements DefaultBlockReader, DefaultBlockWriter, private final int level; @CompressionParameter - @NameConfig.Parameter + @NameConfig.Parameter(optional = true) private final boolean useZlib; private final transient GzipParameters parameters = new GzipParameters(); From bec29965b77a6981c42848c8eb578ada7883f904 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 23 Dec 2024 11:44:26 -0500 Subject: [PATCH 071/423] fix: BlockWriter should not close stream * rather, should be closed where it is opened * this change enables the stream to be re-used by multiple block writers, e.g. during sharding --- src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index fd7450ef2..badf9f245 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -82,7 +82,6 @@ public static void writeBlock( stream = codec.encode(stream); dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); - stream.close(); } public static void writeFromStream(final DataBlock dataBlock, final OutputStream out) throws IOException { From 2c753a1a0b191fc1221693055eb7d65b130a0589 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 2 Jan 2025 15:01:10 -0500 Subject: [PATCH 072/423] Revert "fix: BlockWriter should not close stream" This reverts commit bec29965b77a6981c42848c8eb578ada7883f904. --- src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index badf9f245..fd7450ef2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -82,6 +82,7 @@ public static void writeBlock( stream = codec.encode(stream); dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); + stream.close(); } public static void writeFromStream(final DataBlock dataBlock, final OutputStream out) throws IOException { From 275aaa61d4a45f1c1d7978de645ad9a369f4b14a Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 2 Jan 2025 15:34:05 -0500 Subject: [PATCH 073/423] feat(test): parameterize ShardDemo read/write test; add new test --- .../saalfeldlab/n5/shard/ShardDemos.java | 53 +++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java index 42cc14748..f06e9dae4 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -1,12 +1,15 @@ package org.janelia.saalfeldlab.n5.shard; import com.google.gson.GsonBuilder; +import org.janelia.saalfeldlab.n5.Bzip2Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.saalfeldlab.n5.Lz4Compression; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.XzCompression; import org.janelia.saalfeldlab.n5.codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; @@ -16,14 +19,21 @@ import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.net.MalformedURLException; import java.nio.ByteOrder; import java.nio.file.FileSystems; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +@RunWith(Parameterized.class) public class ShardDemos { public static void main(String[] args) throws MalformedURLException { @@ -62,6 +72,30 @@ public static void main(String[] args) throws MalformedURLException { System.out.println(Arrays.toString(dataReRead)); } + @Parameterized.Parameters(name = "IndexLocation({0}), Block ByteOrder({1}), Index ByteOrder({2})") + public static Collection data() { + final ArrayList params = new ArrayList<>(); + for (IndexLocation indexLoc : IndexLocation.values()) { + for (ByteOrder blockByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { + for (ByteOrder indexByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { + params.add(new Object[]{indexLoc, blockByteOrder, indexByteOrder}); + } + } + } + final Object[][] paramArray = new Object[params.size()][]; + Arrays.setAll(paramArray, params::get); + return Arrays.asList(paramArray); + } + + @Parameterized.Parameter() + public IndexLocation indexLocation; + + @Parameterized.Parameter(1) + public ByteOrder dataByteOrder; + + @Parameterized.Parameter(2) + public ByteOrder indexByteOrder; + @Test public void writeReadBlockTest() { @@ -78,9 +112,9 @@ public void writeReadBlockTest() { new int[]{4, 4}, new int[]{2, 2}, DataType.UINT8, - new Codec[]{new N5BlockCodec(ByteOrder.LITTLE_ENDIAN), new GzipCompression(4)}, - new DeterministicSizeCodec[]{new BytesCodec(ByteOrder.BIG_ENDIAN), new Crc32cChecksumCodec()}, - IndexLocation.END + new Codec[]{new N5BlockCodec(dataByteOrder), new GzipCompression(4)}, + new DeterministicSizeCodec[]{new BytesCodec(indexByteOrder), new Crc32cChecksumCodec()}, + indexLocation ); writer.createDataset("shard", datasetAttributes); writer.deleteBlock("shard", 0, 0); @@ -89,9 +123,10 @@ public void writeReadBlockTest() { final DataType dataType = datasetAttributes.getDataType(); final int numElements = 2 * 2; + final HashMap writtenBlocks = new HashMap<>(); + for (int idx1 = 1; idx1 >= 0; idx1--) { for (int idx2 = 1; idx2 >= 0; idx2--) { - final long[] gridPosition = {idx1, idx2}; final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(blockSize, gridPosition, numElements); byte[] data = dataBlock.getData(); @@ -101,8 +136,16 @@ public void writeReadBlockTest() { writer.writeBlock("shard", datasetAttributes, dataBlock); final DataBlock block = (DataBlock)writer.readBlock("shard", datasetAttributes, gridPosition); - Assert.assertArrayEquals("Read from shard doesn't match", data, block.getData()); + + for (Map.Entry entry : writtenBlocks.entrySet()) { + final long[] otherGridPosition = entry.getKey(); + final byte[] otherData = entry.getValue(); + final DataBlock otherBlock = (DataBlock)writer.readBlock("shard", datasetAttributes, otherGridPosition); + Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, otherBlock.getData()); + } + + writtenBlocks.put(gridPosition, data); } } } From 9183d11f72971130cf526d9b61c03b40393f6bfb Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 2 Jan 2025 15:42:50 -0500 Subject: [PATCH 074/423] feat(wip): toward an implementation of writeShard --- .../saalfeldlab/n5/DatasetAttributes.java | 2 +- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 4 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 19 +++- .../org/janelia/saalfeldlab/n5/N5Writer.java | 26 ++--- .../n5/ShardedDatasetAttributes.java | 50 ++++++++- .../saalfeldlab/n5/shard/InMemoryShard.java | 100 +++++++++++++----- .../janelia/saalfeldlab/n5/shard/Shard.java | 58 +++++++--- .../saalfeldlab/n5/shard/ShardIndex.java | 7 +- .../n5/shard/ShardIndexBuilder.java | 18 +++- .../saalfeldlab/n5/shard/ShardWriter.java | 35 ++---- .../saalfeldlab/n5/shard/VirtualShard.java | 15 +-- .../saalfeldlab/n5/util/GridIterator.java | 98 +++++++++++++++++ 12 files changed, 330 insertions(+), 102 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 1cabbc3ac..04ec1b35b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -221,7 +221,7 @@ private static Compression getCompressionVersion0(final String compressionVersio return null; } - private Codec[] concatenateCodecs() { + protected Codec[] concatenateCodecs() { final Codec[] allCodecs = new Codec[byteCodecs.length + 1]; allCodecs[0] = arrayCodec; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index e6a3429f1..288136ba5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -103,8 +103,8 @@ default DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { - if (datasetAttributes instanceof ShardedDatasetAttributes) { - final ShardedDatasetAttributes shardedAttrs = (ShardedDatasetAttributes)datasetAttributes; + final ShardedDatasetAttributes shardedAttrs = datasetAttributes.getShardAttributes(); + if (shardedAttrs != null) { final long[] shardPosition = shardedAttrs.getShardPositionForBlock(gridPosition); final Shard shard = getShard(pathName, shardedAttrs, shardPosition); return shard.getBlock(gridPosition); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 3039f79ae..ee5ab2039 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -33,6 +33,7 @@ import java.util.Map; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.shard.InMemoryShard; import org.janelia.saalfeldlab.n5.shard.Shard; import com.google.gson.Gson; @@ -231,7 +232,7 @@ default void writeBlock( final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition()); try (final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath)) { - try ( final OutputStream out = lock.newOutputStream()) { + try (final OutputStream out = lock.newOutputStream()) { DefaultBlockWriter.writeBlock(out, datasetAttributes, dataBlock); } } catch (final IOException | UncheckedIOException e) { @@ -244,10 +245,22 @@ default void writeBlock( @Override default void writeShard( final String path, - final ShardedDatasetAttributes datasetAttributes, + final DatasetAttributes datasetAttributes, final Shard shard) throws N5Exception { - throw new N5Exception("not implemented"); + if( datasetAttributes.getShardAttributes() == null ) + throw new N5IOException("Tried to write shard into a not-sharded dataset: " + path); + + final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shard.getGridPosition()); + try (final LockedChannel lock = getKeyValueAccess().lockForWriting(shardPath)) { + try (final OutputStream out = lock.newOutputStream()) { + InMemoryShard.fromShard(shard).write(out); + out.close(); + } + } catch (final IOException | UncheckedIOException e) { + throw new N5IOException( + "Failed to write shard " + Arrays.toString(shard.getGridPosition()) + " into dataset " + path, e); + } } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 93c86bcfe..b0ed462f5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -295,26 +295,26 @@ default void writeBlocks( final String datasetPath, final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { - //TODO Caleb: write this + + // TODO Caleb: write this + + // default method is naive + for (DataBlock block : dataBlocks) + writeBlock(datasetPath, datasetAttributes, block); } /** - * Writes a complete {@link Shard} to a dataset. + * Writes a {@link Shard}. * - * @param datasetPath - * dataset path - * @param datasetAttributes - * the sharded dataset attributes - * @param dataBlock - * the data block - * @param - * the data block data type - * @throws N5Exception - * if the requested dataset is not sharded + * @param datasetPath dataset path + * @param datasetAttributes the dataset attributes + * @param shard the shard + * @param the data block data type + * @throws N5Exception the exception */ void writeShard( final String datasetPath, - final ShardedDatasetAttributes datasetAttributes, + final DatasetAttributes datasetAttributes, final Shard shard) throws N5Exception; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index 0c04c372e..c8d3ee6ab 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -48,6 +48,10 @@ public ShardedDatasetAttributes( this.shardingCodec = codec; } + public ShardedDatasetAttributes getShardAttributes() { + return this; + } + public ShardingCodec getShardingCodec() { return shardingCodec; } @@ -62,6 +66,12 @@ public ShardingCodec getShardingCodec() { return shardingCodec.getCodecs(); } + @Override + protected Codec[] concatenateCodecs() { + + return new Codec[] { shardingCodec }; + } + /** * The size of the blocks in pixel units. * @@ -126,10 +136,11 @@ public long[] getShardPositionForBlock(final long... blockGridPosition) { /** * Returns the block at the given position relative to this shard, or null if this shard does not contain the given block. * - * @return the shard position + * @return the block position */ public long[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { + // TODO check correctness final long[] shardPos = getShardPositionForBlock(blockPosition); if (!Arrays.equals(shardPosition, shardPos)) return null; @@ -143,6 +154,43 @@ public long[] getBlockPositionInShard(final long[] shardPosition, final long[] b return blockShardPos; } + /** + * Given a block's position relative to a shard, returns its position in pixels + * relative to the image. + * + * @return the block position + */ + public long[] getBlockMinFromShardPosition(final long[] shardPosition, final long[] blockPosition) { + + // is this useful? + final int[] blockSize = getBlockSize(); + final int[] shardSize = getShardSize(); + final long[] blockImagePos = new long[shardSize.length]; + for (int i = 0; i < shardSize.length; i++) { + blockImagePos[i] = (shardPosition[i] * shardSize[i]) + (blockPosition[i] * blockSize[i]); + } + + return blockImagePos; + } + + /** + * Given a block's position relative to a shard, returns its position relative + * to the image. + * + * @return the block position + */ + public long[] getBlockPositionFromShardPosition(final long[] shardPosition, final long[] blockPosition) { + + // is this useful? + final int[] shardBlockSize = getBlocksPerShard(); + final long[] blockImagePos = new long[shardSize.length]; + for (int i = 0; i < shardSize.length; i++) { + blockImagePos[i] = (shardPosition[i] * shardBlockSize[i]) + (blockPosition[i]); + } + + return blockImagePos; + } + /** * @return the number of blocks per shard */ diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index e79bf5aca..a3d9d1680 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.commons.io.output.ByteArrayOutputStream; @@ -24,6 +25,13 @@ public class InMemoryShard extends AbstractShard { * (later) */ + public InMemoryShard(final ShardedDatasetAttributes datasetAttributes, final long[] gridPosition) { + + this( datasetAttributes, gridPosition, null); + indexBuilder = new ShardIndexBuilder(this); + indexBuilder.indexLocation(datasetAttributes.getIndexLocation()); + } + public InMemoryShard(final ShardedDatasetAttributes datasetAttributes, final long[] gridPosition, ShardIndex index) { @@ -38,7 +46,7 @@ public void writeBlock(DataBlock block) { } public void addBlock(DataBlock block) { - + blocks.add(block); } @@ -52,25 +60,49 @@ public DataBlock getBlock(int i) { return blocks.get(i); } + protected IndexLocation indexLocation() { + + if (index != null) + return index.getLocation(); + else + return indexBuilder.getLocation(); + } + @Override - public void writeShard() { + public ShardIndex getIndex() { + if( index != null ) + return index; + else + return indexBuilder.build(); } - - public static void writeShard( - final OutputStream out, - InMemoryShard shard ) throws IOException { - final ShardedDatasetAttributes datasetAttributes = shard.getDatasetAttributes(); - - if( shard.getIndex().getLocation() == IndexLocation.END) - writeShardEnd( out, shard); + public void write(final OutputStream out) throws IOException { + + if (indexLocation() == IndexLocation.END) + writeShardEnd(out, this); else - writeShardStart( out, shard); + writeShardStart(out, this); + } + public static void writeShard(final OutputStream out, final Shard shard) throws IOException { + + fromShard(shard).write(out); } - protected static void writeShardEnd( + public static InMemoryShard fromShard(Shard shard) { + + if (shard instanceof InMemoryShard) + return (InMemoryShard) shard; + + final InMemoryShard inMemoryShard = new InMemoryShard(shard.getDatasetAttributes(), + shard.getGridPosition()); + + shard.forEach(blk -> inMemoryShard.addBlock(blk)); + return inMemoryShard; + } + + protected static void writeShardEndStream( final OutputStream out, InMemoryShard shard ) throws IOException { @@ -87,7 +119,7 @@ protected static void writeShardEnd( final DataBlock block = shard.getBlock(i); DefaultBlockWriter.writeBlock(cout, datasetAttributes, block); - indexBuilder.addBLock( block.getGridPosition(), offset); + indexBuilder.addBlock( block.getGridPosition(), offset); offset = cout.getByteCount(); } @@ -95,6 +127,29 @@ protected static void writeShardEnd( DefaultBlockWriter.writeBlock(out, datasetAttributes, index); } + protected static void writeShardEnd( + final OutputStream out, + InMemoryShard shard ) throws IOException { + + final ShardedDatasetAttributes datasetAttributes = shard.getDatasetAttributes(); + + final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); + indexBuilder.indexLocation(IndexLocation.END); + indexBuilder.setCodecs(datasetAttributes.getShardingCodec().getIndexCodecs()); + + for (int i = 0; i < shard.numBlocks(); i++) { + + final ByteArrayOutputStream os = new ByteArrayOutputStream(); + final DataBlock block = shard.getBlock(i); + DefaultBlockWriter.writeBlock(os, datasetAttributes, block); + + indexBuilder.addBlock(block.getGridPosition(), os.size()); + out.write(os.toByteArray()); + } + + ShardIndex.write(indexBuilder.build(), out); + } + protected static void writeShardStart( final OutputStream out, InMemoryShard shard ) throws IOException { @@ -102,25 +157,23 @@ protected static void writeShardStart( final ShardedDatasetAttributes datasetAttributes = shard.getDatasetAttributes(); final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); indexBuilder.indexLocation(IndexLocation.START); + indexBuilder.setCodecs(datasetAttributes.getShardingCodec().getIndexCodecs()); - long offset = 0; final List blockData = new ArrayList<>(shard.numBlocks()); for (int i = 0; i < shard.numBlocks(); i++) { - final DataBlock block = shard.getBlock(i); - final ByteArrayOutputStream os = new ByteArrayOutputStream(); + final DataBlock block = shard.getBlock(i); DefaultBlockWriter.writeBlock(os, datasetAttributes, block); - final byte[] data = os.toByteArray(); - blockData.add(data); - indexBuilder.addBLock( block.getGridPosition(), offset); - offset += data.length; + blockData.add(os.toByteArray()); + indexBuilder.addBlock(block.getGridPosition(), os.size()); } - - final ShardIndex index = indexBuilder.build(); + try { - DefaultBlockWriter.writeBlock(out, datasetAttributes, index); + final ByteArrayOutputStream os = new ByteArrayOutputStream(); + ShardIndex.write(indexBuilder.build(), os); + out.write(os.toByteArray()); for( byte[] data : blockData ) out.write(data); @@ -131,5 +184,4 @@ protected static void writeShardStart( } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 704e6c981..abaf7e164 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -1,11 +1,13 @@ package org.janelia.saalfeldlab.n5.shard; import java.util.Arrays; +import java.util.Iterator; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.util.GridIterator; -public interface Shard { +public interface Shard extends Iterable> { long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; @@ -63,6 +65,21 @@ default long[] getBlockPosition(long... blockPosition) { final long[] shardPos = getDatasetAttributes().getShardPositionForBlock(blockPosition); return getDatasetAttributes().getBlockPositionInShard(shardPos, blockPosition); } + + /** + * Returns the position in pixels of the + * + * @return the min + */ + default long[] getShardMinPosition(long... shardPosition) { + + final int[] shardSize = getSize(); + final long[] shardMin = new long[shardSize.length]; + for (int i = 0; i < shardSize.length; i++) { + shardMin[i] = shardPosition[i] * shardSize[i]; + } + return shardMin; + } /** * Returns the position of the shard containing the block with the given block position. @@ -84,7 +101,10 @@ default long[] getShard(long... blockPosition) { public void writeBlock(DataBlock block); - public void writeShard(); + default Iterator> iterator() { + + return new DataBlockIterator(this); + } default DataBlock[] getAllBlocks(long... position) { //TODO Caleb: Do we want this? @@ -111,21 +131,27 @@ public static long flatIndex(long[] gridPosition, int[] gridSize) { } return index; } - - /** - * - * @param - * the type - * @param dataBlocks - * an array - * @return a shard containing the given blocks - */ - public static Shard fromDataBlocks( - final ShardedDatasetAttributes attributes, - final DataBlock[] dataBlocks) { - // TODO implement me - return null; + public static class DataBlockIterator implements Iterator> { + + private final GridIterator it; + private final Shard shard; + + public DataBlockIterator(final Shard shard) { + + this.shard = shard; + it = new GridIterator(shard.getBlockGridSize()); + } + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public DataBlock next() { + return shard.getBlock(it.next()); + } } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 1c840b1ea..b400b8639 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -144,13 +144,18 @@ public static void write( final long start = index.location == IndexLocation.START ? 0 : keyValueAccess.size(key); try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(key, start, index.numBytes())) { try (final OutputStream os = lockedChannel.newOutputStream()) { - DefaultBlockWriter.writeBlock(os, index.getIndexAttributes(), index); + write(index, os); } } catch (final IOException | UncheckedIOException e) { throw new N5IOException("Failed to write shard index to " + key, e); } } + public static void write(final ShardIndex index, OutputStream out) throws IOException { + + DefaultBlockWriter.writeBlock(out, index.getIndexAttributes(), index); + } + private DatasetAttributes getIndexAttributes() { final DatasetAttributes indexAttributes = diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java index 35073c665..7a7480f86 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java @@ -9,7 +9,7 @@ public class ShardIndexBuilder { private final Shard shard; - private final ShardIndex temporaryIndex; + private ShardIndex temporaryIndex; private IndexLocation location = IndexLocation.END; @@ -35,16 +35,30 @@ public ShardIndex build() { public ShardIndexBuilder indexLocation(IndexLocation location) { this.location = location; + this.temporaryIndex = new ShardIndex(shard.getBlockGridSize(), location); + + if (location == IndexLocation.END) + currentOffset = 0; + else + currentOffset = temporaryIndex.numBytes(); + return this; } + public IndexLocation getLocation() { + + return this.location; + } + public ShardIndexBuilder setCodecs(DeterministicSizeCodec... codecs) { this.codecs = codecs; + final ShardIndex newIndex = new ShardIndex(temporaryIndex.getSize(), temporaryIndex.getLocation(), codecs); + this.temporaryIndex = newIndex; return this; } - public ShardIndexBuilder addBLock(long[] blockPosition, long numBytes) { + public ShardIndexBuilder addBlock(long[] blockPosition, long numBytes) { final long[] blockPositionInShard = shard.getDatasetAttributes().getBlockPositionInShard( shard.getGridPosition(), diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java index 4a288878a..792019fdc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java @@ -19,7 +19,7 @@ public class ShardWriter { private final List> blocks; - private final ShardedDatasetAttributes datasetAttributes; + private ShardedDatasetAttributes attributes; private ByteBuffer blockSizes; @@ -32,15 +32,14 @@ public class ShardWriter { public ShardWriter(final ShardedDatasetAttributes datasetAttributes) { blocks = new ArrayList<>(); - this.datasetAttributes = datasetAttributes; + attributes = datasetAttributes; } public void reset() { blocks.clear(); - blockSizes = null; blockBytes.clear(); - + blockSizes = null; indexData = null; } @@ -49,21 +48,12 @@ public void addBlock(final DataBlock block) { blocks.add(block); } - public void write(final OutputStream out) throws IOException { - - // TODO need codecs - - // prepareForWriting(); - // if (datasetAttributes.getShardingConfiguration().getIndexLocation()) { - // writeIndexes(out); - // writeBlocks(out); - // } else { - // writeBlocks(out); - // writeIndexes(out); - // } + public void write(final Shard shard, final OutputStream out) throws IOException { + + attributes = shard.getDatasetAttributes(); prepareForWritingDataBlock(); - if (datasetAttributes.getIndexLocation() == ShardingCodec.IndexLocation.START) { + if (attributes.getIndexLocation() == ShardingCodec.IndexLocation.START) { writeIndexBlock(out); writeBlocks(out); } else { @@ -77,14 +67,14 @@ private void prepareForWritingDataBlock() throws IOException { // final ShardingProperties shardProps = new ShardingProperties(datasetAttributes); // indexData = new ShardIndexDataBlock(shardProps.getIndexDimensions()); - indexData = datasetAttributes.createIndex(); + indexData = attributes.createIndex(); blockBytes = new ArrayList<>(); long cumulativeBytes = 0; final long[] shardPosition = new long[1]; for (int i = 0; i < blocks.size(); i++) { try (final ByteArrayOutputStream blockOut = new ByteArrayOutputStream()) { - DefaultBlockWriter.writeBlock(blockOut, datasetAttributes, blocks.get(i)); + DefaultBlockWriter.writeBlock(blockOut, attributes, blocks.get(i)); System.out.println(String.format("block %d is %d bytes", i, blockOut.size())); shardPosition[0] = i; @@ -108,7 +98,7 @@ private void prepareForWriting() throws IOException { try (final ByteArrayOutputStream blockOut = new ByteArrayOutputStream()) { - DefaultBlockWriter.writeBlock(blockOut, datasetAttributes, blocks.get(i)); + DefaultBlockWriter.writeBlock(blockOut, attributes, blocks.get(i)); System.out.println(String.format("block %d is %d bytes", i, blockOut.size())); blockIndexes.putLong(cumulativeBytes); @@ -126,11 +116,6 @@ private void writeBlocks(final OutputStream out) throws IOException { out.write(bytes); } - private void writeIndexes(final OutputStream out) throws IOException { - - out.write(blockSizes.array()); - } - private void writeIndexBlock(final OutputStream out) throws IOException { final DataOutputStream dos = new DataOutputStream(out); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 270994322..1a7698874 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -4,10 +4,8 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; -import java.util.Arrays; import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.DefaultBlockReader; import org.janelia.saalfeldlab.n5.DefaultBlockWriter; import org.janelia.saalfeldlab.n5.KeyValueAccess; @@ -39,12 +37,12 @@ public DataBlock getBlock(long... blockGridPosition) { final ShardIndex idx = getIndex(); - final long startByte = idx.getOffset(relativePosition); if (startByte == Shard.EMPTY_INDEX_NBYTES ) return null; + System.out.println("read from start: " + startByte); final long size = idx.getNumBytes(relativePosition); try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path, startByte, size)) { try ( final InputStream channelIn = lockedChannel.newInputStream()) { @@ -95,17 +93,6 @@ public void writeBlock(final DataBlock block) { } } - @Override - public void writeShard() { - - // TODO - } - - private static int numBlockElements(DatasetAttributes datasetAttributes) { - - return Arrays.stream(datasetAttributes.getBlockSize()).reduce(1, (x, y) -> x * y); - } - public ShardIndex createIndex() { // Empty index of the correct size diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java new file mode 100644 index 000000000..1fcb118cc --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java @@ -0,0 +1,98 @@ +package org.janelia.saalfeldlab.n5.util; + +import java.util.Iterator; + +/** + * Essentially imglib2's IntervalIterator, but N5 does not depend on imglib2. + */ +public class GridIterator implements Iterator { + + final protected long[] dimensions; + + final protected long[] steps; + + final protected long[] position; + + final protected int lastIndex; + + protected int index = -1; + + public GridIterator(final long[] dimensions) { + + final int n = dimensions.length; + this.dimensions = new long[n]; + this.position = new long[n]; + steps = new long[n]; + + final int m = n - 1; + long k = steps[0] = 1; + for (int d = 0; d < m;) { + final long dimd = dimensions[d]; + this.dimensions[d] = dimd; + k *= dimd; + steps[++d] = k; + } + final long dimm = dimensions[m]; + this.dimensions[m] = dimm; + lastIndex = (int)(k * dimm - 1); + } + + public GridIterator(final int[] dimensions) { + + this(int2long(dimensions)); + } + + public void fwd() { + ++index; + } + + public void reset() { + index = -1; + } + + @Override + public boolean hasNext() { + return index < lastIndex; + } + + @Override + public long[] next() { + fwd(); + indexToPosition(index, dimensions, position); + return position; + } + + public int getIndex() { + return index; + } + + final static public void indexToPosition(long index, final long[] dimensions, final long[] position) { + final int maxDim = dimensions.length - 1; + for (int d = 0; d < maxDim; ++d) { + final long j = index / dimensions[d]; + position[d] = index - j * dimensions[d]; + index = j; + } + position[maxDim] = index; + + } + + final static public int[] long2int(final long[] a) { + final int[] i = new int[a.length]; + + for (int d = 0; d < a.length; ++d) + i[d] = (int) a[d]; + + return i; + } + + final static public long[] int2long(final int[] i) { + final long[] l = new long[i.length]; + + for (int d = 0; d < l.length; ++d) + l[d] = i[d]; + + return l; + } + +} From d4142c5fe1df9091342d8e5e14a32fb9c69669ae Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 2 Jan 2025 15:43:09 -0500 Subject: [PATCH 075/423] chore: stop using deprecated BoundedInputStream constructor --- .../org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 317c2c5de..05afb2d1b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -163,7 +163,7 @@ public Writer newWriter() throws IOException { @Override public InputStream newInputStream() throws IOException { - return new BoundedInputStream(Channels.newInputStream(channel), len); + return BoundedInputStream.builder().setInputStream(Channels.newInputStream(channel)).setMaxCount(len).get(); } @Override From bc5d103662c0d993e78bb589db34ae94c866d400 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 2 Jan 2025 15:51:06 -0500 Subject: [PATCH 076/423] fix: be quiet --- src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 1a7698874..7765d42be 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -42,7 +42,6 @@ public DataBlock getBlock(long... blockGridPosition) { if (startByte == Shard.EMPTY_INDEX_NBYTES ) return null; - System.out.println("read from start: " + startByte); final long size = idx.getNumBytes(relativePosition); try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path, startByte, size)) { try ( final InputStream channelIn = lockedChannel.newInputStream()) { From 1c9e6018ae9de65e40fed328d03519f75802e087 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 2 Jan 2025 15:58:57 -0500 Subject: [PATCH 077/423] fix(test): failing on github actions --- src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java index f911ec6d2..5d58657d3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java @@ -43,6 +43,7 @@ public void testSerialization() { new IdentityCodec() } ); + writer.createGroup("shard"); //Should already exist, but this will ensure. writer.setAttribute("shard", "/", datasetAttributes); final DatasetAttributes deserialized = writer.getAttribute("shard", "/", DatasetAttributes.class); From b62dc00db0805ad4be3297eac94eb1fbbb7ef3ad Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 2 Jan 2025 16:31:59 -0500 Subject: [PATCH 078/423] fix/test: add writeShardTest * fix ShardIndexBuilder --- .../n5/shard/ShardIndexBuilder.java | 19 +++--- .../saalfeldlab/n5/shard/ShardDemos.java | 58 ++++++++++++++++++- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java index 7a7480f86..c8511dc8e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java @@ -36,12 +36,7 @@ public ShardIndexBuilder indexLocation(IndexLocation location) { this.location = location; this.temporaryIndex = new ShardIndex(shard.getBlockGridSize(), location); - - if (location == IndexLocation.END) - currentOffset = 0; - else - currentOffset = temporaryIndex.numBytes(); - + updateInitialOffset(); return this; } @@ -53,8 +48,9 @@ public IndexLocation getLocation() { public ShardIndexBuilder setCodecs(DeterministicSizeCodec... codecs) { this.codecs = codecs; - final ShardIndex newIndex = new ShardIndex(temporaryIndex.getSize(), temporaryIndex.getLocation(), codecs); + final ShardIndex newIndex = new ShardIndex(shard.getBlockGridSize(), temporaryIndex.getLocation(), codecs); this.temporaryIndex = newIndex; + updateInitialOffset(); return this; } @@ -78,4 +74,13 @@ public ShardIndexBuilder addBlock(long[] blockPosition, long numBytes) { return this; } + private void updateInitialOffset() { + + if (location == IndexLocation.END) + currentOffset = 0; + else + currentOffset = temporaryIndex.numBytes(); + + } + } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java index f06e9dae4..ee4a84118 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -1,15 +1,12 @@ package org.janelia.saalfeldlab.n5.shard; import com.google.gson.GsonBuilder; -import org.janelia.saalfeldlab.n5.Bzip2Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.GzipCompression; -import org.janelia.saalfeldlab.n5.Lz4Compression; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; -import org.janelia.saalfeldlab.n5.XzCompression; import org.janelia.saalfeldlab.n5.codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; @@ -150,4 +147,59 @@ public void writeReadBlockTest() { } } + @Test + public void writeReadShardTest() { + + final N5Factory factory = new N5Factory(); + final GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.setPrettyPrinting(); + factory.gsonBuilder(gsonBuilder); + factory.cacheAttributes(false); + + final N5Writer writer = factory.openWriter("src/test/resources/shardExamples/test.n5"); + + final ShardedDatasetAttributes datasetAttributes = new ShardedDatasetAttributes( + new long[]{4, 4}, + new int[]{4, 4}, + new int[]{2, 2}, + DataType.UINT8, + new Codec[]{new N5BlockCodec(dataByteOrder)}, + new DeterministicSizeCodec[]{new BytesCodec(indexByteOrder), new Crc32cChecksumCodec()}, + indexLocation + ); + writer.createDataset("wholeShard", datasetAttributes); + writer.deleteBlock("wholeShard", 0, 0); + + final int[] blockSize = datasetAttributes.getBlockSize(); + final DataType dataType = datasetAttributes.getDataType(); + final int numElements = 2 * 2; + + final HashMap writtenBlocks = new HashMap<>(); + + final InMemoryShard shard = new InMemoryShard(datasetAttributes, new long[]{0, 0}); + + for (int idx1 = 1; idx1 >= 0; idx1--) { + for (int idx2 = 1; idx2 >= 0; idx2--) { + final long[] gridPosition = {idx1, idx2}; + final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(blockSize, gridPosition, numElements); + byte[] data = dataBlock.getData(); + for (int i = 0; i < data.length; i++) { + data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); + } + + shard.addBlock(dataBlock); + writtenBlocks.put(gridPosition, data); + } + } + + writer.writeShard("wholeShard", datasetAttributes, shard); + + for (Map.Entry entry : writtenBlocks.entrySet()) { + final long[] otherGridPosition = entry.getKey(); + final byte[] otherData = entry.getValue(); + final DataBlock otherBlock = (DataBlock)writer.readBlock("wholeShard", datasetAttributes, otherGridPosition); + Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, otherBlock.getData()); + } + } + } From 1ced5708260110a7699031772c09dcce8e26e556 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 2 Jan 2025 17:03:53 -0500 Subject: [PATCH 079/423] feat: writeShardEndStream --- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 1 - .../saalfeldlab/n5/shard/InMemoryShard.java | 29 +++++++++++++------ .../n5/shard/ShardIndexBuilder.java | 2 +- .../saalfeldlab/n5/shard/ShardDemos.java | 1 + 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index ee5ab2039..53f95c5e7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -255,7 +255,6 @@ default void writeShard( try (final LockedChannel lock = getKeyValueAccess().lockForWriting(shardPath)) { try (final OutputStream out = lock.newOutputStream()) { InMemoryShard.fromShard(shard).write(out); - out.close(); } } catch (final IOException | UncheckedIOException e) { throw new N5IOException( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index a3d9d1680..61194500b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -8,6 +8,7 @@ import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.io.output.CountingOutputStream; +import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DefaultBlockWriter; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; @@ -80,7 +81,7 @@ public ShardIndex getIndex() { public void write(final OutputStream out) throws IOException { if (indexLocation() == IndexLocation.END) - writeShardEnd(out, this); + writeShardEndStream(out, this); else writeShardStart(out, this); } @@ -110,21 +111,31 @@ protected static void writeShardEndStream( final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); indexBuilder.indexLocation(IndexLocation.END); + indexBuilder.setCodecs(datasetAttributes.getShardingCodec().getIndexCodecs()); - final CountingOutputStream cout = new CountingOutputStream(out); - - long offset = 0; + final ProxyOutputStream nop = new ProxyOutputStream(out) { + + @Override public void close() { + //nop + } + }; + + final CountingOutputStream cout = new CountingOutputStream(nop); + + long bytesWritten = 0; for (int i = 0; i < shard.numBlocks(); i++) { final DataBlock block = shard.getBlock(i); DefaultBlockWriter.writeBlock(cout, datasetAttributes, block); - - indexBuilder.addBlock( block.getGridPosition(), offset); - offset = cout.getByteCount(); + + + final long size = cout.getByteCount() - bytesWritten; + bytesWritten = cout.getByteCount(); + + indexBuilder.addBlock( block.getGridPosition(), size); } - final ShardIndex index = indexBuilder.build(); - DefaultBlockWriter.writeBlock(out, datasetAttributes, index); + ShardIndex.write(indexBuilder.build(), out); } protected static void writeShardEnd( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java index c8511dc8e..cf16f7190 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java @@ -55,7 +55,7 @@ public ShardIndexBuilder setCodecs(DeterministicSizeCodec... codecs) { } public ShardIndexBuilder addBlock(long[] blockPosition, long numBytes) { - + //TODO Caleb: Maybe move to ShardIndex? final long[] blockPositionInShard = shard.getDatasetAttributes().getBlockPositionInShard( shard.getGridPosition(), blockPosition); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java index ee4a84118..0ab1c4666 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java @@ -71,6 +71,7 @@ public static void main(String[] args) throws MalformedURLException { @Parameterized.Parameters(name = "IndexLocation({0}), Block ByteOrder({1}), Index ByteOrder({2})") public static Collection data() { + final ArrayList params = new ArrayList<>(); for (IndexLocation indexLoc : IndexLocation.values()) { for (ByteOrder blockByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { From 0e045cd05deac296b331e22144da044031e21e6c Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 3 Jan 2025 10:55:38 -0500 Subject: [PATCH 080/423] feat: serialize shardSize in DatasetAttributes for n5 --- .../saalfeldlab/n5/DatasetAttributes.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 04ec1b35b..dc2b984c0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -43,6 +43,7 @@ public class DatasetAttributes implements Serializable { public static final String DIMENSIONS_KEY = "dimensions"; public static final String BLOCK_SIZE_KEY = "blockSize"; + public static final String SHARD_SIZE_KEY = "shardSize"; public static final String DATA_TYPE_KEY = "dataType"; public static final String COMPRESSION_KEY = "compression"; public static final String CODEC_KEY = "codecs"; @@ -250,6 +251,12 @@ public static class DatasetAttributesAdapter implements JsonSerializer blocksPerShard[i] * blockSize[i]); return new ShardedDatasetAttributes( dimensions, shardSize, @@ -287,6 +291,12 @@ public static class DatasetAttributesAdapter implements JsonSerializer Date: Fri, 3 Jan 2025 11:39:04 -0500 Subject: [PATCH 081/423] test: BytesTest operates on n5 container --- .../saalfeldlab/n5/codec/BytesTests.java | 4 +-- .../shardExamples/test.n5/attributes.json | 3 ++ .../test.n5/mid_sharded/attributes.json | 30 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/shardExamples/test.n5/attributes.json create mode 100644 src/test/resources/shardExamples/test.n5/mid_sharded/attributes.json diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java index 5d58657d3..66ef86321 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java @@ -27,9 +27,9 @@ public void testSerialization() { gsonBuilder.registerTypeAdapter(ByteOrder.class, BytesCodec.byteOrderAdapter); factory.gsonBuilder(gsonBuilder); - final N5Writer reader = factory.openWriter("n5:src/test/resources/shardExamples/test.zarr"); + final N5Writer reader = factory.openWriter("n5:src/test/resources/shardExamples/test.n5"); final Codec bytes = reader.getAttribute("mid_sharded", "codecs[0]/configuration/codecs[0]", Codec.class); - assertTrue("as BytesCodec", bytes instanceof N5BlockCodec); + assertTrue("as BytesCodec", bytes instanceof BytesCodec); final N5Writer writer = factory.openWriter("n5:src/test/resources/shardExamples/test.n5"); diff --git a/src/test/resources/shardExamples/test.n5/attributes.json b/src/test/resources/shardExamples/test.n5/attributes.json new file mode 100644 index 000000000..573b01888 --- /dev/null +++ b/src/test/resources/shardExamples/test.n5/attributes.json @@ -0,0 +1,3 @@ +{ + "n5": "4.0.0" +} \ No newline at end of file diff --git a/src/test/resources/shardExamples/test.n5/mid_sharded/attributes.json b/src/test/resources/shardExamples/test.n5/mid_sharded/attributes.json new file mode 100644 index 000000000..b9e575b2a --- /dev/null +++ b/src/test/resources/shardExamples/test.n5/mid_sharded/attributes.json @@ -0,0 +1,30 @@ +{ + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2, + 3 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + } + ], + "index_location": "end" + } + } + ] +} From cc5b8c531c2ff1225ea657c005abbaf18569eea2 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 3 Jan 2025 16:33:39 -0500 Subject: [PATCH 082/423] feat: ShardedDatasetAttributes validate shard/block size on construction --- .../n5/ShardedDatasetAttributes.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index c8d3ee6ab..835a3a8b1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -28,6 +28,13 @@ public ShardedDatasetAttributes ( final IndexLocation indexLocation ) { super(dimensions, blockSize, dataType, null, blocksCodecs); + + if (!validateShardBlockSize(shardSize, blockSize)) { + throw new N5Exception(String.format("Invalid shard %s / block size %s", + Arrays.toString(shardSize), + Arrays.toString(blockSize))); + } + this.shardSize = shardSize; this.shardingCodec = new ShardingCodec( blockSize, @@ -48,6 +55,26 @@ public ShardedDatasetAttributes( this.shardingCodec = codec; } + /** + * Returns whether the given shard and block sizes are valid. Specifically, is + * the shard size a multiple of the block size in every dimension. + * + * @param shardSize size of the shard in pixels + * @param blockSize size of a block in pixels + * @return + */ + public static boolean validateShardBlockSize(final int[] shardSize, final int[] blockSize) { + + if (shardSize.length != blockSize.length) + return false; + + for (int i = 0; i < shardSize.length; i++) { + if (shardSize[i] % blockSize[i] != 0) + return false; + } + return true; + } + public ShardedDatasetAttributes getShardAttributes() { return this; } From 367b987c0406da4af4d27cea1633a3f6b26062fe Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 3 Jan 2025 13:53:11 -0500 Subject: [PATCH 083/423] refactor: remove `DatasetAttributes#getShardedAttributes()` unnecessary as you can do the same with an instance check --- .../saalfeldlab/n5/DatasetAttributes.java | 21 ++----------------- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 4 ++-- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 5 +---- .../org/janelia/saalfeldlab/n5/N5Writer.java | 2 +- 4 files changed, 6 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index dc2b984c0..e67761c8f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -149,22 +149,6 @@ public BytesCodec[] getCodecs() { return byteCodecs; } - public ShardedDatasetAttributes getShardAttributes() { - - if (getArrayCodec() instanceof ShardingCodec) { - - final ShardingCodec shardingCodec = (ShardingCodec)getArrayCodec(); - return new ShardedDatasetAttributes( - dimensions, - blockSize, - shardingCodec.getBlockSize(), - getDataType(), - shardingCodec); - - } else - return null; - } - public HashMap asMap() { final HashMap map = new HashMap<>(); @@ -292,9 +276,8 @@ public static class DatasetAttributesAdapter implements JsonSerializer readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { - final ShardedDatasetAttributes shardedAttrs = datasetAttributes.getShardAttributes(); - if (shardedAttrs != null) { + if (datasetAttributes instanceof ShardedDatasetAttributes) { + final ShardedDatasetAttributes shardedAttrs = (ShardedDatasetAttributes) datasetAttributes; final long[] shardPosition = shardedAttrs.getShardPositionForBlock(gridPosition); final Shard shard = getShard(pathName, shardedAttrs, shardPosition); return shard.getBlock(gridPosition); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 53f95c5e7..28e93806a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -245,12 +245,9 @@ default void writeBlock( @Override default void writeShard( final String path, - final DatasetAttributes datasetAttributes, + final ShardedDatasetAttributes datasetAttributes, final Shard shard) throws N5Exception { - if( datasetAttributes.getShardAttributes() == null ) - throw new N5IOException("Tried to write shard into a not-sharded dataset: " + path); - final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shard.getGridPosition()); try (final LockedChannel lock = getKeyValueAccess().lockForWriting(shardPath)) { try (final OutputStream out = lock.newOutputStream()) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index b0ed462f5..2927a1085 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -314,7 +314,7 @@ default void writeBlocks( */ void writeShard( final String datasetPath, - final DatasetAttributes datasetAttributes, + final ShardedDatasetAttributes datasetAttributes, final Shard shard) throws N5Exception; /** From 20a9677dbfd0f7d5f5bcce13aedc9c5fbf56a1c3 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 3 Jan 2025 16:25:50 -0500 Subject: [PATCH 084/423] feat: writeBlocks aggregate shard --- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 29 +++++++++++++ .../org/janelia/saalfeldlab/n5/N5Writer.java | 2 - .../n5/ShardedDatasetAttributes.java | 1 - .../saalfeldlab/n5/shard/AbstractShard.java | 6 --- .../saalfeldlab/n5/shard/InMemoryShard.java | 43 ++++++++++--------- .../janelia/saalfeldlab/n5/shard/Shard.java | 2 +- 6 files changed, 53 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 28e93806a..e3ceb7593 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -29,6 +29,7 @@ import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -214,6 +215,34 @@ default boolean removeAttributes(final String pathName, final List attri return removed; } + @Override default void writeBlocks(final String datasetPath, final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { + + if (datasetAttributes instanceof ShardedDatasetAttributes) { + final ShardedDatasetAttributes shardAttributes = (ShardedDatasetAttributes)datasetAttributes; + /* Group by shard index */ + final HashMap> shardBlockMap = new HashMap<>(); + + for (DataBlock dataBlock : dataBlocks) { + final long[] shardPosition = shardAttributes.getShardPositionForBlock(dataBlock.getGridPosition()); + final int shardHash = Arrays.hashCode(shardPosition); + if (!shardBlockMap.containsKey(shardHash)) + shardBlockMap.put(shardHash, new InMemoryShard<>(shardAttributes, shardPosition)); + final InMemoryShard shard = shardBlockMap.get(shardHash); + shard.addBlock(dataBlock); + } + + for (InMemoryShard shard : shardBlockMap.values()) { + writeShard(datasetPath, shardAttributes, shard); + } + } else { + /* Just write each block */ + for (DataBlock dataBlock : dataBlocks) { + writeBlock(datasetPath, datasetAttributes, dataBlock); + } + } + + } + @Override default void writeBlock( final String path, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 2927a1085..4883bd448 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -296,8 +296,6 @@ default void writeBlocks( final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { - // TODO Caleb: write this - // default method is naive for (DataBlock block : dataBlocks) writeBlock(datasetPath, datasetAttributes, block); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index 835a3a8b1..8ee40baf4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -150,7 +150,6 @@ public int[] getBlocksPerShard() { */ public long[] getShardPositionForBlock(final long... blockGridPosition) { - // TODO have this return a shard final int[] blocksPerShard = getBlocksPerShard(); final long[] shardGridPosition = new long[blockGridPosition.length]; for (int i = 0; i < shardGridPosition.length; i++) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java index fc30eaa6c..6aecd1a4a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java @@ -43,12 +43,6 @@ public long[] getGridPosition() { return gridPosition; } - @Override - public DataBlock getBlock(long... position) { - - return null; - } - @Override public ShardIndex getIndex() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 61194500b..7e699a1b0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -4,6 +4,7 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import org.apache.commons.io.output.ByteArrayOutputStream; @@ -16,8 +17,8 @@ public class InMemoryShard extends AbstractShard { - private List> blocks; - + /* Map of a hash of the DataBlocks `gridPosition` to the block */ + private final HashMap> blocks; private ShardIndexBuilder indexBuilder; /* @@ -26,9 +27,9 @@ public class InMemoryShard extends AbstractShard { * (later) */ - public InMemoryShard(final ShardedDatasetAttributes datasetAttributes, final long[] gridPosition) { + public InMemoryShard(final ShardedDatasetAttributes datasetAttributes, final long[] shardPosition) { - this( datasetAttributes, gridPosition, null); + this( datasetAttributes, shardPosition, null); indexBuilder = new ShardIndexBuilder(this); indexBuilder.indexLocation(datasetAttributes.getIndexLocation()); } @@ -37,7 +38,17 @@ public InMemoryShard(final ShardedDatasetAttributes datasetAttributes, final lon ShardIndex index) { super(datasetAttributes, gridPosition, index); - blocks = new ArrayList<>(); + blocks = new HashMap<>(); + } + + private void storeBlock(DataBlock block) { + + blocks.put(Arrays.hashCode(block.getGridPosition()), block); + } + + @Override public DataBlock getBlock(long... blockGridPosition) { + + return blocks.get(Arrays.hashCode(blockGridPosition)); } @Override @@ -48,7 +59,7 @@ public void writeBlock(DataBlock block) { public void addBlock(DataBlock block) { - blocks.add(block); + storeBlock(block); } public int numBlocks() { @@ -56,9 +67,9 @@ public int numBlocks() { return blocks.size(); } - public DataBlock getBlock(int i) { + public List> getBlocks() { - return blocks.get(i); + return new ArrayList<>(blocks.values()); } protected IndexLocation indexLocation() { @@ -113,8 +124,8 @@ protected static void writeShardEndStream( indexBuilder.indexLocation(IndexLocation.END); indexBuilder.setCodecs(datasetAttributes.getShardingCodec().getIndexCodecs()); + // Neccesary to stop `close()` when writing blocks from closing out base OutputStream final ProxyOutputStream nop = new ProxyOutputStream(out) { - @Override public void close() { //nop } @@ -123,12 +134,8 @@ protected static void writeShardEndStream( final CountingOutputStream cout = new CountingOutputStream(nop); long bytesWritten = 0; - for (int i = 0; i < shard.numBlocks(); i++) { - - final DataBlock block = shard.getBlock(i); + for (DataBlock block : shard.getBlocks()) { DefaultBlockWriter.writeBlock(cout, datasetAttributes, block); - - final long size = cout.getByteCount() - bytesWritten; bytesWritten = cout.getByteCount(); @@ -148,10 +155,8 @@ protected static void writeShardEnd( indexBuilder.indexLocation(IndexLocation.END); indexBuilder.setCodecs(datasetAttributes.getShardingCodec().getIndexCodecs()); - for (int i = 0; i < shard.numBlocks(); i++) { - + for (DataBlock block : shard.getBlocks()) { final ByteArrayOutputStream os = new ByteArrayOutputStream(); - final DataBlock block = shard.getBlock(i); DefaultBlockWriter.writeBlock(os, datasetAttributes, block); indexBuilder.addBlock(block.getGridPosition(), os.size()); @@ -171,10 +176,8 @@ protected static void writeShardStart( indexBuilder.setCodecs(datasetAttributes.getShardingCodec().getIndexCodecs()); final List blockData = new ArrayList<>(shard.numBlocks()); - for (int i = 0; i < shard.numBlocks(); i++) { - + for (DataBlock block : shard.getBlocks()) { final ByteArrayOutputStream os = new ByteArrayOutputStream(); - final DataBlock block = shard.getBlock(i); DefaultBlockWriter.writeBlock(os, datasetAttributes, block); blockData.add(os.toByteArray()); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index abaf7e164..eef1c9604 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -97,7 +97,7 @@ default long[] getShard(long... blockPosition) { return shardGridPosition; } - public DataBlock getBlock(long... position); + public DataBlock getBlock(long... blockGridPosition); public void writeBlock(DataBlock block); From c3c3ceb20dfa409ff6aae340d321c787caa52571 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 3 Jan 2025 16:27:37 -0500 Subject: [PATCH 085/423] fix: index offset calculation --- .../org/janelia/saalfeldlab/n5/shard/ShardIndex.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index b400b8639..561e9ea75 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -82,14 +82,11 @@ public void set(long offset, long nbytes, long[] gridPosition) { private int getOffsetIndex(long... gridPosition) { - int idx = 0; - long stride = 2; - for (int i = 0; i < gridPosition.length; i++) { - idx += gridPosition[i] * stride; - stride *= size[i]; + int idx = (int) gridPosition[0]; + for (int i = 1; i < gridPosition.length; i++) { + idx += gridPosition[i] * size[i]; } - - return idx; + return idx * 2; } private int getNumBytesIndex(long... gridPosition) { From 5badbb7110e14ffcff999d577e1f62058f5dbfa3 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 3 Jan 2025 16:28:15 -0500 Subject: [PATCH 086/423] feat(test): wip shard writeBlocks refactor(test): use temp writer via N5FSTest --- .../saalfeldlab/n5/AbstractN5Test.java | 2 +- .../shard/{ShardDemos.java => ShardTest.java} | 167 ++++++++++-------- 2 files changed, 91 insertions(+), 78 deletions(-) rename src/test/java/org/janelia/saalfeldlab/n5/shard/{ShardDemos.java => ShardTest.java} (57%) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index d4d3591ba..a19163273 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -92,7 +92,7 @@ public abstract class AbstractN5Test { protected final HashSet tempWriters = new HashSet<>(); - protected final N5Writer createTempN5Writer() { + public final N5Writer createTempN5Writer() { try { return createTempN5Writer(tempN5Location()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java similarity index 57% rename from src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java rename to src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 0ab1c4666..2a39e161f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardDemos.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -1,10 +1,12 @@ package org.janelia.saalfeldlab.n5.shard; -import com.google.gson.GsonBuilder; +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.N5FSTest; +import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.codec.BytesCodec; @@ -13,17 +15,13 @@ import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; -import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import java.net.MalformedURLException; import java.nio.ByteOrder; -import java.nio.file.FileSystems; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -31,43 +29,9 @@ import java.util.Map; @RunWith(Parameterized.class) -public class ShardDemos { +public class ShardTest { - public static void main(String[] args) throws MalformedURLException { - - final Path p = Paths.get("src/test/resources/shardExamples/test.zarr/mid_sharded/c/0/0"); - System.out.println(p); - - final String key = p.toString(); - final ShardedDatasetAttributes dsetAttrs = new ShardedDatasetAttributes( - new long[]{6, 4}, - new int[]{6, 4}, - new int[]{3, 2}, - DataType.UINT8, - new Codec[]{new N5BlockCodec()}, - new DeterministicSizeCodec[]{new BytesCodec(), new Crc32cChecksumCodec()}, - IndexLocation.END - ); - - final FileSystemKeyValueAccess kva = new FileSystemKeyValueAccess(FileSystems.getDefault()); - final VirtualShard shard = new VirtualShard<>(dsetAttrs, new long[]{0, 0}, kva, key); - - final DataBlock blk = shard.getBlock(0, 0); - - final byte[] data = blk.getData(); - System.out.println(Arrays.toString(data)); - - // fill the block with a weird value - Arrays.fill(data, (byte)123); - - // write the block - shard.writeBlock(blk); - - // re-read the block and check the data it contains - final DataBlock blkReread = shard.getBlock(0, 0); - final byte[] dataReRead = blkReread.getData(); - System.out.println(Arrays.toString(dataReRead)); - } + private static final N5FSTest tempN5Factory = new N5FSTest(); @Parameterized.Parameters(name = "IndexLocation({0}), Block ByteOrder({1}), Index ByteOrder({2})") public static Collection data() { @@ -94,26 +58,82 @@ public static Collection data() { @Parameterized.Parameter(2) public ByteOrder indexByteOrder; - @Test - public void writeReadBlockTest() { - - final N5Factory factory = new N5Factory(); - final GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.setPrettyPrinting(); - factory.gsonBuilder(gsonBuilder); - factory.cacheAttributes(false); - - final N5Writer writer = factory.openWriter("src/test/resources/shardExamples/test.n5"); + @After + public void removeTempWriters() { + tempN5Factory.removeTempWriters(); + } - final ShardedDatasetAttributes datasetAttributes = new ShardedDatasetAttributes( - new long[]{8, 8}, - new int[]{4, 4}, - new int[]{2, 2}, + private ShardedDatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] blockSize) { + return new ShardedDatasetAttributes( + dimensions, + shardSize, + blockSize, DataType.UINT8, - new Codec[]{new N5BlockCodec(dataByteOrder), new GzipCompression(4)}, + new Codec[]{new N5BlockCodec(dataByteOrder) , new GzipCompression(4)}, new DeterministicSizeCodec[]{new BytesCodec(indexByteOrder), new Crc32cChecksumCodec()}, indexLocation ); + } + + private ShardedDatasetAttributes getTestAttributes() { + return getTestAttributes(new long[]{8, 8}, new int[]{4, 4}, new int[]{2, 2}); + } + + @Test + public void writeReadBlocksTest() { + + final N5Writer writer = tempN5Factory.createTempN5Writer(); + final ShardedDatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24,24}, + new int[]{8,8}, + new int[]{2,2} + ); + + writer.createDataset("shard", datasetAttributes); + + final int[] blockSize = datasetAttributes.getBlockSize(); + final int numElements = blockSize[0] * blockSize[1]; + + final byte[] data = new byte[numElements]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)((100) + (10) + i); + } + + writer.writeBlocks( + "shard", + datasetAttributes, + /* shard (0, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{0,0}, data), + new ByteArrayDataBlock(blockSize, new long[]{0,1}, data), + new ByteArrayDataBlock(blockSize, new long[]{1,0}, data), + new ByteArrayDataBlock(blockSize, new long[]{1,1}, data), + + /* shard (1, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{4,0}, data), + new ByteArrayDataBlock(blockSize, new long[]{5,0}, data), + + /* shard (2, 2) */ + new ByteArrayDataBlock(blockSize, new long[]{11,11}, data) + ); + + final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); + String p = writer.getURI().getPath(); + final String shard00 = kva.compose(writer.getURI(), "shard", "0", "0"); + kva.exists(shard00); + + final String shard10 = kva.compose(writer.getURI(), "shard", "0", "0"); + kva.exists(shard10); + + final String shard33 = kva.compose(writer.getURI(), "shard", "0", "0"); + kva.exists(shard33); + } + + @Test + public void writeReadBlockTest() { + + final N5Writer writer = tempN5Factory.createTempN5Writer(); + final ShardedDatasetAttributes datasetAttributes = getTestAttributes(); + writer.createDataset("shard", datasetAttributes); writer.deleteBlock("shard", 0, 0); @@ -126,21 +146,21 @@ public void writeReadBlockTest() { for (int idx1 = 1; idx1 >= 0; idx1--) { for (int idx2 = 1; idx2 >= 0; idx2--) { final long[] gridPosition = {idx1, idx2}; - final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(blockSize, gridPosition, numElements); - byte[] data = dataBlock.getData(); + final DataBlock dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); + byte[] data = (byte[])dataBlock.getData(); for (int i = 0; i < data.length; i++) { data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); } writer.writeBlock("shard", datasetAttributes, dataBlock); - final DataBlock block = (DataBlock)writer.readBlock("shard", datasetAttributes, gridPosition); - Assert.assertArrayEquals("Read from shard doesn't match", data, block.getData()); + final DataBlock block = writer.readBlock("shard", datasetAttributes, gridPosition); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); for (Map.Entry entry : writtenBlocks.entrySet()) { final long[] otherGridPosition = entry.getKey(); final byte[] otherData = entry.getValue(); - final DataBlock otherBlock = (DataBlock)writer.readBlock("shard", datasetAttributes, otherGridPosition); - Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, otherBlock.getData()); + final DataBlock otherBlock = writer.readBlock("shard", datasetAttributes, otherGridPosition); + Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); } writtenBlocks.put(gridPosition, data); @@ -151,13 +171,7 @@ public void writeReadBlockTest() { @Test public void writeReadShardTest() { - final N5Factory factory = new N5Factory(); - final GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.setPrettyPrinting(); - factory.gsonBuilder(gsonBuilder); - factory.cacheAttributes(false); - - final N5Writer writer = factory.openWriter("src/test/resources/shardExamples/test.n5"); + final N5Writer writer = tempN5Factory.createTempN5Writer(); final ShardedDatasetAttributes datasetAttributes = new ShardedDatasetAttributes( new long[]{4, 4}, @@ -182,13 +196,12 @@ public void writeReadShardTest() { for (int idx1 = 1; idx1 >= 0; idx1--) { for (int idx2 = 1; idx2 >= 0; idx2--) { final long[] gridPosition = {idx1, idx2}; - final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(blockSize, gridPosition, numElements); - byte[] data = dataBlock.getData(); + final DataBlock dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); + byte[] data = (byte[])dataBlock.getData(); for (int i = 0; i < data.length; i++) { data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); } - - shard.addBlock(dataBlock); + shard.addBlock((DataBlock)dataBlock); writtenBlocks.put(gridPosition, data); } } @@ -198,8 +211,8 @@ public void writeReadShardTest() { for (Map.Entry entry : writtenBlocks.entrySet()) { final long[] otherGridPosition = entry.getKey(); final byte[] otherData = entry.getValue(); - final DataBlock otherBlock = (DataBlock)writer.readBlock("wholeShard", datasetAttributes, otherGridPosition); - Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, otherBlock.getData()); + final DataBlock otherBlock = writer.readBlock("wholeShard", datasetAttributes, otherGridPosition); + Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); } } From eb9fbc1b4203245599e8e532843d0b5c432960be Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 6 Jan 2025 16:29:41 -0500 Subject: [PATCH 087/423] feat/refactor: add BlockParameters and ShardParameters interfaces * will make zarr implementation less repetative --- .../saalfeldlab/n5/BlockParameters.java | 11 ++ .../saalfeldlab/n5/DatasetAttributes.java | 5 +- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 7 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 31 ++-- .../org/janelia/saalfeldlab/n5/N5Writer.java | 8 +- .../n5/ShardedDatasetAttributes.java | 142 ++---------------- .../saalfeldlab/n5/shard/AbstractShard.java | 12 +- .../saalfeldlab/n5/shard/InMemoryShard.java | 35 ++--- .../janelia/saalfeldlab/n5/shard/Shard.java | 14 +- .../n5/shard/ShardIndexBuilder.java | 4 +- .../saalfeldlab/n5/shard/ShardParameters.java | 140 +++++++++++++++++ .../saalfeldlab/n5/shard/ShardWriter.java | 10 +- .../saalfeldlab/n5/shard/VirtualShard.java | 6 +- .../saalfeldlab/n5/shard/ShardTest.java | 2 +- 14 files changed, 235 insertions(+), 192 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/BlockParameters.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BlockParameters.java b/src/main/java/org/janelia/saalfeldlab/n5/BlockParameters.java new file mode 100644 index 000000000..65a214972 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/BlockParameters.java @@ -0,0 +1,11 @@ +package org.janelia.saalfeldlab.n5; + +public interface BlockParameters { + + public long[] getDimensions(); + + public int getNumDimensions(); + + public int[] getBlockSize(); + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index e67761c8f..684339a27 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -37,7 +37,7 @@ * @author Stephan Saalfeld * */ -public class DatasetAttributes implements Serializable { +public class DatasetAttributes implements BlockParameters, Serializable { private static final long serialVersionUID = -4521467080388947553L; @@ -114,16 +114,19 @@ public DatasetAttributes( this(dimensions, blockSize, dataType, compression, null); } + @Override public long[] getDimensions() { return dimensions; } + @Override public int getNumDimensions() { return dimensions.length; } + @Override public int[] getBlockSize() { return blockSize; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 244bc1cab..8b0c49cb3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -31,6 +31,7 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.ShardParameters; import org.janelia.saalfeldlab.n5.shard.VirtualShard; import com.google.gson.Gson; @@ -89,8 +90,8 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { } @SuppressWarnings("rawtypes") - default Shard getShard(final String pathName, - final ShardedDatasetAttributes datasetAttributes, + default Shard getShard(final String pathName, + final A datasetAttributes, long... shardGridPosition) { final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), shardGridPosition); @@ -106,7 +107,7 @@ default DataBlock readBlock( if (datasetAttributes instanceof ShardedDatasetAttributes) { final ShardedDatasetAttributes shardedAttrs = (ShardedDatasetAttributes) datasetAttributes; final long[] shardPosition = shardedAttrs.getShardPositionForBlock(gridPosition); - final Shard shard = getShard(pathName, shardedAttrs, shardPosition); + final Shard shard = getShard(pathName, shardedAttrs, shardPosition); return shard.getBlock(gridPosition); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index e3ceb7593..e044c6bc7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -36,6 +36,7 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.shard.InMemoryShard; import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.ShardParameters; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -215,32 +216,34 @@ default boolean removeAttributes(final String pathName, final List attri return removed; } + @SuppressWarnings({ "rawtypes", "unchecked" }) @Override default void writeBlocks(final String datasetPath, final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { - if (datasetAttributes instanceof ShardedDatasetAttributes) { - final ShardedDatasetAttributes shardAttributes = (ShardedDatasetAttributes)datasetAttributes; + if (datasetAttributes instanceof ShardParameters) { /* Group by shard index */ - final HashMap> shardBlockMap = new HashMap<>(); + final HashMap> shardBlockMap = new HashMap<>(); + final ShardParameters shardAttributes = (ShardParameters)datasetAttributes; for (DataBlock dataBlock : dataBlocks) { final long[] shardPosition = shardAttributes.getShardPositionForBlock(dataBlock.getGridPosition()); final int shardHash = Arrays.hashCode(shardPosition); if (!shardBlockMap.containsKey(shardHash)) - shardBlockMap.put(shardHash, new InMemoryShard<>(shardAttributes, shardPosition)); - final InMemoryShard shard = shardBlockMap.get(shardHash); + shardBlockMap.put(shardHash, new InMemoryShard<>((DatasetAttributes & ShardParameters)shardAttributes, shardPosition)); + + final InMemoryShard shard = shardBlockMap.get(shardHash); shard.addBlock(dataBlock); } - for (InMemoryShard shard : shardBlockMap.values()) { - writeShard(datasetPath, shardAttributes, shard); + for (InMemoryShard shard : shardBlockMap.values()) { + writeShard(datasetPath, (DatasetAttributes & ShardParameters)shardAttributes, (Shard)shard); } + } else { /* Just write each block */ for (DataBlock dataBlock : dataBlocks) { writeBlock(datasetPath, datasetAttributes, dataBlock); } } - } @Override @@ -250,11 +253,11 @@ default void writeBlock( final DataBlock dataBlock) throws N5Exception { /* Delegate to shard for writing block? How to know what type of shard? */ - if (datasetAttributes instanceof ShardedDatasetAttributes) { - ShardedDatasetAttributes shardDatasetAttrs = (ShardedDatasetAttributes)datasetAttributes; + if (datasetAttributes instanceof ShardParameters) { + ShardParameters shardDatasetAttrs = (ShardParameters)datasetAttributes; final long[] shardPos = shardDatasetAttrs.getShardPositionForBlock(dataBlock.getGridPosition()); final String shardPath = absoluteShardPath(N5URI.normalizeGroupPath(path), shardPos); - final VirtualShard shard = new VirtualShard<>(shardDatasetAttrs, shardPos, getKeyValueAccess(), shardPath); + final VirtualShard shard = new VirtualShard<>((DatasetAttributes & ShardParameters)shardDatasetAttrs, shardPos, getKeyValueAccess(), shardPath); shard.writeBlock(dataBlock); return; } @@ -272,10 +275,10 @@ default void writeBlock( } @Override - default void writeShard( + default void writeShard( final String path, - final ShardedDatasetAttributes datasetAttributes, - final Shard shard) throws N5Exception { + final A datasetAttributes, + final Shard shard) throws N5Exception { final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shard.getGridPosition()); try (final LockedChannel lock = getKeyValueAccess().lockForWriting(shardPath)) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 4883bd448..463d232a3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -36,6 +36,7 @@ import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.ShardParameters; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; @@ -308,12 +309,13 @@ default void writeBlocks( * @param datasetAttributes the dataset attributes * @param shard the shard * @param the data block data type + * @param the attribute type * @throws N5Exception the exception */ - void writeShard( + void writeShard( final String datasetPath, - final ShardedDatasetAttributes datasetAttributes, - final Shard shard) throws N5Exception; + final A datasetAttributes, + final Shard shard) throws N5Exception; /** * Deletes the block at {@code gridPosition} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index 8ee40baf4..a42025295 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -7,10 +7,11 @@ import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.shard.ShardIndex; +import org.janelia.saalfeldlab.n5.shard.ShardParameters; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; -public class ShardedDatasetAttributes extends DatasetAttributes { +public class ShardedDatasetAttributes extends DatasetAttributes implements ShardParameters { private static final long serialVersionUID = -4559068841006651814L; @@ -75,10 +76,6 @@ public static boolean validateShardBlockSize(final int[] shardSize, final int[] return true; } - public ShardedDatasetAttributes getShardAttributes() { - return this; - } - public ShardingCodec getShardingCodec() { return shardingCodec; } @@ -99,130 +96,26 @@ protected Codec[] concatenateCodecs() { return new Codec[] { shardingCodec }; } - /** - * The size of the blocks in pixel units. - * - * @return the number of pixels per dimension for this shard. - */ - public int[] getShardSize() { - - return shardSize; - } - - /** - * Returns the number of shards per dimension for the dataset. - * - * @return the size of the shard grid of a dataset - */ - public int[] getShardBlockGridSize() { - - final int nd = getNumDimensions(); - final int[] shardBlockGridSize = new int[nd]; - final int[] blockSize = getBlockSize(); - for (int i = 0; i < nd; i++) - shardBlockGridSize[i] = (int)(Math.ceil((double)getDimensions()[i] / blockSize[i])); - - return shardBlockGridSize; - } - - /** - * Returns the number of blocks per dimension for a shard. - * - * @return the size of the block grid of a shard - */ - public int[] getBlocksPerShard() { - - final int nd = getNumDimensions(); - final int[] blocksPerShard = new int[nd]; - final int[] blockSize = getBlockSize(); - for (int i = 0; i < nd; i++) - blocksPerShard[i] = getShardSize()[i] / blockSize[i]; - - return blocksPerShard; - } - - /** - * Given a block's position relative to the array, returns the position of the shard containing that block relative to the shard grid. - * - * @param blockGridPosition - * position of a block relative to the array - * @return the position of the containing shard in the shard grid - */ - public long[] getShardPositionForBlock(final long... blockGridPosition) { - - final int[] blocksPerShard = getBlocksPerShard(); - final long[] shardGridPosition = new long[blockGridPosition.length]; - for (int i = 0; i < shardGridPosition.length; i++) { - shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / blocksPerShard[i]); - } - - return shardGridPosition; - } - - /** - * Returns the block at the given position relative to this shard, or null if this shard does not contain the given block. - * - * @return the block position - */ - public long[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { - - // TODO check correctness - final long[] shardPos = getShardPositionForBlock(blockPosition); - if (!Arrays.equals(shardPosition, shardPos)) - return null; - - final int[] shardSize = getBlocksPerShard(); - final long[] blockShardPos = new long[shardSize.length]; - for (int i = 0; i < shardSize.length; i++) { - blockShardPos[i] = blockPosition[i] % shardSize[i]; - } + @Override + public IndexLocation getIndexLocation() { - return blockShardPos; + return getShardingCodec().getIndexLocation(); } - /** - * Given a block's position relative to a shard, returns its position in pixels - * relative to the image. - * - * @return the block position - */ - public long[] getBlockMinFromShardPosition(final long[] shardPosition, final long[] blockPosition) { - - // is this useful? - final int[] blockSize = getBlockSize(); - final int[] shardSize = getShardSize(); - final long[] blockImagePos = new long[shardSize.length]; - for (int i = 0; i < shardSize.length; i++) { - blockImagePos[i] = (shardPosition[i] * shardSize[i]) + (blockPosition[i] * blockSize[i]); - } - - return blockImagePos; + @Override + public ShardIndex createIndex() { + return new ShardIndex(getBlocksPerShard(), getIndexLocation(), getShardingCodec().getIndexCodecs()); } /** - * Given a block's position relative to a shard, returns its position relative - * to the image. + * The size of the blocks in pixel units. * - * @return the block position - */ - public long[] getBlockPositionFromShardPosition(final long[] shardPosition, final long[] blockPosition) { - - // is this useful? - final int[] shardBlockSize = getBlocksPerShard(); - final long[] blockImagePos = new long[shardSize.length]; - for (int i = 0; i < shardSize.length; i++) { - blockImagePos[i] = (shardPosition[i] * shardBlockSize[i]) + (blockPosition[i]); - } - - return blockImagePos; - } - - /** - * @return the number of blocks per shard + * @return the number of pixels per dimension for this shard. */ - public long getNumBlocks() { + @Override + public int[] getShardSize() { - return Arrays.stream(getBlocksPerShard()).reduce(1, (x, y) -> x * y); + return shardSize; } public static int[] getBlockSize(Codec[] codecs) { @@ -233,13 +126,4 @@ public static int[] getBlockSize(Codec[] codecs) { return null; } - - public IndexLocation getIndexLocation() { - - return getShardingCodec().getIndexLocation(); - } - - public ShardIndex createIndex() { - return new ShardIndex(getBlocksPerShard(), getIndexLocation(), getShardingCodec().getIndexCodecs()); - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java index 6aecd1a4a..5ecc67c70 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java @@ -1,17 +1,16 @@ package org.janelia.saalfeldlab.n5.shard; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.DatasetAttributes; -public abstract class AbstractShard implements Shard { +public abstract class AbstractShard implements Shard { - protected final ShardedDatasetAttributes datasetAttributes; + protected final A datasetAttributes; protected ShardIndex index; private final long[] gridPosition; - public AbstractShard(final ShardedDatasetAttributes datasetAttributes, final long[] gridPosition, + public AbstractShard(final A datasetAttributes, final long[] gridPosition, final ShardIndex index) { this.datasetAttributes = datasetAttributes; @@ -20,7 +19,7 @@ public AbstractShard(final ShardedDatasetAttributes datasetAttributes, final lon } @Override - public ShardedDatasetAttributes getDatasetAttributes() { + public A getDatasetAttributes() { return datasetAttributes; } @@ -49,5 +48,4 @@ public ShardIndex getIndex() { return index; } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 7e699a1b0..36a112e2c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -11,11 +11,11 @@ import org.apache.commons.io.output.CountingOutputStream; import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.DefaultBlockWriter; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; -public class InMemoryShard extends AbstractShard { +public class InMemoryShard extends AbstractShard { /* Map of a hash of the DataBlocks `gridPosition` to the block */ private final HashMap> blocks; @@ -27,14 +27,14 @@ public class InMemoryShard extends AbstractShard { * (later) */ - public InMemoryShard(final ShardedDatasetAttributes datasetAttributes, final long[] shardPosition) { + public InMemoryShard(final A datasetAttributes, final long[] shardPosition) { this( datasetAttributes, shardPosition, null); indexBuilder = new ShardIndexBuilder(this); indexBuilder.indexLocation(datasetAttributes.getIndexLocation()); } - public InMemoryShard(final ShardedDatasetAttributes datasetAttributes, final long[] gridPosition, + public InMemoryShard(final A datasetAttributes, final long[] gridPosition, ShardIndex index) { super(datasetAttributes, gridPosition, index); @@ -97,28 +97,29 @@ public void write(final OutputStream out) throws IOException { writeShardStart(out, this); } - public static void writeShard(final OutputStream out, final Shard shard) throws IOException { + public static void writeShard(final OutputStream out, final Shard shard) throws IOException { fromShard(shard).write(out); } - public static InMemoryShard fromShard(Shard shard) { + public static InMemoryShard fromShard(Shard shard) { if (shard instanceof InMemoryShard) - return (InMemoryShard) shard; + return (InMemoryShard) shard; - final InMemoryShard inMemoryShard = new InMemoryShard(shard.getDatasetAttributes(), + final InMemoryShard inMemoryShard = new InMemoryShard( + shard.getDatasetAttributes(), shard.getGridPosition()); shard.forEach(blk -> inMemoryShard.addBlock(blk)); return inMemoryShard; } - protected static void writeShardEndStream( + protected static void writeShardEndStream( final OutputStream out, - InMemoryShard shard ) throws IOException { + InMemoryShard shard ) throws IOException { - final ShardedDatasetAttributes datasetAttributes = shard.getDatasetAttributes(); + final A datasetAttributes = shard.getDatasetAttributes(); final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); indexBuilder.indexLocation(IndexLocation.END); @@ -145,11 +146,11 @@ protected static void writeShardEndStream( ShardIndex.write(indexBuilder.build(), out); } - protected static void writeShardEnd( + protected static void writeShardEnd( final OutputStream out, - InMemoryShard shard ) throws IOException { + InMemoryShard shard ) throws IOException { - final ShardedDatasetAttributes datasetAttributes = shard.getDatasetAttributes(); + final A datasetAttributes = shard.getDatasetAttributes(); final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); indexBuilder.indexLocation(IndexLocation.END); @@ -166,11 +167,11 @@ protected static void writeShardEnd( ShardIndex.write(indexBuilder.build(), out); } - protected static void writeShardStart( + protected static void writeShardStart( final OutputStream out, - InMemoryShard shard ) throws IOException { + InMemoryShard shard ) throws IOException { - final ShardedDatasetAttributes datasetAttributes = shard.getDatasetAttributes(); + final A datasetAttributes = shard.getDatasetAttributes(); final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); indexBuilder.indexLocation(IndexLocation.START); indexBuilder.setCodecs(datasetAttributes.getShardingCodec().getIndexCodecs()); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index eef1c9604..3e672ba58 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -4,10 +4,10 @@ import java.util.Iterator; import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.util.GridIterator; -public interface Shard extends Iterable> { +public interface Shard extends Iterable> { long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; @@ -30,7 +30,7 @@ default int[] getBlockGridSize() { return blockGridSize; } - public ShardedDatasetAttributes getDatasetAttributes(); + public A getDatasetAttributes(); /** * Returns the size of shards in pixel units. @@ -113,12 +113,12 @@ default DataBlock[] getAllBlocks(long... position) { public ShardIndex getIndex(); - public static Shard createEmpty(final ShardedDatasetAttributes attributes, long... shardPosition) { + public static Shard createEmpty(final A attributes, long... shardPosition) { final long[] emptyIndex = new long[(int)(2 * attributes.getNumBlocks())]; Arrays.fill(emptyIndex, EMPTY_INDEX_NBYTES); final ShardIndex shardIndex = new ShardIndex(attributes.getBlocksPerShard(), emptyIndex, ShardingCodec.IndexLocation.END); - return new InMemoryShard(attributes, shardPosition, shardIndex); + return new InMemoryShard(attributes, shardPosition, shardIndex); } public static long flatIndex(long[] gridPosition, int[] gridSize) { @@ -135,9 +135,9 @@ public static long flatIndex(long[] gridPosition, int[] gridSize) { public static class DataBlockIterator implements Iterator> { private final GridIterator it; - private final Shard shard; + private final Shard shard; - public DataBlockIterator(final Shard shard) { + public DataBlockIterator(final Shard shard) { this.shard = shard; it = new GridIterator(shard.getBlockGridSize()); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java index cf16f7190..f6f81dabf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java @@ -7,7 +7,7 @@ public class ShardIndexBuilder { - private final Shard shard; + private final Shard shard; private ShardIndex temporaryIndex; @@ -17,7 +17,7 @@ public class ShardIndexBuilder { private long currentOffset = 0; - public ShardIndexBuilder(Shard shard) { + public ShardIndexBuilder(Shard shard) { this.shard = shard; this.temporaryIndex = new ShardIndex(shard.getBlockGridSize(), location); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java new file mode 100644 index 000000000..055666b43 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java @@ -0,0 +1,140 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.BlockParameters; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; + +public interface ShardParameters extends BlockParameters { + + public ShardingCodec getShardingCodec(); + + /** + * The size of the blocks in pixel units. + * + * @return the number of pixels per dimension for this shard. + */ + public int[] getShardSize(); + + public IndexLocation getIndexLocation(); + + public ShardIndex createIndex(); + + /** + * Returns the number of blocks per dimension for a shard. + * + * @return the size of the block grid of a shard + */ + default int[] getBlocksPerShard() { + + final int nd = getNumDimensions(); + final int[] blocksPerShard = new int[nd]; + final int[] blockSize = getBlockSize(); + for (int i = 0; i < nd; i++) + blocksPerShard[i] = getShardSize()[i] / blockSize[i]; + + return blocksPerShard; + } + + /** + * Given a block's position relative to the array, returns the position of the shard containing that block relative to the shard grid. + * + * @param blockGridPosition + * position of a block relative to the array + * @return the position of the containing shard in the shard grid + */ + default long[] getShardPositionForBlock(final long... blockGridPosition) { + + final int[] blocksPerShard = getBlocksPerShard(); + final long[] shardGridPosition = new long[blockGridPosition.length]; + for (int i = 0; i < shardGridPosition.length; i++) { + shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / blocksPerShard[i]); + } + + return shardGridPosition; + } + + /** + * Returns the number of shards per dimension for the dataset. + * + * @return the size of the shard grid of a dataset + */ + default int[] getShardBlockGridSize() { + + final int nd = getNumDimensions(); + final int[] shardBlockGridSize = new int[nd]; + final int[] blockSize = getBlockSize(); + for (int i = 0; i < nd; i++) + shardBlockGridSize[i] = (int)(Math.ceil((double)getDimensions()[i] / blockSize[i])); + + return shardBlockGridSize; + } + + /** + * Returns the block at the given position relative to this shard, or null if this shard does not contain the given block. + * + * @return the block position + */ + default long[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { + + // TODO check correctness + final long[] shardPos = getShardPositionForBlock(blockPosition); + if (!Arrays.equals(shardPosition, shardPos)) + return null; + + final int[] shardSize = getBlocksPerShard(); + final long[] blockShardPos = new long[shardSize.length]; + for (int i = 0; i < shardSize.length; i++) { + blockShardPos[i] = blockPosition[i] % shardSize[i]; + } + + return blockShardPos; + } + + + /** + * Given a block's position relative to a shard, returns its position in pixels + * relative to the image. + * + * @return the block position + */ + default long[] getBlockMinFromShardPosition(final long[] shardPosition, final long[] blockPosition) { + + // is this useful? + final int[] blockSize = getBlockSize(); + final int[] shardSize = getShardSize(); + final long[] blockImagePos = new long[shardSize.length]; + for (int i = 0; i < shardSize.length; i++) { + blockImagePos[i] = (shardPosition[i] * shardSize[i]) + (blockPosition[i] * blockSize[i]); + } + + return blockImagePos; + } + + /** + * Given a block's position relative to a shard, returns its position relative + * to the image. + * + * @return the block position + */ + default long[] getBlockPositionFromShardPosition(final long[] shardPosition, final long[] blockPosition) { + + // is this useful? + final int[] shardBlockSize = getBlocksPerShard(); + final long[] blockImagePos = new long[getNumDimensions()]; + for (int i = 0; i < getNumDimensions(); i++) { + blockImagePos[i] = (shardPosition[i] * shardBlockSize[i]) + (blockPosition[i]); + } + + return blockImagePos; + } + + /** + * @return the number of blocks per shard + */ + default long getNumBlocks() { + + return Arrays.stream(getBlocksPerShard()).reduce(1, (x, y) -> x * y); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java index 792019fdc..b8596c8db 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java @@ -1,8 +1,8 @@ package org.janelia.saalfeldlab.n5.shard; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.DefaultBlockWriter; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; @@ -13,13 +13,13 @@ import java.util.Arrays; import java.util.List; -public class ShardWriter { +public class ShardWriter { private static final int BYTES_PER_LONG = 8; private final List> blocks; - private ShardedDatasetAttributes attributes; + private A attributes; private ByteBuffer blockSizes; @@ -29,7 +29,7 @@ public class ShardWriter { private List blockBytes; - public ShardWriter(final ShardedDatasetAttributes datasetAttributes) { + public ShardWriter(final A datasetAttributes) { blocks = new ArrayList<>(); attributes = datasetAttributes; @@ -48,7 +48,7 @@ public void addBlock(final DataBlock block) { blocks.add(block); } - public void write(final Shard shard, final OutputStream out) throws IOException { + public void write(final Shard shard, final OutputStream out) throws IOException { attributes = shard.getDatasetAttributes(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 7765d42be..56c928712 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -6,20 +6,20 @@ import java.io.UncheckedIOException; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.DefaultBlockReader; import org.janelia.saalfeldlab.n5.DefaultBlockWriter; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; -public class VirtualShard extends AbstractShard { +public class VirtualShard extends AbstractShard { final private KeyValueAccess keyValueAccess; final private String path; - public VirtualShard(final ShardedDatasetAttributes datasetAttributes, long[] gridPosition, + public VirtualShard(final A datasetAttributes, long[] gridPosition, final KeyValueAccess keyValueAccess, final String path) { super(datasetAttributes, gridPosition, null); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 2a39e161f..8e12ac075 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -191,7 +191,7 @@ public void writeReadShardTest() { final HashMap writtenBlocks = new HashMap<>(); - final InMemoryShard shard = new InMemoryShard(datasetAttributes, new long[]{0, 0}); + final InMemoryShard shard = new InMemoryShard<>(datasetAttributes, new long[]{0, 0}); for (int idx1 = 1; idx1 >= 0; idx1--) { for (int idx2 = 1; idx2 >= 0; idx2--) { From b0092d315567f9e477203376f599e04a4b3e2dcc Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 7 Jan 2025 09:53:31 -0500 Subject: [PATCH 088/423] fix: null compression should result in empty byteCodecs list --- src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 684339a27..eca190d96 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -73,7 +73,7 @@ public DatasetAttributes( this.blockSize = blockSize; this.dataType = dataType; if (codecs == null && !(compression instanceof RawCompression)) { - byteCodecs = new BytesCodec[]{compression}; + byteCodecs = new BytesCodec[]{}; arrayCodec = new N5BlockCodec(); } else if (codecs == null || codecs.length == 0) { byteCodecs = new BytesCodec[]{}; From 458295cd7e1bc7b783d3d91bf057e8aca3f6a9b7 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 7 Jan 2025 09:54:08 -0500 Subject: [PATCH 089/423] refactor: createIndex now a default method in ShardParameters --- .../org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java | 5 ----- .../org/janelia/saalfeldlab/n5/shard/ShardParameters.java | 4 +++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index a42025295..04a8a0371 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -102,11 +102,6 @@ public IndexLocation getIndexLocation() { return getShardingCodec().getIndexLocation(); } - @Override - public ShardIndex createIndex() { - return new ShardIndex(getBlocksPerShard(), getIndexLocation(), getShardingCodec().getIndexCodecs()); - } - /** * The size of the blocks in pixel units. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java index 055666b43..fad791309 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java @@ -18,7 +18,9 @@ public interface ShardParameters extends BlockParameters { public IndexLocation getIndexLocation(); - public ShardIndex createIndex(); + default ShardIndex createIndex() { + return new ShardIndex(getBlocksPerShard(), getIndexLocation(), getShardingCodec().getIndexCodecs()); + } /** * Returns the number of blocks per dimension for a shard. From da9b9eddb19549b45881736e1c5af1f530350a93 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 7 Jan 2025 09:54:27 -0500 Subject: [PATCH 090/423] feat: add getByteOrder method for ArrayCodecs --- .../java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java | 4 ++++ .../java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java index 76b015cf1..66e1a8b21 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java @@ -44,6 +44,10 @@ public BytesCodec(final ByteOrder byteOrder) { this.byteOrder = byteOrder; } + public ByteOrder getByteOrder() { + return byteOrder; + } + @Override public DataBlockInputStream decode(final DatasetAttributes attributes, final long[] gridPosition, InputStream in) throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java index 82f118bb7..7232d9aca 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java @@ -37,6 +37,9 @@ public N5BlockCodec(final ByteOrder byteOrder) { this.byteOrder = byteOrder; } + public ByteOrder getByteOrder() { + return byteOrder; + } @Override public DataBlockInputStream decode(final DatasetAttributes attributes, final long[] gridPosition, InputStream in) throws IOException { From 52f762eeb9bbef3f57f95f1545c363cd22ca0d90 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 8 Jan 2025 10:05:48 -0500 Subject: [PATCH 091/423] feat: writeBlocks respects existing blocks in a given shard if not overwriting those blocks explicitly --- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 9 ++- .../saalfeldlab/n5/shard/InMemoryShard.java | 1 + .../janelia/saalfeldlab/n5/shard/Shard.java | 22 ++++++- .../saalfeldlab/n5/shard/ShardIndex.java | 33 +++++++--- .../saalfeldlab/n5/shard/ShardTest.java | 66 ++++++++++++++++--- 5 files changed, 109 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index e3ceb7593..faa7cf535 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -232,6 +232,14 @@ default boolean removeAttributes(final String pathName, final List attri } for (InMemoryShard shard : shardBlockMap.values()) { + + /* Add existing blocks before overwriting shard */ + final Shard currentShard = (Shard)getShard(datasetPath, shardAttributes, shard.getGridPosition()); + for (DataBlock currentBlock : currentShard.getBlocks()) { + if (shard.getBlock(currentBlock.getGridPosition()) == null) + shard.addBlock(currentBlock); + } + writeShard(datasetPath, shardAttributes, shard); } } else { @@ -240,7 +248,6 @@ default boolean removeAttributes(final String pathName, final List attri writeBlock(datasetPath, datasetAttributes, dataBlock); } } - } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 7e699a1b0..4ac202bf5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -67,6 +67,7 @@ public int numBlocks() { return blocks.size(); } + @Override public List> getBlocks() { return new ArrayList<>(blocks.values()); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index eef1c9604..5a88f370f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -1,7 +1,9 @@ package org.janelia.saalfeldlab.n5.shard; +import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; +import java.util.List; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; @@ -106,9 +108,23 @@ default Iterator> iterator() { return new DataBlockIterator(this); } - default DataBlock[] getAllBlocks(long... position) { - //TODO Caleb: Do we want this? - return null; + default List> getBlocks() { + + final ShardIndex shardIndex = getIndex(); + final ShardedDatasetAttributes attrs = getDatasetAttributes(); + final List> blocks = new ArrayList<>(); + for (long blockIdx = 0; blockIdx < attrs.getNumBlocks(); blockIdx++) { + int shardOffset = (int)blockIdx * 2; + final long[] index = shardIndex.getData(); + if (index[shardOffset] == Shard.EMPTY_INDEX_NBYTES || index[shardOffset+1] == EMPTY_INDEX_NBYTES) + continue; + + final long[] blockPosInShard = ShardIndex.shardPositionFromIndexOffset(shardOffset, attrs.getBlocksPerShard()); + final long[] blockPosInImg = attrs.getBlockPositionFromShardPosition(getGridPosition(), blockPosInShard); + blocks.add(getBlock(blockPosInImg)); + } + + return blocks; } public ShardIndex getIndex(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 561e9ea75..44bca3c5c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -236,18 +236,31 @@ private static int[] prepend(final int value, final int[] array) { return indexBlockSize; } - public static void main(String[] args) { - - final ShardIndex ib = new ShardIndex(new int[]{2, 2}); + /** + * Calculate the block position in the shard grid for a given index offset. + * + * @param offset the offset into the index + * @param blocksPerShard the dimensions of the shard in blocks + * @return the relative position in the shard grid + */ + public static long[] shardPositionFromIndexOffset(int offset, int[] blocksPerShard) { + + int maxOffset = 1; + for (int i = 0; i < blocksPerShard.length; i++) { + maxOffset *= blocksPerShard[i]; + } + if (offset >= maxOffset*2) { + throw new IllegalArgumentException("Shard Index Offset " + offset + " is out of bounds for shard dimensions " + Arrays.toString(blocksPerShard)); + } - ib.set(8, 9, new long[]{1, 1}); + final long[] position = new long[blocksPerShard.length]; + int remainder = offset / 2; - // System.out.println(ib.getIndex(0, 0)); - // System.out.println(ib.getIndex(1, 0)); - // System.out.println(ib.getIndex(0, 1)); - // System.out.println(ib.getIndex(1, 1)); + for (int dim = blocksPerShard.length - 1; dim >= 0; dim--) { // Iterate backwards + position[dim] = remainder % blocksPerShard[dim]; // Calculate position for this dimension + remainder /= blocksPerShard[dim]; // Update the remainder + } - System.out.println("done"); + return position; } - } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 2a39e161f..c5ab89cbc 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -69,7 +69,7 @@ private ShardedDatasetAttributes getTestAttributes(long[] dimensions, int[] shar shardSize, blockSize, DataType.UINT8, - new Codec[]{new N5BlockCodec(dataByteOrder) , new GzipCompression(4)}, + new Codec[]{new N5BlockCodec(dataByteOrder)}, // , new GzipCompression(4)}, new DeterministicSizeCodec[]{new BytesCodec(indexByteOrder), new Crc32cChecksumCodec()}, indexLocation ); @@ -99,6 +99,7 @@ public void writeReadBlocksTest() { data[i] = (byte)((100) + (10) + i); } + writer.writeBlocks( "shard", datasetAttributes, @@ -117,15 +118,64 @@ public void writeReadBlocksTest() { ); final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); - String p = writer.getURI().getPath(); - final String shard00 = kva.compose(writer.getURI(), "shard", "0", "0"); - kva.exists(shard00); - final String shard10 = kva.compose(writer.getURI(), "shard", "0", "0"); - kva.exists(shard10); + final String[][] keys = new String[][]{ + {"shard", "0", "0"}, + {"shard", "1", "0"}, + {"shard", "2", "2"} + }; + for (String[] key : keys) { + final String shard = kva.compose(writer.getURI(), key); + Assert.assertTrue("Shard at" + Arrays.toString(key) + "Does not exist", kva.exists(shard)); + } + + final long[][] blockIndices = new long[][]{ {0,0}, {0,1}, {1,0}, {1,1}, {4,0}, {5,0}, {11,11}}; + for (long[] blockIndex : blockIndices) { + final DataBlock block = writer.readBlock("shard", datasetAttributes, blockIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + } + + final byte[] data2 = new byte[numElements]; + for (int i = 0; i < data2.length; i++) { + data2[i] = (byte)(10 + i); + } + writer.writeBlocks( + "shard", + datasetAttributes, + /* shard (0, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{0,0}, data2), + new ByteArrayDataBlock(blockSize, new long[]{1,1}, data2), - final String shard33 = kva.compose(writer.getURI(), "shard", "0", "0"); - kva.exists(shard33); + /* shard (0, 1) */ + new ByteArrayDataBlock(blockSize, new long[]{0,4}, data2), + new ByteArrayDataBlock(blockSize, new long[]{0,5}, data2), + + /* shard (2, 2) */ + new ByteArrayDataBlock(blockSize, new long[]{10,10}, data2) + ); + + final String[][] keys2 = new String[][]{ + {"shard", "0", "0"}, + {"shard", "1", "0"}, + {"shard", "0", "1"}, + {"shard", "2", "2"} + }; + for (String[] key : keys2) { + final String shard = kva.compose(writer.getURI(), key); + Assert.assertTrue("Shard at" + Arrays.toString(key) + "Does not exist", kva.exists(shard)); + } + + final long[][] oldBlockIndices = new long[][]{{0,1}, {1,0}, {4,0}, {5,0}, {11,11}}; + for (long[] blockIndex : oldBlockIndices) { + final DataBlock block = writer.readBlock("shard", datasetAttributes, blockIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + } + + final long[][] newBlockIndices = new long[][]{{0,0}, {1,1}, {0,4}, {0,5}, {10,10}}; + for (long[] blockIndex : newBlockIndices) { + final DataBlock block = writer.readBlock("shard", datasetAttributes, blockIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])block.getData()); + } } @Test From d4dcbe8ae72f23374a9b79de848a82b3ab01226f Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 8 Jan 2025 11:57:25 -0500 Subject: [PATCH 092/423] refactor: remove generic from ShardParameter --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 4 ++-- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 10 ++++----- .../org/janelia/saalfeldlab/n5/N5Writer.java | 2 +- .../saalfeldlab/n5/shard/AbstractShard.java | 13 ++++++------ .../saalfeldlab/n5/shard/InMemoryShard.java | 21 ++++++++++--------- .../janelia/saalfeldlab/n5/shard/Shard.java | 13 ++++++------ .../n5/shard/ShardIndexBuilder.java | 4 ++-- .../saalfeldlab/n5/shard/ShardWriter.java | 19 ++++++++++------- .../saalfeldlab/n5/shard/VirtualShard.java | 9 ++++---- .../saalfeldlab/n5/shard/ShardTest.java | 2 +- 10 files changed, 52 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 8b0c49cb3..d8165146e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -90,7 +90,7 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { } @SuppressWarnings("rawtypes") - default Shard getShard(final String pathName, + default Shard getShard(final String pathName, final A datasetAttributes, long... shardGridPosition) { @@ -107,7 +107,7 @@ default DataBlock readBlock( if (datasetAttributes instanceof ShardedDatasetAttributes) { final ShardedDatasetAttributes shardedAttrs = (ShardedDatasetAttributes) datasetAttributes; final long[] shardPosition = shardedAttrs.getShardPositionForBlock(gridPosition); - final Shard shard = getShard(pathName, shardedAttrs, shardPosition); + final Shard shard = getShard(pathName, shardedAttrs, shardPosition); return shard.getBlock(gridPosition); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index e044c6bc7..2ece8d052 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -221,7 +221,7 @@ default boolean removeAttributes(final String pathName, final List attri if (datasetAttributes instanceof ShardParameters) { /* Group by shard index */ - final HashMap> shardBlockMap = new HashMap<>(); + final HashMap> shardBlockMap = new HashMap<>(); final ShardParameters shardAttributes = (ShardParameters)datasetAttributes; for (DataBlock dataBlock : dataBlocks) { @@ -230,11 +230,11 @@ default boolean removeAttributes(final String pathName, final List attri if (!shardBlockMap.containsKey(shardHash)) shardBlockMap.put(shardHash, new InMemoryShard<>((DatasetAttributes & ShardParameters)shardAttributes, shardPosition)); - final InMemoryShard shard = shardBlockMap.get(shardHash); + final InMemoryShard shard = shardBlockMap.get(shardHash); shard.addBlock(dataBlock); } - for (InMemoryShard shard : shardBlockMap.values()) { + for (InMemoryShard shard : shardBlockMap.values()) { writeShard(datasetPath, (DatasetAttributes & ShardParameters)shardAttributes, (Shard)shard); } @@ -257,7 +257,7 @@ default void writeBlock( ShardParameters shardDatasetAttrs = (ShardParameters)datasetAttributes; final long[] shardPos = shardDatasetAttrs.getShardPositionForBlock(dataBlock.getGridPosition()); final String shardPath = absoluteShardPath(N5URI.normalizeGroupPath(path), shardPos); - final VirtualShard shard = new VirtualShard<>((DatasetAttributes & ShardParameters)shardDatasetAttrs, shardPos, getKeyValueAccess(), shardPath); + final VirtualShard shard = new VirtualShard<>((DatasetAttributes & ShardParameters)shardDatasetAttrs, shardPos, getKeyValueAccess(), shardPath); shard.writeBlock(dataBlock); return; } @@ -278,7 +278,7 @@ default void writeBlock( default void writeShard( final String path, final A datasetAttributes, - final Shard shard) throws N5Exception { + final Shard shard) throws N5Exception { final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shard.getGridPosition()); try (final LockedChannel lock = getKeyValueAccess().lockForWriting(shardPath)) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 463d232a3..74df1da32 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -315,7 +315,7 @@ default void writeBlocks( void writeShard( final String datasetPath, final A datasetAttributes, - final Shard shard) throws N5Exception; + final Shard shard) throws N5Exception; /** * Deletes the block at {@code gridPosition} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java index 5ecc67c70..2cdb392f2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java @@ -2,15 +2,16 @@ import org.janelia.saalfeldlab.n5.DatasetAttributes; -public abstract class AbstractShard implements Shard { - protected final A datasetAttributes; +public abstract class AbstractShard implements Shard { + + protected final DatasetAttributes datasetAttributes; protected ShardIndex index; private final long[] gridPosition; - public AbstractShard(final A datasetAttributes, final long[] gridPosition, + public AbstractShard(final A datasetAttributes, final long[] gridPosition, final ShardIndex index) { this.datasetAttributes = datasetAttributes; @@ -19,15 +20,15 @@ public AbstractShard(final A datasetAttributes, final long[] gridPosition, } @Override - public A getDatasetAttributes() { + public A getDatasetAttributes() { - return datasetAttributes; + return (A)datasetAttributes; } @Override public int[] getSize() { - return datasetAttributes.getShardSize(); + return getDatasetAttributes().getShardSize(); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 36a112e2c..164f51167 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -10,12 +10,13 @@ import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.io.output.CountingOutputStream; import org.apache.commons.io.output.ProxyOutputStream; +import org.checkerframework.checker.units.qual.A; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.DefaultBlockWriter; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; -public class InMemoryShard extends AbstractShard { +public class InMemoryShard extends AbstractShard { /* Map of a hash of the DataBlocks `gridPosition` to the block */ private final HashMap> blocks; @@ -27,14 +28,14 @@ public class InMemoryShard exte * (later) */ - public InMemoryShard(final A datasetAttributes, final long[] shardPosition) { + public InMemoryShard(final A datasetAttributes, final long[] shardPosition) { this( datasetAttributes, shardPosition, null); indexBuilder = new ShardIndexBuilder(this); indexBuilder.indexLocation(datasetAttributes.getIndexLocation()); } - public InMemoryShard(final A datasetAttributes, final long[] gridPosition, + public InMemoryShard(final A datasetAttributes, final long[] gridPosition, ShardIndex index) { super(datasetAttributes, gridPosition, index); @@ -97,17 +98,17 @@ public void write(final OutputStream out) throws IOException { writeShardStart(out, this); } - public static void writeShard(final OutputStream out, final Shard shard) throws IOException { + public static void writeShard(final OutputStream out, final Shard shard) throws IOException { fromShard(shard).write(out); } - public static InMemoryShard fromShard(Shard shard) { + public static InMemoryShard fromShard(Shard shard) { if (shard instanceof InMemoryShard) - return (InMemoryShard) shard; + return (InMemoryShard) shard; - final InMemoryShard inMemoryShard = new InMemoryShard( + final InMemoryShard inMemoryShard = new InMemoryShard( shard.getDatasetAttributes(), shard.getGridPosition()); @@ -117,7 +118,7 @@ public static InMemoryShard void writeShardEndStream( final OutputStream out, - InMemoryShard shard ) throws IOException { + InMemoryShard shard ) throws IOException { final A datasetAttributes = shard.getDatasetAttributes(); @@ -148,7 +149,7 @@ protected static void writeSh protected static void writeShardEnd( final OutputStream out, - InMemoryShard shard ) throws IOException { + InMemoryShard shard ) throws IOException { final A datasetAttributes = shard.getDatasetAttributes(); @@ -169,7 +170,7 @@ protected static void writeSh protected static void writeShardStart( final OutputStream out, - InMemoryShard shard ) throws IOException { + InMemoryShard shard ) throws IOException { final A datasetAttributes = shard.getDatasetAttributes(); final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 3e672ba58..5407154a0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -3,11 +3,12 @@ import java.util.Arrays; import java.util.Iterator; +import org.checkerframework.checker.units.qual.A; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.util.GridIterator; -public interface Shard extends Iterable> { +public interface Shard extends Iterable> { long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; @@ -30,7 +31,7 @@ default int[] getBlockGridSize() { return blockGridSize; } - public A getDatasetAttributes(); + public A getDatasetAttributes(); /** * Returns the size of shards in pixel units. @@ -113,12 +114,12 @@ default DataBlock[] getAllBlocks(long... position) { public ShardIndex getIndex(); - public static Shard createEmpty(final A attributes, long... shardPosition) { + public static Shard createEmpty(final A attributes, long... shardPosition) { final long[] emptyIndex = new long[(int)(2 * attributes.getNumBlocks())]; Arrays.fill(emptyIndex, EMPTY_INDEX_NBYTES); final ShardIndex shardIndex = new ShardIndex(attributes.getBlocksPerShard(), emptyIndex, ShardingCodec.IndexLocation.END); - return new InMemoryShard(attributes, shardPosition, shardIndex); + return new InMemoryShard(attributes, shardPosition, shardIndex); } public static long flatIndex(long[] gridPosition, int[] gridSize) { @@ -135,9 +136,9 @@ public static long flatIndex(long[] gridPosition, int[] gridSize) { public static class DataBlockIterator implements Iterator> { private final GridIterator it; - private final Shard shard; + private final Shard shard; - public DataBlockIterator(final Shard shard) { + public DataBlockIterator(final Shard shard) { this.shard = shard; it = new GridIterator(shard.getBlockGridSize()); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java index f6f81dabf..cf16f7190 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java @@ -7,7 +7,7 @@ public class ShardIndexBuilder { - private final Shard shard; + private final Shard shard; private ShardIndex temporaryIndex; @@ -17,7 +17,7 @@ public class ShardIndexBuilder { private long currentOffset = 0; - public ShardIndexBuilder(Shard shard) { + public ShardIndexBuilder(Shard shard) { this.shard = shard; this.temporaryIndex = new ShardIndex(shard.getBlockGridSize(), location); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java index b8596c8db..bff0f976b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.shard; +import org.checkerframework.checker.units.qual.A; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.DefaultBlockWriter; @@ -13,13 +14,13 @@ import java.util.Arrays; import java.util.List; -public class ShardWriter { +public class ShardWriter { private static final int BYTES_PER_LONG = 8; private final List> blocks; - private A attributes; + private DatasetAttributes attributes; private ByteBuffer blockSizes; @@ -29,12 +30,17 @@ public class ShardWriter { private List blockBytes; - public ShardWriter(final A datasetAttributes) { + public ShardWriter(final A datasetAttributes) { blocks = new ArrayList<>(); attributes = datasetAttributes; } + public A getAttributes() { + + return (A)attributes; + } + public void reset() { blocks.clear(); @@ -48,12 +54,10 @@ public void addBlock(final DataBlock block) { blocks.add(block); } - public void write(final Shard shard, final OutputStream out) throws IOException { - - attributes = shard.getDatasetAttributes(); + public void write(final Shard shard, final OutputStream out) throws IOException { prepareForWritingDataBlock(); - if (attributes.getIndexLocation() == ShardingCodec.IndexLocation.START) { + if (shard.getDatasetAttributes().getIndexLocation() == ShardingCodec.IndexLocation.START) { writeIndexBlock(out); writeBlocks(out); } else { @@ -67,7 +71,6 @@ private void prepareForWritingDataBlock() throws IOException { // final ShardingProperties shardProps = new ShardingProperties(datasetAttributes); // indexData = new ShardIndexDataBlock(shardProps.getIndexDimensions()); - indexData = attributes.createIndex(); blockBytes = new ArrayList<>(); long cumulativeBytes = 0; final long[] shardPosition = new long[1]; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 56c928712..86b87947a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -5,6 +5,7 @@ import java.io.OutputStream; import java.io.UncheckedIOException; +import org.checkerframework.checker.units.qual.A; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.DefaultBlockReader; @@ -14,12 +15,12 @@ import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -public class VirtualShard extends AbstractShard { +public class VirtualShard extends AbstractShard { final private KeyValueAccess keyValueAccess; final private String path; - public VirtualShard(final A datasetAttributes, long[] gridPosition, + public VirtualShard(final A datasetAttributes, long[] gridPosition, final KeyValueAccess keyValueAccess, final String path) { super(datasetAttributes, gridPosition, null); @@ -95,14 +96,14 @@ public void writeBlock(final DataBlock block) { public ShardIndex createIndex() { // Empty index of the correct size - return datasetAttributes.createIndex(); + return getDatasetAttributes().createIndex(); } @Override public ShardIndex getIndex() { try { - final ShardIndex readIndex = ShardIndex.read(keyValueAccess, path, datasetAttributes.createIndex()); + final ShardIndex readIndex = ShardIndex.read(keyValueAccess, path, getDatasetAttributes().createIndex()); index = readIndex == null ? createIndex() : readIndex; } catch (final N5Exception.N5NoSuchKeyException e) { index = createIndex(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 8e12ac075..31f14498d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -191,7 +191,7 @@ public void writeReadShardTest() { final HashMap writtenBlocks = new HashMap<>(); - final InMemoryShard shard = new InMemoryShard<>(datasetAttributes, new long[]{0, 0}); + final InMemoryShard shard = new InMemoryShard<>(datasetAttributes, new long[]{0, 0}); for (int idx1 = 1; idx1 >= 0; idx1--) { for (int idx2 = 1; idx2 >= 0; idx2--) { From b2b3d2ba2e5dff263556b857fe0ef5514bb58fa0 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 8 Jan 2025 11:59:36 -0500 Subject: [PATCH 093/423] refactor: some signatures --- .../n5/ShardedDatasetAttributes.java | 10 ++-- .../janelia/saalfeldlab/n5/shard/Shard.java | 10 ++-- .../saalfeldlab/n5/shard/ShardIndex.java | 51 ++++++++++--------- .../n5/shard/ShardIndexBuilder.java | 2 +- .../saalfeldlab/n5/shard/ShardReader.java | 11 +--- .../saalfeldlab/n5/shard/ShardWriter.java | 5 +- .../saalfeldlab/n5/shard/VirtualShard.java | 4 +- 7 files changed, 43 insertions(+), 50 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index 8ee40baf4..089e332d9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -164,7 +164,7 @@ public long[] getShardPositionForBlock(final long... blockGridPosition) { * * @return the block position */ - public long[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { + public int[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { // TODO check correctness final long[] shardPos = getShardPositionForBlock(blockPosition); @@ -172,9 +172,9 @@ public long[] getBlockPositionInShard(final long[] shardPosition, final long[] b return null; final int[] shardSize = getBlocksPerShard(); - final long[] blockShardPos = new long[shardSize.length]; + final int[] blockShardPos = new int[shardSize.length]; for (int i = 0; i < shardSize.length; i++) { - blockShardPos[i] = blockPosition[i] % shardSize[i]; + blockShardPos[i] = (int)(blockPosition[i] % shardSize[i]); } return blockShardPos; @@ -205,7 +205,7 @@ public long[] getBlockMinFromShardPosition(final long[] shardPosition, final lon * * @return the block position */ - public long[] getBlockPositionFromShardPosition(final long[] shardPosition, final long[] blockPosition) { + public long[] getBlockPositionFromShardPosition(final long[] shardPosition, final int[] blockPosition) { // is this useful? final int[] shardBlockSize = getBlocksPerShard(); @@ -220,7 +220,7 @@ public long[] getBlockPositionFromShardPosition(final long[] shardPosition, fina /** * @return the number of blocks per shard */ - public long getNumBlocks() { + public int getNumBlocks() { return Arrays.stream(getBlocksPerShard()).reduce(1, (x, y) -> x * y); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 5a88f370f..211ad69e6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -62,7 +62,7 @@ default int[] getBlockGridSize() { * * @return the shard position */ - default long[] getBlockPosition(long... blockPosition) { + default int[] getBlockPosition(long... blockPosition) { final long[] shardPos = getDatasetAttributes().getShardPositionForBlock(blockPosition); return getDatasetAttributes().getBlockPositionInShard(shardPos, blockPosition); @@ -113,13 +113,11 @@ default List> getBlocks() { final ShardIndex shardIndex = getIndex(); final ShardedDatasetAttributes attrs = getDatasetAttributes(); final List> blocks = new ArrayList<>(); - for (long blockIdx = 0; blockIdx < attrs.getNumBlocks(); blockIdx++) { - int shardOffset = (int)blockIdx * 2; - final long[] index = shardIndex.getData(); - if (index[shardOffset] == Shard.EMPTY_INDEX_NBYTES || index[shardOffset+1] == EMPTY_INDEX_NBYTES) + for (int blockIdx = 0; blockIdx < attrs.getNumBlocks(); blockIdx++) { + if (!shardIndex.exists(blockIdx)) continue; - final long[] blockPosInShard = ShardIndex.shardPositionFromIndexOffset(shardOffset, attrs.getBlocksPerShard()); + final int[] blockPosInShard = ShardIndex.blockPosition(blockIdx, attrs.getBlocksPerShard()); final long[] blockPosInImg = attrs.getBlockPositionFromShardPosition(getGridPosition(), blockPosInShard); blocks.add(getBlock(blockPosInImg)); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 44bca3c5c..a91c470c8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -52,35 +52,41 @@ public ShardIndex(int[] shardBlockGridSize, DeterministicSizeCodec... codecs) { this(shardBlockGridSize, emptyIndexData(shardBlockGridSize), IndexLocation.END, codecs); } - public boolean exists(long... gridPosition) { + public boolean exists(int[] gridPosition) { - return getOffset(gridPosition) != Shard.EMPTY_INDEX_NBYTES && + return getOffset(gridPosition) != Shard.EMPTY_INDEX_NBYTES || getNumBytes(gridPosition) != Shard.EMPTY_INDEX_NBYTES; } + public boolean exists(int blockNum) { + + return data[blockNum * 2] != Shard.EMPTY_INDEX_NBYTES || + data[blockNum * 2 + 1] != Shard.EMPTY_INDEX_NBYTES; + } + public IndexLocation getLocation() { return location; } - public long getOffset(long... gridPosition) { + public long getOffset(int... gridPosition) { return data[getOffsetIndex(gridPosition)]; } - public long getNumBytes(long... gridPosition) { + public long getNumBytes(int... gridPosition) { return data[getNumBytesIndex(gridPosition)]; } - public void set(long offset, long nbytes, long[] gridPosition) { + public void set(long offset, long nbytes, int[] gridPosition) { final int i = getOffsetIndex(gridPosition); data[i] = offset; data[i + 1] = nbytes; } - private int getOffsetIndex(long... gridPosition) { + private int getOffsetIndex(int... gridPosition) { int idx = (int) gridPosition[0]; for (int i = 1; i < gridPosition.length; i++) { @@ -89,7 +95,7 @@ private int getOffsetIndex(long... gridPosition) { return idx * 2; } - private int getNumBytesIndex(long... gridPosition) { + private int getNumBytesIndex(int... gridPosition) { return getOffsetIndex(gridPosition) + 1; } @@ -237,28 +243,25 @@ private static int[] prepend(final int value, final int[] array) { } /** - * Calculate the block position in the shard grid for a given index offset. + * Calculate the relative block position in the shard for a given block index. * - * @param offset the offset into the index - * @param blocksPerShard the dimensions of the shard in blocks - * @return the relative position in the shard grid + * @param blockIdx the block index in the shard + * @param blocksPerShard the dimensions of the shard in blocks + * @return the relative position in the shard */ - public static long[] shardPositionFromIndexOffset(int offset, int[] blocksPerShard) { + public static int[] blockPosition(int blockIdx, int[] blocksPerShard) { + + int numBlocks = Arrays.stream(blocksPerShard).reduce(1, (x, y) -> x * y); + if (blockIdx >= numBlocks) + throw new IllegalArgumentException("Shard Index Offset " + blockIdx + " is out of bounds for shard dimensions " + Arrays.toString(blocksPerShard)); - int maxOffset = 1; - for (int i = 0; i < blocksPerShard.length; i++) { - maxOffset *= blocksPerShard[i]; - } - if (offset >= maxOffset*2) { - throw new IllegalArgumentException("Shard Index Offset " + offset + " is out of bounds for shard dimensions " + Arrays.toString(blocksPerShard)); - } - final long[] position = new long[blocksPerShard.length]; - int remainder = offset / 2; + final int[] position = new int[blocksPerShard.length]; + int remainder = blockIdx ; - for (int dim = blocksPerShard.length - 1; dim >= 0; dim--) { // Iterate backwards - position[dim] = remainder % blocksPerShard[dim]; // Calculate position for this dimension - remainder /= blocksPerShard[dim]; // Update the remainder + for (int dim = blocksPerShard.length - 1; dim >= 0; dim--) { + position[dim] = remainder % blocksPerShard[dim]; + remainder /= blocksPerShard[dim]; } return position; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java index cf16f7190..41d505af0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java @@ -56,7 +56,7 @@ public ShardIndexBuilder setCodecs(DeterministicSizeCodec... codecs) { public ShardIndexBuilder addBlock(long[] blockPosition, long numBytes) { //TODO Caleb: Maybe move to ShardIndex? - final long[] blockPositionInShard = shard.getDatasetAttributes().getBlockPositionInShard( + final int[] blockPositionInShard = shard.getDatasetAttributes().getBlockPositionInShard( shard.getGridPosition(), blockPosition); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java index edab4ec0d..991b3acd8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java @@ -37,16 +37,7 @@ public DataBlock readBlock( final FileChannel in, long... blockPosition) throws IOException { - // TODO generalize from FileChannel - // TODO this assumes the "file" holding the shard is known, - // the logic to figure that out will have to go somewhere - - final ShardIndex index = readIndexes(in); - - final long[] shardPosition = datasetAttributes.getShardPositionForBlock(blockPosition); - in.position(index.getOffset(shardPosition)); - final InputStream is = Channels.newInputStream(in); - return DefaultBlockReader.readBlock(is, datasetAttributes, indexes); + throw new IOException("Remove this!"); } private long getIndexIndex(long... shardPosition) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java index 792019fdc..318568376 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java @@ -13,6 +13,7 @@ import java.util.Arrays; import java.util.List; +@Deprecated public class ShardWriter { private static final int BYTES_PER_LONG = 8; @@ -66,11 +67,10 @@ private void prepareForWritingDataBlock() throws IOException { // final ShardingProperties shardProps = new ShardingProperties(datasetAttributes); // indexData = new ShardIndexDataBlock(shardProps.getIndexDimensions()); - indexData = attributes.createIndex(); blockBytes = new ArrayList<>(); long cumulativeBytes = 0; - final long[] shardPosition = new long[1]; + final int[] shardPosition = new int[1]; for (int i = 0; i < blocks.size(); i++) { try (final ByteArrayOutputStream blockOut = new ByteArrayOutputStream()) { @@ -86,6 +86,7 @@ private void prepareForWritingDataBlock() throws IOException { } System.out.println(Arrays.toString(indexData.getData())); + throw new IOException("Remove this!"); } private void prepareForWriting() throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 7765d42be..edca50853 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -31,7 +31,7 @@ public VirtualShard(final ShardedDatasetAttributes datasetAttributes, long[] gri @Override public DataBlock getBlock(long... blockGridPosition) { - final long[] relativePosition = getBlockPosition(blockGridPosition); + final int[] relativePosition = getBlockPosition(blockGridPosition); if (relativePosition == null) throw new N5IOException("Attempted to read a block from the wrong shard."); @@ -57,7 +57,7 @@ public DataBlock getBlock(long... blockGridPosition) { @Override public void writeBlock(final DataBlock block) { - final long[] relativePosition = getBlockPosition(block.getGridPosition()); + final int[] relativePosition = getBlockPosition(block.getGridPosition()); if (relativePosition == null) throw new N5IOException("Attempted to write block in the wrong shard."); From 52751f5a22e4ccbe6b77a114d282f67999278fcf Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 8 Jan 2025 15:39:34 -0500 Subject: [PATCH 094/423] fix: Shard as Iterator> --- .../saalfeldlab/n5/codec/N5BlockCodec.java | 4 +- .../janelia/saalfeldlab/n5/shard/Shard.java | 46 ++++++------------- .../saalfeldlab/n5/shard/ShardIndex.java | 25 ---------- .../saalfeldlab/n5/util/GridIterator.java | 17 ++++--- .../saalfeldlab/n5/shard/ShardTest.java | 3 ++ 5 files changed, 29 insertions(+), 66 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java index 82f118bb7..92158041b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java @@ -63,9 +63,9 @@ public DataBlock allocateDataBlock() throws IOException { start = false; } if (mode != 2) { - return attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); + return attributes.getDataType().createDataBlock(blockSize, gridPosition.clone(), numElements); } else { - return attributes.getDataType().createDataBlock(null, gridPosition, numElements); + return attributes.getDataType().createDataBlock(null, gridPosition.clone(), numElements); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 211ad69e6..70f8150fc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -13,6 +13,8 @@ public interface Shard extends Iterable> { long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; + public ShardedDatasetAttributes getDatasetAttributes(); + /** * Returns the number of blocks this shard contains along all dimensions. * @@ -23,30 +25,26 @@ public interface Shard extends Iterable> { */ default int[] getBlockGridSize() { - final int[] sz = getSize(); - final int[] blkSz = getBlockSize(); - final int[] blockGridSize = new int[sz.length]; - for (int i = 0; i < sz.length; i++) - blockGridSize[i] = (int)(sz[i] / blkSz[i]); - - return blockGridSize; + return getDatasetAttributes().getBlocksPerShard(); } - public ShardedDatasetAttributes getDatasetAttributes(); - /** * Returns the size of shards in pixel units. * * @return shard size */ - public int[] getSize(); + default int[] getSize() { + return getDatasetAttributes().getShardSize(); + } /** * Returns the size of blocks in pixel units. * * @return block size */ - public int[] getBlockSize(); + default int[] getBlockSize() { + return getDatasetAttributes().getBlockSize(); + } /** * Returns the position of this shard on the shard grid. @@ -105,23 +103,16 @@ default long[] getShard(long... blockPosition) { default Iterator> iterator() { - return new DataBlockIterator(this); + return new DataBlockIterator<>(this); } default List> getBlocks() { - final ShardIndex shardIndex = getIndex(); - final ShardedDatasetAttributes attrs = getDatasetAttributes(); final List> blocks = new ArrayList<>(); - for (int blockIdx = 0; blockIdx < attrs.getNumBlocks(); blockIdx++) { - if (!shardIndex.exists(blockIdx)) - continue; - - final int[] blockPosInShard = ShardIndex.blockPosition(blockIdx, attrs.getBlocksPerShard()); - final long[] blockPosInImg = attrs.getBlockPositionFromShardPosition(getGridPosition(), blockPosInShard); - blocks.add(getBlock(blockPosInImg)); + for (DataBlock block : this) { + if (block != null) + blocks.add(block); } - return blocks; } @@ -167,15 +158,4 @@ public DataBlock next() { return shard.getBlock(it.next()); } } - - /** - * Say we want async datablock access - * - * Say we construct shard then getBlockAt - * - * (this could be how we do the aggregation) multiple getblockAt calls don't trigger reading read triggers reading of all blocks that were requested - * - * Shard doesn't hold the data directly, but is the metadata about how the blocks are stored - * - */ } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index a91c470c8..9c0d266ac 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -241,29 +241,4 @@ private static int[] prepend(final int value, final int[] array) { System.arraycopy(array, 0, indexBlockSize, 1, array.length); return indexBlockSize; } - - /** - * Calculate the relative block position in the shard for a given block index. - * - * @param blockIdx the block index in the shard - * @param blocksPerShard the dimensions of the shard in blocks - * @return the relative position in the shard - */ - public static int[] blockPosition(int blockIdx, int[] blocksPerShard) { - - int numBlocks = Arrays.stream(blocksPerShard).reduce(1, (x, y) -> x * y); - if (blockIdx >= numBlocks) - throw new IllegalArgumentException("Shard Index Offset " + blockIdx + " is out of bounds for shard dimensions " + Arrays.toString(blocksPerShard)); - - - final int[] position = new int[blocksPerShard.length]; - int remainder = blockIdx ; - - for (int dim = blocksPerShard.length - 1; dim >= 0; dim--) { - position[dim] = remainder % blocksPerShard[dim]; - remainder /= blocksPerShard[dim]; - } - - return position; - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java index 1fcb118cc..8b9ab8fc8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.util; +import java.util.Arrays; import java.util.Iterator; /** @@ -68,13 +69,10 @@ public int getIndex() { final static public void indexToPosition(long index, final long[] dimensions, final long[] position) { final int maxDim = dimensions.length - 1; - for (int d = 0; d < maxDim; ++d) { - final long j = index / dimensions[d]; - position[d] = index - j * dimensions[d]; - index = j; + for (int dim = maxDim; dim >= 0; dim--) { + position[dim] = index % dimensions[dim]; + index /= dimensions[dim]; } - position[maxDim] = index; - } final static public int[] long2int(final long[] a) { @@ -95,4 +93,11 @@ final static public long[] int2long(final int[] i) { return l; } + public static void main(String[] args) { + + final GridIterator it = new GridIterator(new int[]{2, 2, 2}); + while (it.hasNext()) { + System.out.println(Arrays.toString(it.next())); + } + } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index c5ab89cbc..f16aee3d3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -7,6 +7,7 @@ import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5FSTest; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; +import org.janelia.saalfeldlab.n5.N5URI; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.codec.BytesCodec; @@ -26,6 +27,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; +import java.util.List; import java.util.Map; @RunWith(Parameterized.class) From 6e3cbe524e9b4af81afe29850c012ba46a5011db Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 8 Jan 2025 16:54:46 -0500 Subject: [PATCH 095/423] test: ShardIndexTest --- .../saalfeldlab/n5/shard/ShardIndex.java | 74 ++++++++++++--- .../saalfeldlab/n5/shard/ShardIndexTest.java | 93 +++++++++++++++++++ 2 files changed, 154 insertions(+), 13 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 9c0d266ac..27b9eace0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.shard; +import org.apache.commons.io.input.BoundedInputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; @@ -15,6 +16,7 @@ import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; +import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; @@ -113,6 +115,28 @@ public long numBytes() { return totalNumBytes; } + public static ShardIndex read(byte[] data, final ShardIndex index) throws IOException { + + final IndexByteBounds byteBounds = byteBounds(index, data.length); + final ByteArrayInputStream is = new ByteArrayInputStream(data); + is.skip(byteBounds.start); + BoundedInputStream bIs = BoundedInputStream.builder() + .setInputStream(is) + .setMaxCount(byteBounds.size).get(); + + return read(bIs, index); + } + + public static ShardIndex read(InputStream in, final ShardIndex index) throws IOException { + + @SuppressWarnings("unchecked") + final DataBlock indexBlock = (DataBlock) DefaultBlockReader.readBlock(in, + index.getIndexAttributes(), index.gridPosition); + final long[] indexData = indexBlock.getData(); + System.arraycopy(indexData, 0, index.data, 0, index.data.length); + return index; + } + public static ShardIndex read( final KeyValueAccess keyValueAccess, final String key, @@ -121,16 +145,9 @@ public static ShardIndex read( final IndexByteBounds byteBounds = byteBounds(index, keyValueAccess.size(key)); try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(key, byteBounds.start, byteBounds.end)) { - final long[] indexData; try (final InputStream in = lockedChannel.newInputStream()) { - final DataBlock indexBlock = (DataBlock)DefaultBlockReader.readBlock( - in, - index.getIndexAttributes(), - index.gridPosition); - indexData = indexBlock.getData(); + return read(in,index); } - System.arraycopy(indexData, 0, index.data, 0, index.data.length); - return index; } catch (final N5Exception.N5NoSuchKeyException e) { return null; } catch (final IOException | UncheckedIOException e) { @@ -144,7 +161,7 @@ public static void write( final String key ) throws IOException { - final long start = index.location == IndexLocation.START ? 0 : keyValueAccess.size(key); + final long start = index.location == IndexLocation.START ? 0 : sizeOrZero( keyValueAccess, key) ; try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(key, start, index.numBytes())) { try (final OutputStream os = lockedChannel.newOutputStream()) { write(index, os); @@ -154,6 +171,14 @@ public static void write( } } + private static long sizeOrZero(final KeyValueAccess keyValueAccess, final String key) { + try { + return keyValueAccess.size(key); + } catch (Exception e) { + return 0; + } + } + public static void write(final ShardIndex index, OutputStream out) throws IOException { DefaultBlockWriter.writeBlock(out, index.getIndexAttributes(), index); @@ -191,15 +216,17 @@ public static IndexByteBounds byteBounds(final long indexSize, final IndexLocati } } - private static class IndexByteBounds { + public static class IndexByteBounds { - private final long start; - private final long end; + public final long start; + public final long end; + public final long size; - private IndexByteBounds(long start, long end) { + public IndexByteBounds(long start, long end) { this.start = start; this.end = end; + this.size = end - start + 1; } } @@ -241,4 +268,25 @@ private static int[] prepend(final int value, final int[] array) { System.arraycopy(array, 0, indexBlockSize, 1, array.length); return indexBlockSize; } + + @Override + public boolean equals(Object other) { + + if (other instanceof ShardIndex) { + + final ShardIndex index = (ShardIndex) other; + if (this.location != index.location) + return false; + + if (!Arrays.equals(this.size, index.size)) + return false; + + if (!Arrays.equals(this.data, index.data)) + return false; + + } + return true; + } + } + diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java new file mode 100644 index 000000000..39d6ab3e7 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -0,0 +1,93 @@ +package org.janelia.saalfeldlab.n5.shard; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.nio.file.Paths; + +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.N5FSTest; +import org.janelia.saalfeldlab.n5.N5KeyValueWriter; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; +import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; +import org.junit.After; +import org.junit.Ignore; +import org.junit.Test; + +public class ShardIndexTest { + + private static final N5FSTest tempN5Factory = new N5FSTest(); + + @After + public void removeTempWriters() { + tempN5Factory.removeTempWriters(); + } + + @Test + public void testReadVirtual() throws IOException { + + final N5KeyValueWriter writer = (N5KeyValueWriter) tempN5Factory.createTempN5Writer(); + final KeyValueAccess kva = writer.getKeyValueAccess(); + + final int[] shardBlockGridSize = new int[] { 6, 5 }; + final IndexLocation indexLocation = IndexLocation.END; + final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { new BytesCodec(), + new Crc32cChecksumCodec() }; + + final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "0").toString(); + + final ShardIndex index = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); + index.set(0, 6, new int[] { 0, 0 }); + index.set(19, 32, new int[] { 1, 0 }); + index.set(93, 111, new int[] { 3, 0 }); + index.set(143, 1, new int[] { 1, 2 }); + ShardIndex.write(index, kva, path); + + final ShardIndex other = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); + ShardIndex.read(kva, path, other); + + assertEquals(index, other); + } + + @Test + @Ignore + public void testReadInMemory() throws IOException { + + final N5KeyValueWriter writer = (N5KeyValueWriter) tempN5Factory.createTempN5Writer(); + final KeyValueAccess kva = writer.getKeyValueAccess(); + + final int[] shardBlockGridSize = new int[] { 6, 5 }; + final IndexLocation indexLocation = IndexLocation.END; + final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { new BytesCodec(), + new Crc32cChecksumCodec() }; + final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "0").toString(); + + final ShardIndex index = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); + index.set(0, 6, new int[] { 0, 0 }); + index.set(19, 32, new int[] { 1, 0 }); + index.set(93, 111, new int[] { 3, 0 }); + index.set(143, 1, new int[] { 1, 2 }); + ShardIndex.write(index, kva, path); + + ShardedDatasetAttributes attrs = new ShardedDatasetAttributes( + new long[]{6,5}, + shardBlockGridSize, + new int[]{1,1}, + DataType.UINT8, + new Codec[]{new N5BlockCodec(), new GzipCompression(4)}, + new DeterministicSizeCodec[]{new BytesCodec(), new Crc32cChecksumCodec()}, + indexLocation + ); + + final InMemoryShard shard = InMemoryShard.readShard(kva, path, new long[] {0,0}, attrs); + + assertEquals(index, shard.index); + } +} From 06477551d7b7f49b9d394874ddd5aebcfc921fe9 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 8 Jan 2025 16:55:15 -0500 Subject: [PATCH 096/423] feat: toward direct reading of InMemoryShard --- .../saalfeldlab/n5/shard/InMemoryShard.java | 63 ++++++++++++++++++- .../saalfeldlab/n5/shard/VirtualShard.java | 15 +++++ .../saalfeldlab/n5/util/GridIterator.java | 11 ++-- 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index c527df26d..70340b8e6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -1,20 +1,27 @@ package org.janelia.saalfeldlab.n5.shard; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; +import org.apache.commons.io.input.BoundedInputStream; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.io.output.CountingOutputStream; import org.apache.commons.io.output.ProxyOutputStream; import org.checkerframework.checker.units.qual.A; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.DefaultBlockReader; import org.janelia.saalfeldlab.n5.DefaultBlockWriter; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; +import org.janelia.saalfeldlab.n5.util.GridIterator; public class InMemoryShard extends AbstractShard { @@ -27,7 +34,6 @@ public class InMemoryShard extends AbstractShard { * Use morton- or c-ording instead of writing blocks out in the order they're added? * (later) */ - public InMemoryShard(final A datasetAttributes, final long[] shardPosition) { this( datasetAttributes, shardPosition, null); @@ -99,6 +105,61 @@ public void write(final OutputStream out) throws IOException { writeShardStart(out, this); } + public static InMemoryShard readShard( + final KeyValueAccess kva, final String key, final long[] gridPosition, final A attributes) + throws IOException { + + try (final LockedChannel lockedChannel = kva.lockForReading(key)) { + try (final InputStream is = lockedChannel.newInputStream()) { + return readShard(is, gridPosition, attributes); + } + } + } + + public static InMemoryShard readShard( + final InputStream inputStream, final long[] gridPosition, final A attributes) throws IOException { + + try (ByteArrayOutputStream result = new ByteArrayOutputStream()) { + byte[] buffer = new byte[1024]; + for (int length; (length = inputStream.read(buffer)) != -1;) { + result.write(buffer, 0, length); + } + return readShard(result.toByteArray(), gridPosition, attributes); + } + } + + public static InMemoryShard readShard(final byte[] data, + long[] shardPosition, final A attributes) throws IOException { + + final ShardIndex index = attributes.createIndex(); + ShardIndex.read(data, index); + + final InMemoryShard shard = new InMemoryShard(attributes, shardPosition, index); + final GridIterator it = new GridIterator(attributes.getBlocksPerShard()); + while (it.hasNext()) { + + final long[] p = it.next(); + final int[] pInt = GridIterator.long2int(p); + + if (index.exists(pInt)) { + + final ByteArrayInputStream is = new ByteArrayInputStream(data); + is.skip(index.getOffset(pInt)); + BoundedInputStream bIs = BoundedInputStream.builder().setInputStream(is) + .setMaxCount(index.getNumBytes(pInt)).get(); + + final long[] blockGridPosition = attributes.getBlockPositionFromShardPosition(shardPosition, p); + @SuppressWarnings("unchecked") + final DataBlock blk = (DataBlock) DefaultBlockReader.readBlock(bIs, attributes, + blockGridPosition); + shard.addBlock(blk); + bIs.close(); + } + } + + return shard; + } + public static void writeShard(final OutputStream out, final Shard shard) throws IOException { fromShard(shard).write(out); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 3ebe47d36..3a36ac14c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; +import java.util.function.Supplier; import org.checkerframework.checker.units.qual.A; import org.janelia.saalfeldlab.n5.DataBlock; @@ -28,6 +29,20 @@ public VirtualShard(final A data this.path = path; } + public VirtualShard(final A datasetAttributes, long[] gridPosition) { + + this(datasetAttributes, gridPosition, null, null); + } + + @SuppressWarnings("unchecked") + public DataBlock getBlock(Supplier inputSupplier, long... blockGridPosition) throws IOException { + + // TODO this method is just a wrapper around readBlock and probably not worth keeping + try (InputStream is = inputSupplier.get()) { + return (DataBlock) DefaultBlockReader.readBlock(is, datasetAttributes, blockGridPosition); + } + } + @SuppressWarnings("unchecked") @Override public DataBlock getBlock(long... blockGridPosition) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java index 8b9ab8fc8..1ea16efb3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java @@ -63,6 +63,10 @@ public long[] next() { return position; } + public int[] nextAsInt() { + return long2int(next()); + } + public int getIndex() { return index; } @@ -93,11 +97,4 @@ final static public long[] int2long(final int[] i) { return l; } - public static void main(String[] args) { - - final GridIterator it = new GridIterator(new int[]{2, 2, 2}); - while (it.hasNext()) { - System.out.println(Arrays.toString(it.next())); - } - } } From b17cb1fbbb9a037e5cdaac33b4bd708e010def85 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 8 Jan 2025 21:45:05 -0500 Subject: [PATCH 097/423] feat/test: add block position iterator for shard * add ShardProperties test --- .../janelia/saalfeldlab/n5/shard/Shard.java | 13 +++ .../saalfeldlab/n5/shard/ShardParameters.java | 10 ++- .../saalfeldlab/n5/util/GridIterator.java | 17 ++-- .../n5/shard/ShardPropertiesTests.java | 90 +++++++++++++++++++ 4 files changed, 122 insertions(+), 8 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index a90a2b12f..b0e551a1f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -116,6 +116,18 @@ default List> getBlocks() { return blocks; } + /** + * Returns an {@link Iterator} over block positions contained in this shard. + * + * @return + */ + default Iterator blockPositionIterator() { + + final int nd = getSize().length; + long[] min = getDatasetAttributes().getBlockPositionFromShardPosition( getGridPosition(), new long[nd]); + return new GridIterator(GridIterator.int2long(getBlockGridSize()), min); + } + public ShardIndex getIndex(); public static Shard createEmpty(final A attributes, long... shardPosition) { @@ -158,4 +170,5 @@ public DataBlock next() { return shard.getBlock(it.next()); } } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java index 27e9bf5b6..31d385f8d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java @@ -97,8 +97,10 @@ default int[] getBlockPositionInShard(final long[] shardPosition, final long[] b /** * Given a block's position relative to a shard, returns its position in pixels * relative to the image. - * - * @return the block position + * + * @param shardPosition shard position in the shard grid + * @param blockPosition block position the + * @return the block's min pixel coordinate */ default long[] getBlockMinFromShardPosition(final long[] shardPosition, final long[] blockPosition) { @@ -117,7 +119,9 @@ default long[] getBlockMinFromShardPosition(final long[] shardPosition, final lo * Given a block's position relative to a shard, returns its position relative * to the image. * - * @return the block position + * @param shardPosition shard position in the shard grid + * @param blockPosition block position relative to the shard + * @return the block position in the block grid */ default long[] getBlockPositionFromShardPosition(final long[] shardPosition, final long[] blockPosition) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java index 1ea16efb3..299fda863 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java @@ -1,6 +1,5 @@ package org.janelia.saalfeldlab.n5.util; -import java.util.Arrays; import java.util.Iterator; /** @@ -14,15 +13,18 @@ public class GridIterator implements Iterator { final protected long[] position; + final protected long[] min; + final protected int lastIndex; protected int index = -1; - public GridIterator(final long[] dimensions) { + public GridIterator(final long[] dimensions, final long[] min) { final int n = dimensions.length; this.dimensions = new long[n]; this.position = new long[n]; + this.min = min; steps = new long[n]; final int m = n - 1; @@ -38,6 +40,11 @@ public GridIterator(final long[] dimensions) { lastIndex = (int)(k * dimm - 1); } + public GridIterator(final long[] dimensions) { + + this(dimensions, new long[dimensions.length]); + } + public GridIterator(final int[] dimensions) { this(int2long(dimensions)); @@ -59,7 +66,7 @@ public boolean hasNext() { @Override public long[] next() { fwd(); - indexToPosition(index, dimensions, position); + indexToPosition(index, dimensions, min, position); return position; } @@ -71,10 +78,10 @@ public int getIndex() { return index; } - final static public void indexToPosition(long index, final long[] dimensions, final long[] position) { + final static public void indexToPosition(long index, final long[] dimensions, final long[] min, final long[] position) { final int maxDim = dimensions.length - 1; for (int dim = maxDim; dim >= 0; dim--) { - position[dim] = index % dimensions[dim]; + position[dim] = index % dimensions[dim] + min[dim]; index /= dimensions[dim]; } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java new file mode 100644 index 000000000..5f661f037 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java @@ -0,0 +1,90 @@ +package org.janelia.saalfeldlab.n5.shard; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.Iterator; + +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; +import org.junit.Test; + +public class ShardPropertiesTests { + + @Test + public void testShardProperties() throws Exception { + + final long[] arraySize = new long[]{16, 16}; + final int[] shardSize = new int[]{16, 16}; + final long[] shardPosition = new long[]{1, 1}; + final int[] blkSize = new int[]{4, 4}; + + final ShardedDatasetAttributes dsetAttrs = new ShardedDatasetAttributes( + arraySize, + shardSize, + blkSize, + DataType.UINT8, + new Codec[]{}, + new DeterministicSizeCodec[]{}, + IndexLocation.END); + + @SuppressWarnings({"rawtypes", "unchecked"}) + final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); + + assertArrayEquals(new int[]{4, 4}, shard.getBlockGridSize()); + + assertArrayEquals(new long[]{0, 0}, shard.getShard(0, 0)); + assertArrayEquals(new long[]{1, 1}, shard.getShard(5, 5)); + assertArrayEquals(new long[]{1, 0}, shard.getShard(5, 0)); + assertArrayEquals(new long[]{0, 1}, shard.getShard(0, 5)); + +// assertNull(shard.getBlockPosition(0, 0)); +// assertNull(shard.getBlockPosition(3, 3)); + + assertArrayEquals(new int[]{0, 0}, shard.getBlockPosition(4, 4)); + assertArrayEquals(new int[]{1, 1}, shard.getBlockPosition(5, 5)); + assertArrayEquals(new int[]{2, 2}, shard.getBlockPosition(6, 6)); + assertArrayEquals(new int[]{3, 3}, shard.getBlockPosition(7, 7)); + } + + @Test + public void testShardBlockPositionIterator() throws Exception { + + final long[] arraySize = new long[]{16, 16}; + final int[] shardSize = new int[]{16, 16}; + final long[] shardPosition = new long[]{1, 1}; + final int[] blkSize = new int[]{4, 4}; + + final ShardedDatasetAttributes dsetAttrs = new ShardedDatasetAttributes( + arraySize, + shardSize, + blkSize, + DataType.UINT8, + new Codec[]{}, + new DeterministicSizeCodec[]{}, + IndexLocation.END); + + @SuppressWarnings({"rawtypes", "unchecked"}) + final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); + + int i = 0; + Iterator it = shard.blockPositionIterator(); + long[] p = null; + while (it.hasNext()) { + + p = it.next(); + if( i == 0 ) + assertArrayEquals(new long[]{4,4}, p); + + i++; + } + assertEquals(16,i); + assertArrayEquals(new long[]{7,7}, p); + + } + +} From 334e46c64dc60053fddc6a420ab176b49069c329 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 9 Jan 2025 16:57:48 -0500 Subject: [PATCH 098/423] feat: ShardIndex get properties by block index --- .../org/janelia/saalfeldlab/n5/shard/ShardIndex.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 27b9eace0..4b3c0b015 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -76,11 +76,20 @@ public long getOffset(int... gridPosition) { return data[getOffsetIndex(gridPosition)]; } + public long getOffsetByBlockIndex(int index) { + return data[index * 2]; + } + public long getNumBytes(int... gridPosition) { return data[getNumBytesIndex(gridPosition)]; } + public long getNumBytesByBlockIndex(int index) { + + return data[index * 2 + 1]; + } + public void set(long offset, long nbytes, int[] gridPosition) { final int i = getOffsetIndex(gridPosition); From 84493829977de0cd7786aef8f8b339028e7a0e81 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 9 Jan 2025 17:00:40 -0500 Subject: [PATCH 099/423] feat: add Shard.getNumBlocks --- src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index b0e551a1f..ad18f87f8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -106,6 +106,11 @@ default Iterator> iterator() { return new DataBlockIterator<>(this); } + default int getNumBlocks() { + + return Arrays.stream(getBlockGridSize()).reduce(1, (x, y) -> x * y); + } + default List> getBlocks() { final List> blocks = new ArrayList<>(); From b6a5b4f307b3e624f6d350b5c11f8eeceb3b7ffa Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 9 Jan 2025 17:00:53 -0500 Subject: [PATCH 100/423] perf: VirtualShard smart override of getBlocks --- .../saalfeldlab/n5/shard/VirtualShard.java | 90 ++++++++++++++++--- 1 file changed, 80 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 3a36ac14c..c4cab674d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -4,9 +4,14 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; -import java.util.function.Supplier; - -import org.checkerframework.checker.units.qual.A; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.apache.commons.io.input.BoundedInputStream; +import org.apache.commons.io.input.ProxyInputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.DefaultBlockReader; @@ -15,6 +20,7 @@ import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.util.GridIterator; public class VirtualShard extends AbstractShard { @@ -35,15 +41,79 @@ public VirtualShard(final A data } @SuppressWarnings("unchecked") - public DataBlock getBlock(Supplier inputSupplier, long... blockGridPosition) throws IOException { + public DataBlock getBlock(InputStream inputStream, long... blockGridPosition) throws IOException { + + // TODO this method is just a wrapper around readBlock + // is it worth keeping/ + return (DataBlock) DefaultBlockReader.readBlock( + new ProxyInputStream( inputStream ) { + @Override + public void close( ) { + //nop + } + }, datasetAttributes, blockGridPosition); + } + + @Override + public List> getBlocks() { + + // will not contain nulls + + final ShardIndex index = getIndex(); + // TODO if the index is completely empty, can return right away + + final ArrayList> blocks = new ArrayList<>(); + + // sort index offsets + // and keep track of relevant positions + final long[] indexData = index.getData(); + List sortedOffsets = IntStream.range(0, index.getNumElements() / 2).mapToObj(i -> { + return new long[] { indexData[i * 2], i }; + }).filter(x -> { + return x[0] != Shard.EMPTY_INDEX_NBYTES; + }).collect(Collectors.toList()); + + Collections.sort(sortedOffsets, (a, b) -> Long.compare(((long[]) a)[0], ((long[]) b)[0])); + + final int nd = getDatasetAttributes().getNumDimensions(); + long[] position = new long[nd]; + + final int[] blocksPerShard = getDatasetAttributes().getBlocksPerShard(); + final long[] blockGridMin = IntStream.range(0, nd).mapToLong(i -> { + return blocksPerShard[i] * getGridPosition()[i]; + }).toArray(); + + long streamPosition = 0; + try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path)) { + try (final InputStream channelIn = lockedChannel.newInputStream()) { - // TODO this method is just a wrapper around readBlock and probably not worth keeping - try (InputStream is = inputSupplier.get()) { - return (DataBlock) DefaultBlockReader.readBlock(is, datasetAttributes, blockGridPosition); + for (long[] offsetIndex : sortedOffsets) { + + final long offset = offsetIndex[0]; + if (offset < 0) + continue; + + final long idx = offsetIndex[1]; + GridIterator.indexToPosition(idx, blocksPerShard, blockGridMin, position); + + channelIn.skip(offset - streamPosition); + final long numBytes = index.getNumBytesByBlockIndex((int) idx); + final BoundedInputStream bIs = BoundedInputStream.builder().setInputStream(channelIn) + .setMaxCount(numBytes).get(); + + blocks.add(getBlock(bIs, position.clone())); + streamPosition = offset + numBytes; + } + } + } catch (final N5Exception.N5NoSuchKeyException e) { + return blocks; + } catch (final IOException | UncheckedIOException e) { + throw new N5IOException("Failed to read block from " + path, e); } + + return blocks; } - @SuppressWarnings("unchecked") @Override public DataBlock getBlock(long... blockGridPosition) { @@ -62,7 +132,7 @@ public DataBlock getBlock(long... blockGridPosition) { try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path, startByte, size)) { try ( final InputStream channelIn = lockedChannel.newInputStream()) { final long[] blockPosInImg = getDatasetAttributes().getBlockPositionFromShardPosition(getGridPosition(), blockGridPosition); - return (DataBlock)DefaultBlockReader.readBlock(channelIn, datasetAttributes, blockPosInImg); + return getBlock( channelIn, blockPosInImg ); } } catch (final N5Exception.N5NoSuchKeyException e) { return null; @@ -126,10 +196,10 @@ public ShardIndex getIndex() { } catch (IOException e) { throw new N5IOException("Failed to read index at " + path, e); } + return index; } - static class CountingOutputStream extends OutputStream { private final OutputStream out; private long numBytes; From 55682ee04d271de70f5d0803f753f6f5f1833584 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 9 Jan 2025 17:01:10 -0500 Subject: [PATCH 101/423] fix: GridIterator iteration order --- .../saalfeldlab/n5/util/GridIterator.java | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java index 299fda863..d352f2b2a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java @@ -78,10 +78,32 @@ public int getIndex() { return index; } - final static public void indexToPosition(long index, final long[] dimensions, final long[] min, final long[] position) { - final int maxDim = dimensions.length - 1; - for (int dim = maxDim; dim >= 0; dim--) { - position[dim] = index % dimensions[dim] + min[dim]; + final static public void indexToPosition(long index, final long[] dimensions, final long[] offset, + final long[] position) { + for (int dim = 0; dim < dimensions.length; dim++) { + position[dim] = (index % dimensions[dim]) + offset[dim]; + index /= dimensions[dim]; + } + } + + final static public void indexToPosition(long index, final long[] dimensions, final long[] position) { + for (int dim = 0; dim < dimensions.length; dim++) { + position[dim] = index % dimensions[dim]; + index /= dimensions[dim]; + } + } + + final static public void indexToPosition(long index, final int[] dimensions, final long[] offset, + final long[] position) { + for (int dim = 0; dim < dimensions.length; dim++) { + position[dim] = (index % dimensions[dim]) + offset[dim]; + index /= dimensions[dim]; + } + } + + final static public void indexToPosition(long index, final int[] dimensions, final long[] position) { + for (int dim = 0; dim < dimensions.length; dim++) { + position[dim] = index % dimensions[dim]; index /= dimensions[dim]; } } From 707addf6e9127d91d339176fe4ce7d250b7ac9c3 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 10 Jan 2025 09:51:09 -0500 Subject: [PATCH 102/423] feat: DataBlockIterator skips missing blocks in Shard --- .../janelia/saalfeldlab/n5/shard/Shard.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index ad18f87f8..d3b1c647d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -2,8 +2,10 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; @@ -115,8 +117,7 @@ default List> getBlocks() { final List> blocks = new ArrayList<>(); for (DataBlock block : this) { - if (block != null) - blocks.add(block); + blocks.add(block); } return blocks; } @@ -158,20 +159,34 @@ public static class DataBlockIterator implements Iterator> { private final GridIterator it; private final Shard shard; + private final ShardIndex index; + private final ShardParameters attributes; + private int blockIndex = 0; public DataBlockIterator(final Shard shard) { this.shard = shard; + this.index = shard.getIndex(); + this.attributes = shard.getDatasetAttributes(); + this.blockIndex = 0; it = new GridIterator(shard.getBlockGridSize()); } @Override public boolean hasNext() { - return it.hasNext(); + + for (int i = blockIndex; i < attributes.getNumBlocks(); i++) { + if (index.exists(i)) + return true; + } + return false; } @Override public DataBlock next() { + while (!index.exists(blockIndex++)) + it.fwd(); + return shard.getBlock(it.next()); } } From 9a59de2f52abd7280d452fb2859a758b6e2408b3 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 10 Jan 2025 11:18:46 -0500 Subject: [PATCH 103/423] chore: rm unused ShardReader/Writer classes --- .../saalfeldlab/n5/shard/ShardReader.java | 81 ----------- .../saalfeldlab/n5/shard/ShardWriter.java | 130 ------------------ 2 files changed, 211 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java deleted file mode 100644 index 991b3acd8..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardReader.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DefaultBlockReader; -import org.janelia.saalfeldlab.n5.N5FSReader; -import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; -import org.janelia.saalfeldlab.n5.codec.IdentityCodec; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; -import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; - -public class ShardReader { - - private final ShardedDatasetAttributes datasetAttributes; - private long[] indexes; - - public ShardReader(final ShardedDatasetAttributes datasetAttributes) { - - this.datasetAttributes = datasetAttributes; - } - - public ShardIndex readIndexes(FileChannel channel) throws IOException { - - return ShardIndex.read(channel, datasetAttributes); - } - - public DataBlock readBlock( - final FileChannel in, - long... blockPosition) throws IOException { - - throw new IOException("Remove this!"); - } - - private long getIndexIndex(long... shardPosition) { - - final int[] indexDimensions = datasetAttributes.getBlocksPerShard(); - long idx = 0; - for (int i = 0; i < indexDimensions.length; i++) { - idx += shardPosition[i] * indexDimensions[i]; - } - - return idx; - } - - public static void main(String[] args) { - - final ShardReader reader = new ShardReader(buildTestAttributes()); - - System.out.println(reader.getIndexIndex(0, 0)); - System.out.println(reader.getIndexIndex(0, 1)); - System.out.println(reader.getIndexIndex(1, 0)); - System.out.println(reader.getIndexIndex(1, 1)); - - final N5Reader n5 = new N5FSReader("shard.n5"); - final ShardedDatasetAttributes datasetAttributes = buildTestAttributes(); - n5.readBlock("dataset", datasetAttributes, 0, 0, 0); - - } - - private static ShardedDatasetAttributes buildTestAttributes() { - - return new ShardedDatasetAttributes( - new long[]{4, 4}, - new int[]{2, 2}, - new int[]{2, 2}, - DataType.INT32, - new Codec[]{new N5BlockCodec(), new IdentityCodec()}, - new DeterministicSizeCodec[]{new Crc32cChecksumCodec()}, - IndexLocation.END); - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java deleted file mode 100644 index 5e3002b97..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardWriter.java +++ /dev/null @@ -1,130 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.DefaultBlockWriter; - -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -@Deprecated -public class ShardWriter { - - private static final int BYTES_PER_LONG = 8; - - private final List> blocks; - - private DatasetAttributes attributes; - - private ByteBuffer blockSizes; - - private ByteBuffer blockIndexes; - - private ShardIndex indexData; - - private List blockBytes; - - public ShardWriter(final A datasetAttributes) { - - blocks = new ArrayList<>(); - attributes = datasetAttributes; - } - - public A getAttributes() { - - return (A)attributes; - } - - public void reset() { - - blocks.clear(); - blockBytes.clear(); - blockSizes = null; - indexData = null; - } - - public void addBlock(final DataBlock block) { - - blocks.add(block); - } - - public void write(final Shard shard, final OutputStream out) throws IOException { - - prepareForWritingDataBlock(); - if (shard.getDatasetAttributes().getIndexLocation() == ShardingCodec.IndexLocation.START) { - writeIndexBlock(out); - writeBlocks(out); - } else { - writeBlocks(out); - writeIndexBlock(out); - } - } - - private void prepareForWritingDataBlock() throws IOException { - - // final ShardingProperties shardProps = new ShardingProperties(datasetAttributes); - // indexData = new ShardIndexDataBlock(shardProps.getIndexDimensions()); - - blockBytes = new ArrayList<>(); - long cumulativeBytes = 0; - final int[] shardPosition = new int[1]; - for (int i = 0; i < blocks.size(); i++) { - - try (final ByteArrayOutputStream blockOut = new ByteArrayOutputStream()) { - DefaultBlockWriter.writeBlock(blockOut, attributes, blocks.get(i)); - System.out.println(String.format("block %d is %d bytes", i, blockOut.size())); - - shardPosition[0] = i; - indexData.set(cumulativeBytes, blockOut.size(), shardPosition); - cumulativeBytes += blockOut.size(); - - blockBytes.add(blockOut.toByteArray()); - } - } - - System.out.println(Arrays.toString(indexData.getData())); - throw new IOException("Remove this!"); - } - - private void prepareForWriting() throws IOException { - - blockSizes = ByteBuffer.allocate(BYTES_PER_LONG * blocks.size()); - blockIndexes = ByteBuffer.allocate(BYTES_PER_LONG * blocks.size()); - blockBytes = new ArrayList<>(); - long cumulativeBytes = 0; - for (int i = 0; i < blocks.size(); i++) { - - try (final ByteArrayOutputStream blockOut = new ByteArrayOutputStream()) { - - DefaultBlockWriter.writeBlock(blockOut, attributes, blocks.get(i)); - System.out.println(String.format("block %d is %d bytes", i, blockOut.size())); - - blockIndexes.putLong(cumulativeBytes); - blockSizes.putLong(blockOut.size()); - cumulativeBytes += blockOut.size(); - - blockBytes.add(blockOut.toByteArray()); - } - } - } - - private void writeBlocks(final OutputStream out) throws IOException { - - for (final byte[] bytes : blockBytes) - out.write(bytes); - } - - private void writeIndexBlock(final OutputStream out) throws IOException { - - final DataOutputStream dos = new DataOutputStream(out); - for (final long l : indexData.getData()) - dos.writeLong(l); - } - -} From edb61a5c5b2257a6209d1ce514b3f695e2ed9db4 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 10 Jan 2025 11:20:33 -0500 Subject: [PATCH 104/423] fix/test: clone gridPosition --- .../java/org/janelia/saalfeldlab/n5/shard/ShardTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index da77593b1..4ace0546b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -7,7 +7,6 @@ import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5FSTest; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; -import org.janelia.saalfeldlab.n5.N5URI; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.codec.BytesCodec; @@ -72,7 +71,7 @@ private ShardedDatasetAttributes getTestAttributes(long[] dimensions, int[] shar shardSize, blockSize, DataType.UINT8, - new Codec[]{new N5BlockCodec(dataByteOrder)}, // , new GzipCompression(4)}, + new Codec[]{new N5BlockCodec(dataByteOrder), new GzipCompression(4)}, new DeterministicSizeCodec[]{new BytesCodec(indexByteOrder), new Crc32cChecksumCodec()}, indexLocation ); @@ -210,7 +209,7 @@ public void writeReadBlockTest() { } writer.writeBlock("shard", datasetAttributes, dataBlock); - final DataBlock block = writer.readBlock("shard", datasetAttributes, gridPosition); + final DataBlock block = writer.readBlock("shard", datasetAttributes, gridPosition.clone()); Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); for (Map.Entry entry : writtenBlocks.entrySet()) { From 539959f23b90bd87924055023abdd8e3d71badb6 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 10 Jan 2025 11:31:54 -0500 Subject: [PATCH 105/423] refactor: EMPTY_INDEX_NBYTES to ShardIndex --- .../org/janelia/saalfeldlab/n5/shard/Shard.java | 1 - .../janelia/saalfeldlab/n5/shard/ShardIndex.java | 14 +++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index d3b1c647d..d5e21ecee 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -13,7 +13,6 @@ public interface Shard extends Iterable> { - long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; /** * Returns the number of blocks this shard contains along all dimensions. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 4b3c0b015..c05e3ffd7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -28,11 +28,11 @@ public class ShardIndex extends LongArrayDataBlock { + public static final long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; private static final int BYTES_PER_LONG = 8; - private static final int LONGS_PER_BLOCK = 2; - private static final long[] DUMMY_GRID_POSITION = null; + private final IndexLocation location; private final DeterministicSizeCodec[] codecs; @@ -56,14 +56,14 @@ public ShardIndex(int[] shardBlockGridSize, DeterministicSizeCodec... codecs) { public boolean exists(int[] gridPosition) { - return getOffset(gridPosition) != Shard.EMPTY_INDEX_NBYTES || - getNumBytes(gridPosition) != Shard.EMPTY_INDEX_NBYTES; + return getOffset(gridPosition) != EMPTY_INDEX_NBYTES || + getNumBytes(gridPosition) != EMPTY_INDEX_NBYTES; } public boolean exists(int blockNum) { - return data[blockNum * 2] != Shard.EMPTY_INDEX_NBYTES || - data[blockNum * 2 + 1] != Shard.EMPTY_INDEX_NBYTES; + return data[blockNum * 2] != EMPTY_INDEX_NBYTES || + data[blockNum * 2 + 1] != EMPTY_INDEX_NBYTES; } public IndexLocation getLocation() { @@ -266,7 +266,7 @@ private static long[] emptyIndexData(final int[] size) { final int N = 2 * Arrays.stream(size).reduce(1, (x, y) -> x * y); final long[] data = new long[N]; - Arrays.fill(data, Shard.EMPTY_INDEX_NBYTES); + Arrays.fill(data, EMPTY_INDEX_NBYTES); return data; } From 0affc94d5302be443e85cc08127684e364c96195 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 10 Jan 2025 11:31:54 -0500 Subject: [PATCH 106/423] refactor: EMPTY_INDEX_NBYTES to ShardIndex --- .../org/janelia/saalfeldlab/n5/shard/Shard.java | 3 +-- .../janelia/saalfeldlab/n5/shard/ShardIndex.java | 14 +++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index d3b1c647d..5541bfe53 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -13,7 +13,6 @@ public interface Shard extends Iterable> { - long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; /** * Returns the number of blocks this shard contains along all dimensions. @@ -139,7 +138,7 @@ default Iterator blockPositionIterator() { public static Shard createEmpty(final A attributes, long... shardPosition) { final long[] emptyIndex = new long[(int)(2 * attributes.getNumBlocks())]; - Arrays.fill(emptyIndex, EMPTY_INDEX_NBYTES); + Arrays.fill(emptyIndex, ShardIndex.EMPTY_INDEX_NBYTES); final ShardIndex shardIndex = new ShardIndex(attributes.getBlocksPerShard(), emptyIndex, ShardingCodec.IndexLocation.END); return new InMemoryShard(attributes, shardPosition, shardIndex); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 4b3c0b015..c05e3ffd7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -28,11 +28,11 @@ public class ShardIndex extends LongArrayDataBlock { + public static final long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; private static final int BYTES_PER_LONG = 8; - private static final int LONGS_PER_BLOCK = 2; - private static final long[] DUMMY_GRID_POSITION = null; + private final IndexLocation location; private final DeterministicSizeCodec[] codecs; @@ -56,14 +56,14 @@ public ShardIndex(int[] shardBlockGridSize, DeterministicSizeCodec... codecs) { public boolean exists(int[] gridPosition) { - return getOffset(gridPosition) != Shard.EMPTY_INDEX_NBYTES || - getNumBytes(gridPosition) != Shard.EMPTY_INDEX_NBYTES; + return getOffset(gridPosition) != EMPTY_INDEX_NBYTES || + getNumBytes(gridPosition) != EMPTY_INDEX_NBYTES; } public boolean exists(int blockNum) { - return data[blockNum * 2] != Shard.EMPTY_INDEX_NBYTES || - data[blockNum * 2 + 1] != Shard.EMPTY_INDEX_NBYTES; + return data[blockNum * 2] != EMPTY_INDEX_NBYTES || + data[blockNum * 2 + 1] != EMPTY_INDEX_NBYTES; } public IndexLocation getLocation() { @@ -266,7 +266,7 @@ private static long[] emptyIndexData(final int[] size) { final int N = 2 * Arrays.stream(size).reduce(1, (x, y) -> x * y); final long[] data = new long[N]; - Arrays.fill(data, Shard.EMPTY_INDEX_NBYTES); + Arrays.fill(data, EMPTY_INDEX_NBYTES); return data; } From 7dde5bb0633d17b1c32af7ef9585742345eeec8f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 10 Jan 2025 11:36:29 -0500 Subject: [PATCH 107/423] chore: rm unused method --- .../org/janelia/saalfeldlab/n5/DefaultBlockReader.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index f881aea96..0a49fc19b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -33,7 +33,6 @@ import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.Codec.DataBlockInputStream; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec; /** * Default implementation of {@link BlockReader}. @@ -97,10 +96,4 @@ public static > void readFromStream(final B dataBlock, dataBlock.readData(buffer); } - public static long getShardIndex(final ShardingCodec shardingCodec, final long[] gridPosition) { - - // TODO implement - return -1; - } - } \ No newline at end of file From a791244b42d162a8ffd04ed4b8ca19286071e5f0 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 13 Jan 2025 11:34:39 -0500 Subject: [PATCH 108/423] feat: Codec add composition helpers --- .../org/janelia/saalfeldlab/n5/codec/Codec.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index a78df016a..209d169b6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -23,6 +23,22 @@ @NameConfig.Prefix("codec") public interface Codec extends Serializable { + public static OutputStream encode(OutputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { + OutputStream stream = out; + for (final BytesCodec codec : bytesCodecs) + stream = codec.encode(stream); + + return stream; + } + + public static InputStream decode(InputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { + InputStream stream = out; + for (final BytesCodec codec : bytesCodecs) + stream = codec.decode(stream); + + return stream; + } + public interface BytesCodec extends Codec { /** From cd54053d42c4f12b0fa897182a508da1e38431ca Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 13 Jan 2025 13:09:45 -0500 Subject: [PATCH 109/423] fix: EMPTY_INDEX_NBYTES now in ShardIndex --- .../org/janelia/saalfeldlab/n5/shard/VirtualShard.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index c4cab674d..1b5a59a74 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -5,6 +5,7 @@ import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -67,10 +68,10 @@ public List> getBlocks() { // sort index offsets // and keep track of relevant positions final long[] indexData = index.getData(); - List sortedOffsets = IntStream.range(0, index.getNumElements() / 2).mapToObj(i -> { + List sortedOffsets = Arrays.stream(blockIndexes).mapToObj(i -> { return new long[] { indexData[i * 2], i }; }).filter(x -> { - return x[0] != Shard.EMPTY_INDEX_NBYTES; + return x[0] != ShardIndex.EMPTY_INDEX_NBYTES; }).collect(Collectors.toList()); Collections.sort(sortedOffsets, (a, b) -> Long.compare(((long[]) a)[0], ((long[]) b)[0])); @@ -125,7 +126,7 @@ public DataBlock getBlock(long... blockGridPosition) { final long startByte = idx.getOffset(relativePosition); - if (startByte == Shard.EMPTY_INDEX_NBYTES ) + if (startByte == ShardIndex.EMPTY_INDEX_NBYTES ) return null; final long size = idx.getNumBytes(relativePosition); From 1c92d14a2149e1084947c234e264d31095d86809 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 13 Jan 2025 13:14:02 -0500 Subject: [PATCH 110/423] feat: add getBlocks(int[] blockIndexes) --- .../saalfeldlab/n5/shard/InMemoryShard.java | 21 ++++++++++++++++++- .../janelia/saalfeldlab/n5/shard/Shard.java | 4 +--- .../saalfeldlab/n5/shard/VirtualShard.java | 6 +++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 70340b8e6..52ef6abe3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -80,6 +80,21 @@ public List> getBlocks() { return new ArrayList<>(blocks.values()); } + public List> getBlocks( int[] blockIndexes ) { + + final ArrayList> out = new ArrayList<>(); + final int[] blocksPerShard = getDatasetAttributes().getBlocksPerShard(); + + long[] position = new long[ getSize().length ]; + for( int idx : blockIndexes ) { + GridIterator.indexToPosition(idx, blocksPerShard, position); + DataBlock blk = blocks.get(Arrays.hashCode(position)); + if( blk != null ); + out.add(blk); + } + return out; + } + protected IndexLocation indexLocation() { if (index != null) @@ -114,8 +129,11 @@ public static InMemoryShard(attributes, gridPosition, kva, key)); } + @SuppressWarnings("hiding") public static InMemoryShard readShard( final InputStream inputStream, final long[] gridPosition, final A attributes) throws IOException { @@ -128,7 +146,8 @@ public static InMemoryShard InMemoryShard readShard(final byte[] data, + public static InMemoryShard readShard( + final byte[] data, long[] shardPosition, final A attributes) throws IOException { final ShardIndex index = attributes.createIndex(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 5541bfe53..a8f8ce891 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -1,11 +1,10 @@ package org.janelia.saalfeldlab.n5.shard; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; @@ -13,7 +12,6 @@ public interface Shard extends Iterable> { - /** * Returns the number of blocks this shard contains along all dimensions. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 1b5a59a74..f0586c8cb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -56,7 +56,11 @@ public void close( ) { } @Override - public List> getBlocks() { + public List> getBlocks() { + return getBlocks(IntStream.range(0, getNumBlocks()).toArray()); + } + + public List> getBlocks(final int[] blockIndexes) { // will not contain nulls From 52aaa2ab1c4fa38ba625f72da74eae542d382907 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 13 Jan 2025 15:02:22 -0500 Subject: [PATCH 111/423] feat: InMemoryShard add new write methods --- .../saalfeldlab/n5/shard/InMemoryShard.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 52ef6abe3..0bd3cd476 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -112,6 +112,15 @@ public ShardIndex getIndex() { return indexBuilder.build(); } + public void write(final KeyValueAccess keyValueAccess, final String path) throws IOException { + + try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(path)) { + try (final OutputStream os = lockedChannel.newOutputStream()) { + write(os); + } + } + } + public void write(final OutputStream out) throws IOException { if (indexLocation() == IndexLocation.END) @@ -130,6 +139,7 @@ public static InMemoryShard(attributes, gridPosition, kva, key)); } @@ -179,6 +189,15 @@ public static InMemoryShard void writeShard(final KeyValueAccess keyValueAccess, final String path, final InMemoryShard shard) throws IOException { + + try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(path)) { + try (final OutputStream os = lockedChannel.newOutputStream()) { + writeShard(os, shard); + } + } + } + public static void writeShard(final OutputStream out, final Shard shard) throws IOException { fromShard(shard).write(out); From 6fac5f236102c050222e3cecd47dd8e6c4334bd1 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 13 Jan 2025 16:12:18 -0500 Subject: [PATCH 112/423] wip: minor change to writeBlocks, implement readBlocks * where readBlocks batches when possible --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 47 +++++++++++++++++++ .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 12 ++--- .../org/janelia/saalfeldlab/n5/N5Reader.java | 28 +++++++++++ 3 files changed, 81 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index d8165146e..61ab4e2ba 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -27,9 +27,13 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; +import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.shard.InMemoryShard; import org.janelia.saalfeldlab.n5.shard.Shard; import org.janelia.saalfeldlab.n5.shard.ShardParameters; import org.janelia.saalfeldlab.n5.shard.VirtualShard; @@ -124,6 +128,49 @@ default DataBlock readBlock( } } + @Override + default List> readBlocks( + final String pathName, + final DatasetAttributes datasetAttributes, + final List blockPositions) throws N5Exception { + + // TODO which interface should have this implementation? + if (datasetAttributes instanceof ShardParameters) { + + /* Group by shard index */ + final HashMap> shardBlockMap = new HashMap<>(); + final HashMap> shardPositionMap = new HashMap<>(); + final ShardParameters shardAttributes = (ShardParameters)datasetAttributes; + + for ( long[] blockPosition : blockPositions ) { + final long[] shardPosition = shardAttributes.getShardPositionForBlock(blockPosition); + final int shardHash = Arrays.hashCode(shardPosition); + if (!shardBlockMap.containsKey(shardHash)) { + final Shard shard = getShard(pathName, (DatasetAttributes & ShardParameters)shardAttributes, shardPosition); + shardBlockMap.put(shardHash, shard); + + final ArrayList positionList = new ArrayList<>(); + positionList.add(blockPosition); + shardPositionMap.put(shardHash, positionList); + } + else + shardPositionMap.get(shardBlockMap.get(shardHash)).add(blockPosition); + } + + final ArrayList> blocks = new ArrayList<>(); + for (Shard shard : shardBlockMap.values()) { + /* Add existing blocks before overwriting shard */ + final int shardHash = Arrays.hashCode(shard.getGridPosition()); + for( final long[] blkPosition : shardPositionMap.get(shardHash)) { + blocks.add(shard.getBlock(blkPosition)); + } + } + return blocks; + } else + return GsonN5Reader.super.readBlocks(pathName, datasetAttributes, blockPositions); + + } + @Override default String[] list(final String pathName) throws N5Exception { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 5977f1423..9db10575c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -216,8 +216,10 @@ default boolean removeAttributes(final String pathName, final List attri return removed; } - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Override default void writeBlocks(final String datasetPath, final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { + @Override default void writeBlocks( + final String datasetPath, + final DatasetAttributes datasetAttributes, + final DataBlock... dataBlocks) throws N5Exception { if (datasetAttributes instanceof ShardParameters) { /* Group by shard index */ @@ -236,6 +238,7 @@ default boolean removeAttributes(final String pathName, final List attri for (InMemoryShard shard : shardBlockMap.values()) { /* Add existing blocks before overwriting shard */ + @SuppressWarnings("unchecked") final Shard currentShard = (Shard)getShard(datasetPath, (DatasetAttributes & ShardParameters)shardAttributes, shard.getGridPosition()); for (DataBlock currentBlock : currentShard.getBlocks()) { if (shard.getBlock(currentBlock.getGridPosition()) == null) @@ -246,10 +249,7 @@ default boolean removeAttributes(final String pathName, final List attri } } else { - /* Just write each block */ - for (DataBlock dataBlock : dataBlocks) { - writeBlock(datasetPath, datasetAttributes, dataBlock); - } + GsonN5Writer.super.writeBlocks(datasetPath, datasetAttributes, dataBlocks); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 010e757f6..932768a6f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -293,6 +293,34 @@ DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception; + /** + * Reads multiple {@link DataBlock}s. + *

      + * Implementations may optimize / batch read operations when possible, e.g. + * in the case that the datasets are sharded. + * + * @param pathName + * dataset path + * @param datasetAttributes + * the dataset attributes + * @param gridPositions + * a list of grid positions + * @return a list of data blocks + * @throws N5Exception + * the exception + */ + default List> readBlocks( + final String pathName, + final DatasetAttributes datasetAttributes, + final List gridPositions) throws N5Exception { + + final ArrayList> blocks = new ArrayList<>(); + for( final long[] p : gridPositions ) + blocks.add(readBlock(pathName, datasetAttributes, p)); + + return blocks; + } + /** * Load a {@link DataBlock} as a {@link Serializable}. The offset is given * in From aca03d43e959c8fa627a9478cf6f7becaaf8540c Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 14 Jan 2025 09:02:53 -0500 Subject: [PATCH 113/423] demo: BlockIterators --- .../saalfeldlab/n5/demo/BlockIterators.java | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java new file mode 100644 index 000000000..1631b9450 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java @@ -0,0 +1,95 @@ +package org.janelia.saalfeldlab.n5.demo; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; +import org.janelia.saalfeldlab.n5.util.GridIterator; + +public class BlockIterators { + + public static void main(String[] args) { + +// blockIterator(); + shardBlockIterator(); + } + + public static void shardBlockIterator() { + + final ShardedDatasetAttributes attrs = new ShardedDatasetAttributes( + new long[] {12, 8}, // image size + new int[] {6, 4}, // shard size + new int[] {2, 2}, // block size + DataType.UINT8, + new Codec[] { new BytesCodec() }, + new DeterministicSizeCodec[] { new BytesCodec() }, + IndexLocation.END); + + shardPositions(attrs) + .forEach(x -> System.out.println(Arrays.toString(x))); + } + + public static void blockIterator() { + + final DatasetAttributes attrs = new DatasetAttributes( + new long[] {12, 8}, + new int[] {2, 2}, + DataType.UINT8, + new RawCompression()); + + blockPositions(attrs).forEach(x -> System.out.println(Arrays.toString(x))); + } + + public static long[] blockGridSize(final DatasetAttributes attrs ) { + // this could be a nice method for DatasetAttributes + + return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> { + return (long)Math.ceil(attrs.getDimensions()[i] / attrs.getBlockSize()[i]); + }).toArray(); + + } + + public static long[] shardGridSize(final ShardedDatasetAttributes attrs ) { + // this could be a nice method for DatasetAttributes + + return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> { + return (long)Math.ceil(attrs.getDimensions()[i] / attrs.getShardSize()[i]); + }).toArray(); + + } + + public static Stream blockPositions( DatasetAttributes attrs ) { + return toStream(new GridIterator(blockGridSize(attrs))); + } + + public static Stream shardPositions( ShardedDatasetAttributes attrs ) { + + final int[] blocksPerShard = attrs.getBlocksPerShard(); + return toStream( new GridIterator(shardGridSize(attrs))) + .flatMap( shardPosition -> { + + final int nd = attrs.getNumDimensions(); + final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new long[nd]); + return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); + }); + } + + public static Stream toStream( final Iterator it ) { + return StreamSupport.stream( Spliterators.spliteratorUnknownSize( + it, Spliterator.ORDERED), + false); + } + +} From 367cadbe712473cb901ae984857216eb454bb73f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 15 Jan 2025 16:29:23 -0500 Subject: [PATCH 114/423] fix: ShardIndex.getOffsetIndex --- .../saalfeldlab/n5/shard/ShardIndex.java | 8 ++- .../saalfeldlab/n5/shard/ShardIndexTest.java | 66 +++++++++++++++---- 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index c05e3ffd7..1a4836b7e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -97,16 +97,18 @@ public void set(long offset, long nbytes, int[] gridPosition) { data[i + 1] = nbytes; } - private int getOffsetIndex(int... gridPosition) { + protected int getOffsetIndex(int... gridPosition) { int idx = (int) gridPosition[0]; + int cumulativeSize = 1; for (int i = 1; i < gridPosition.length; i++) { - idx += gridPosition[i] * size[i]; + cumulativeSize *= size[i]; + idx += gridPosition[i] * cumulativeSize; } return idx * 2; } - private int getNumBytesIndex(int... gridPosition) { + protected int getNumBytesIndex(int... gridPosition) { return getOffsetIndex(gridPosition) + 1; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index 39d6ab3e7..f260b7085 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -3,11 +3,14 @@ import static org.junit.Assert.assertEquals; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Paths; +import org.apache.commons.io.output.ByteArrayOutputStream; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.GzipCompression; import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5FSTest; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; @@ -17,6 +20,7 @@ import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; +import org.janelia.saalfeldlab.n5.util.GridIterator; import org.junit.After; import org.junit.Ignore; import org.junit.Test; @@ -30,6 +34,38 @@ public void removeTempWriters() { tempN5Factory.removeTempWriters(); } + @Test + public void testOffsetIndex() throws IOException { + + int[] shardBlockGridSize = new int[]{5,4,3}; + ShardIndex index = new ShardIndex( + shardBlockGridSize, + IndexLocation.END, new BytesCodec()); + + GridIterator it = new GridIterator(shardBlockGridSize); + int i = 0; + while( it.hasNext()) { + int j = index.getOffsetIndex(GridIterator.long2int(it.next())); + assertEquals(i, j); + i+=2; + } + + + shardBlockGridSize = new int[]{5,4,3,13}; + index = new ShardIndex( + shardBlockGridSize, + IndexLocation.END, new BytesCodec()); + + it = new GridIterator(shardBlockGridSize); + i = 0; + while( it.hasNext()) { + int j = index.getOffsetIndex(GridIterator.long2int(it.next())); + assertEquals(i, j); + i+=2; + } + + } + @Test public void testReadVirtual() throws IOException { @@ -57,7 +93,6 @@ public void testReadVirtual() throws IOException { } @Test - @Ignore public void testReadInMemory() throws IOException { final N5KeyValueWriter writer = (N5KeyValueWriter) tempN5Factory.createTempN5Writer(); @@ -65,9 +100,10 @@ public void testReadInMemory() throws IOException { final int[] shardBlockGridSize = new int[] { 6, 5 }; final IndexLocation indexLocation = IndexLocation.END; - final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { new BytesCodec(), + final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { + new BytesCodec(), new Crc32cChecksumCodec() }; - final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "0").toString(); + final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "indexTest").toString(); final ShardIndex index = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); index.set(0, 6, new int[] { 0, 0 }); @@ -76,18 +112,20 @@ public void testReadInMemory() throws IOException { index.set(143, 1, new int[] { 1, 2 }); ShardIndex.write(index, kva, path); - ShardedDatasetAttributes attrs = new ShardedDatasetAttributes( - new long[]{6,5}, - shardBlockGridSize, - new int[]{1,1}, - DataType.UINT8, - new Codec[]{new N5BlockCodec(), new GzipCompression(4)}, - new DeterministicSizeCodec[]{new BytesCodec(), new Crc32cChecksumCodec()}, - indexLocation - ); + final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); + ShardIndex.read(rawBytes(kva, path), indexRead); + + assertEquals(index, indexRead); + } - final InMemoryShard shard = InMemoryShard.readShard(kva, path, new long[] {0,0}, attrs); + private static byte[] rawBytes(KeyValueAccess kva, String path) throws IOException { - assertEquals(index, shard.index); + final byte[] rawBytes = new byte[(int) kva.size(path)]; + try (final LockedChannel lockedChannel = kva.lockForReading(path)) { + try (final InputStream is = lockedChannel.newInputStream()) { + is.read(rawBytes); + } + } + return rawBytes; } } From 1a44168191b161def0f6a73d262608b9dab97c0f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 17 Jan 2025 13:48:40 -0500 Subject: [PATCH 115/423] feat: add Position * so that we can index by position * primitive long arrays are not great as keys for maps --- .../saalfeldlab/n5/util/FinalPosition.java | 38 ++++++++++++ .../janelia/saalfeldlab/n5/util/Position.java | 62 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/util/Position.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java b/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java new file mode 100644 index 000000000..1b7076d54 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java @@ -0,0 +1,38 @@ +package org.janelia.saalfeldlab.n5.util; + +/* + * An immutable {@Position}. + */ +public class FinalPosition implements Position { + + public final long[] position; + + public FinalPosition(long[] position) { + this.position = position; + } + + public FinalPosition(Position p) { + this.position = p.get().clone(); + } + + @Override + public long[] get() { + return position; + } + + @Override + public long get(int i) { + return position[i]; + } + + @Override + public String toString() { + return Position.toString(this); + } + + @Override + public boolean equals(Object obj) { + return Position.equals(this, obj); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java new file mode 100644 index 000000000..5ddb8cf0c --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java @@ -0,0 +1,62 @@ +package org.janelia.saalfeldlab.n5.util; + +import java.util.Arrays; + +/* + * A wrapper around a primitive long array that is lexicographically {@link Comparable} + * and for which we can test equality. + */ +public interface Position extends Comparable { + + public long[] get(); + + public long get(int i); + + default int numDimensions() { + return get().length; + } + + @Override + default int compareTo(Position other) { + + // use Arrays.compare when we update to Java 9+ + final int N = numDimensions() > other.numDimensions() ? numDimensions() : other.numDimensions(); + for (int i = 0; i < N; i++) { + final long diff = get(i) - other.get(i); + if (diff != 0) + return (int) diff; + } + return 0; + } + + public static boolean equals(final Position a, final Object b) { + + if (a == null && b == null) + return true; + + if (b == null) + return false; + + if (!(b instanceof Position)) + return false; + + final Position other = (Position) b; + if (other.numDimensions() != a.numDimensions()) + return false; + + for (int i = 0; i < a.numDimensions(); i++) + if (other.get(i) != a.get(i)) + return false; + + return true; + } + + public static String toString(Position p) { + return "Position: " + Arrays.toString(p.get()); + } + + public static Position wrap(final long[] p) { + return new FinalPosition(p); + } + +} From 43bc1e04eaa6ff8253439032322c18718da6267f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 17 Jan 2025 13:51:04 -0500 Subject: [PATCH 116/423] feat: add positionToIndex static methods in GridIterator --- .../saalfeldlab/n5/util/GridIterator.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java index d352f2b2a..9880e7b30 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java @@ -108,6 +108,46 @@ final static public void indexToPosition(long index, final int[] dimensions, fin } } + final static public long positionToIndex(final long[] dimensions, final long[] position) { + long idx = position[0]; + int cumulativeSize = 1; + for (int i = 0; i < position.length; i++) { + idx += position[i] * cumulativeSize; + cumulativeSize *= dimensions[i]; + } + return idx; + } + + final static public long positionToIndex(final long[] dimensions, final int[] position) { + long idx = position[0]; + int cumulativeSize = 1; + for (int i = 0; i < position.length; i++) { + idx += position[i] * cumulativeSize; + cumulativeSize *= dimensions[i]; + } + return idx; + } + + final static public long positionToIndex(final int[] dimensions, final int[] position) { + long idx = position[0]; + int cumulativeSize = 1; + for (int i = 0; i < position.length; i++) { + idx += position[i] * cumulativeSize; + cumulativeSize *= dimensions[i]; + } + return idx; + } + + final static public long positionToIndex(final int[] dimensions, final long[] position) { + long idx = position[0]; + int cumulativeSize = 1; + for (int i = 0; i < position.length; i++) { + idx += position[i] * cumulativeSize; + cumulativeSize *= dimensions[i]; + } + return idx; + } + final static public int[] long2int(final long[] a) { final int[] i = new int[a.length]; From 0e353e42fc2c6175561ead61258197580fd875c0 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 17 Jan 2025 13:54:18 -0500 Subject: [PATCH 117/423] feat: ShardParameters methods * shardsPerImage, blocksPerImage * grouping DataBlocks by shard postion * stream of block positions ordered by shard --- .../saalfeldlab/n5/shard/ShardParameters.java | 85 ++++++++++++++++++- .../n5/shard/ShardPropertiesTests.java | 30 ++++++- 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java index 31d385f8d..d1952b4b0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java @@ -1,9 +1,25 @@ package org.janelia.saalfeldlab.n5.shard; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; import org.janelia.saalfeldlab.n5.BlockParameters; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; +import org.janelia.saalfeldlab.n5.util.GridIterator; +import org.janelia.saalfeldlab.n5.util.Position; public interface ShardParameters extends BlockParameters { @@ -38,6 +54,28 @@ default int[] getBlocksPerShard() { return blocksPerShard; } + /** + * Returns the number of blocks per dimension that tile the image. + * + * @return blocks per image + */ + default long[] blocksPerImage() { + return IntStream.range(0, getNumDimensions()).mapToLong(i -> { + return (long) Math.ceil(getDimensions()[i] / getBlockSize()[i]); + }).toArray(); + } + + /** + * Returns the number of shards per dimension that tile the image. + * + * @return shards per image + */ + default long[] shardsPerImage() { + return IntStream.range(0, getNumDimensions()).mapToLong(i -> { + return (long)Math.ceil(getDimensions()[i] / getShardSize()[i]); + }).toArray(); + } + /** * Given a block's position relative to the array, returns the position of the shard containing that block relative to the shard grid. * @@ -92,7 +130,6 @@ default int[] getBlockPositionInShard(final long[] shardPosition, final long[] b return blockShardPos; } - /** * Given a block's position relative to a shard, returns its position in pixels @@ -134,7 +171,36 @@ default long[] getBlockPositionFromShardPosition(final long[] shardPosition, fin return blockImagePos; } + + default Map> groupBlockPositions(final List blockPositions) { + + final TreeMap> map = new TreeMap<>(); + for( final long[] blockPos : blockPositions ) { + Position shardPos = Position.wrap(getShardPositionForBlock(blockPos)); + if( !map.containsKey(shardPos)) { + map.put(shardPos, new ArrayList<>()); + } + map.get(shardPos).add(blockPos); + } + + return map; + } + default Map>> groupBlocks(final List> blocks) { + + // figure out how to re-use groupBlockPositions here? + final TreeMap>> map = new TreeMap<>(); + for (final DataBlock block : blocks) { + Position shardPos = Position.wrap(getShardPositionForBlock(block.getGridPosition())); + if (!map.containsKey(shardPos)) { + map.put(shardPos, new ArrayList<>()); + } + map.get(shardPos).add(block); + } + + return map; + } + /** * @return the number of blocks per shard */ @@ -143,4 +209,21 @@ default long getNumBlocks() { return Arrays.stream(getBlocksPerShard()).reduce(1, (x, y) -> x * y); } + default Stream blockPositions() { + + final int[] blocksPerShard = getBlocksPerShard(); + return toStream( new GridIterator(shardsPerImage())) + .flatMap( shardPosition -> { + final int nd = getNumDimensions(); + final long[] min = getBlockPositionFromShardPosition(shardPosition, new long[nd]); + return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); + }); + } + + static Stream toStream( final Iterator it ) { + return StreamSupport.stream( Spliterators.spliteratorUnknownSize( + it, Spliterator.ORDERED), + false); + } + } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java index 5f661f037..77d46ac45 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java @@ -2,15 +2,18 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; +import org.janelia.saalfeldlab.n5.util.Position; import org.junit.Test; public class ShardPropertiesTests { @@ -84,7 +87,32 @@ public void testShardBlockPositionIterator() throws Exception { } assertEquals(16,i); assertArrayEquals(new long[]{7,7}, p); + } + + @Test + public void testShardGrouping() { + + final long[] arraySize = new long[]{8, 12}; + final int[] shardSize = new int[]{4, 6}; + final int[] blkSize = new int[]{2, 3}; + + final ShardedDatasetAttributes attrs = new ShardedDatasetAttributes( + arraySize, + shardSize, + blkSize, + DataType.UINT8, + new Codec[]{}, + new DeterministicSizeCodec[]{}, + IndexLocation.END); + + List blockPositions = attrs.blockPositions().collect(Collectors.toList()); + final Map> result = attrs.groupBlockPositions(blockPositions); + + // there are four shards in this image + assertEquals(4, result.keySet().size()); + // there are four blocks per shard in this image + result.values().stream().forEach( x -> assertEquals(4, x.size())); } } From 709730868111ffcdc584efaffe5379836c5d49a0 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 17 Jan 2025 13:57:17 -0500 Subject: [PATCH 118/423] wip: rm unused flatIndex in Shard * methods in GridIterator replaces this --- .../java/org/janelia/saalfeldlab/n5/shard/Shard.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index a8f8ce891..69ed415fd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -1,6 +1,5 @@ package org.janelia.saalfeldlab.n5.shard; -import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -141,17 +140,6 @@ public static Shard createE return new InMemoryShard(attributes, shardPosition, shardIndex); } - public static long flatIndex(long[] gridPosition, int[] gridSize) { - - long index = gridPosition[0]; - long cumSizes = gridSize[0]; - for (int i = 1; i < gridSize.length; i++) { - index += gridPosition[i] * cumSizes; - cumSizes *= gridSize[i]; - } - return index; - } - public static class DataBlockIterator implements Iterator> { private final GridIterator it; From f67943dcbb804227ca2d1d2ae441fa03dc7e1063 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 17 Jan 2025 13:59:16 -0500 Subject: [PATCH 119/423] refactor: InMemoryShard, read/writeBlocks * Using Position class * using ShardParameters.groupBlocks helper method --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 45 +++++++------------ .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 35 +++++++-------- .../saalfeldlab/n5/shard/InMemoryShard.java | 20 ++++++--- .../saalfeldlab/n5/shard/ShardParameters.java | 3 -- 4 files changed, 46 insertions(+), 57 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 61ab4e2ba..ba8eb7b4e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -29,14 +29,15 @@ import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.shard.InMemoryShard; import org.janelia.saalfeldlab.n5.shard.Shard; import org.janelia.saalfeldlab.n5.shard.ShardParameters; import org.janelia.saalfeldlab.n5.shard.VirtualShard; +import org.janelia.saalfeldlab.n5.util.Position; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -93,10 +94,10 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { } - @SuppressWarnings("rawtypes") - default Shard getShard(final String pathName, - final A datasetAttributes, - long... shardGridPosition) { + @SuppressWarnings({ "unchecked", "rawtypes" }) + @Override + default Shard readShard(final String pathName, + final A datasetAttributes, long... shardGridPosition) { final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), shardGridPosition); return new VirtualShard(datasetAttributes, shardGridPosition, getKeyValueAccess(), path); @@ -111,7 +112,7 @@ default DataBlock readBlock( if (datasetAttributes instanceof ShardedDatasetAttributes) { final ShardedDatasetAttributes shardedAttrs = (ShardedDatasetAttributes) datasetAttributes; final long[] shardPosition = shardedAttrs.getShardPositionForBlock(gridPosition); - final Shard shard = getShard(pathName, shardedAttrs, shardPosition); + final Shard shard = readShard(pathName, shardedAttrs, shardPosition); return shard.getBlock(gridPosition); } @@ -137,38 +138,24 @@ default List> readBlocks( // TODO which interface should have this implementation? if (datasetAttributes instanceof ShardParameters) { - /* Group by shard index */ - final HashMap> shardBlockMap = new HashMap<>(); - final HashMap> shardPositionMap = new HashMap<>(); final ShardParameters shardAttributes = (ShardParameters)datasetAttributes; - for ( long[] blockPosition : blockPositions ) { - final long[] shardPosition = shardAttributes.getShardPositionForBlock(blockPosition); - final int shardHash = Arrays.hashCode(shardPosition); - if (!shardBlockMap.containsKey(shardHash)) { - final Shard shard = getShard(pathName, (DatasetAttributes & ShardParameters)shardAttributes, shardPosition); - shardBlockMap.put(shardHash, shard); + /* Group by shard position */ + final Map> shardBlockMap = shardAttributes.groupBlockPositions(blockPositions); + final ArrayList> blocks = new ArrayList<>(); + for( Entry> e : shardBlockMap.entrySet()) { - final ArrayList positionList = new ArrayList<>(); - positionList.add(blockPosition); - shardPositionMap.put(shardHash, positionList); - } - else - shardPositionMap.get(shardBlockMap.get(shardHash)).add(blockPosition); - } + final Shard shard = readShard(pathName, (DatasetAttributes & ShardParameters) shardAttributes, + e.getKey().get()); - final ArrayList> blocks = new ArrayList<>(); - for (Shard shard : shardBlockMap.values()) { - /* Add existing blocks before overwriting shard */ - final int shardHash = Arrays.hashCode(shard.getGridPosition()); - for( final long[] blkPosition : shardPositionMap.get(shardHash)) { + for (final long[] blkPosition : e.getValue()) { blocks.add(shard.getBlock(blkPosition)); } } + return blocks; } else return GsonN5Reader.super.readBlocks(pathName, datasetAttributes, blockPositions); - } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 9db10575c..ac6f733ab 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -29,9 +29,10 @@ import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.shard.InMemoryShard; @@ -44,6 +45,7 @@ import com.google.gson.JsonObject; import com.google.gson.JsonSyntaxException; import org.janelia.saalfeldlab.n5.shard.VirtualShard; +import org.janelia.saalfeldlab.n5.util.Position; /** * Default implementation of {@link N5Writer} with JSON attributes parsed with @@ -222,30 +224,25 @@ default boolean removeAttributes(final String pathName, final List attri final DataBlock... dataBlocks) throws N5Exception { if (datasetAttributes instanceof ShardParameters) { - /* Group by shard index */ - final HashMap> shardBlockMap = new HashMap<>(); + final ShardParameters shardAttributes = (ShardParameters)datasetAttributes; - for (DataBlock dataBlock : dataBlocks) { - final long[] shardPosition = shardAttributes.getShardPositionForBlock(dataBlock.getGridPosition()); - final int shardHash = Arrays.hashCode(shardPosition); - if (!shardBlockMap.containsKey(shardHash)) - shardBlockMap.put(shardHash, new InMemoryShard<>((DatasetAttributes & ShardParameters)shardAttributes, shardPosition)); + /* Group blocks by shard index */ + final Map>> shardBlockMap = shardAttributes.groupBlocks( + Arrays.stream(dataBlocks).collect(Collectors.toList())); - final InMemoryShard shard = shardBlockMap.get(shardHash); - shard.addBlock(dataBlock); - } + for( final Entry>> e : shardBlockMap.entrySet()) { - for (InMemoryShard shard : shardBlockMap.values()) { - /* Add existing blocks before overwriting shard */ + final long[] shardPosition = e.getKey().get(); @SuppressWarnings("unchecked") - final Shard currentShard = (Shard)getShard(datasetPath, (DatasetAttributes & ShardParameters)shardAttributes, shard.getGridPosition()); - for (DataBlock currentBlock : currentShard.getBlocks()) { - if (shard.getBlock(currentBlock.getGridPosition()) == null) - shard.addBlock(currentBlock); - } + final Shard currentShard = (Shard) readShard(datasetPath, (DatasetAttributes & ShardParameters)shardAttributes, + shardPosition); + + final InMemoryShard newShard = InMemoryShard.fromShard(currentShard); + for( DataBlock blk : e.getValue()) + newShard.addBlock(blk); - writeShard(datasetPath, (DatasetAttributes & ShardParameters)shardAttributes, shard); + writeShard(datasetPath, (DatasetAttributes & ShardParameters)shardAttributes, newShard); } } else { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 0bd3cd476..b36dc5aca 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -8,6 +8,8 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import org.apache.commons.io.input.BoundedInputStream; import org.apache.commons.io.output.ByteArrayOutputStream; @@ -22,13 +24,14 @@ import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; +import org.janelia.saalfeldlab.n5.util.Position; public class InMemoryShard extends AbstractShard { /* Map of a hash of the DataBlocks `gridPosition` to the block */ - private final HashMap> blocks; + private final Map> blocks; private ShardIndexBuilder indexBuilder; - + /* * TODO: * Use morton- or c-ording instead of writing blocks out in the order they're added? @@ -45,17 +48,22 @@ public InMemoryShard(final A dat ShardIndex index) { super(datasetAttributes, gridPosition, index); - blocks = new HashMap<>(); + blocks = new TreeMap<>(); } private void storeBlock(DataBlock block) { - blocks.put(Arrays.hashCode(block.getGridPosition()), block); + blocks.put(Position.wrap(block.getGridPosition()), block); } + /* + * Returns the {@link DataBlock} given a block grid position. + *

      + * The block grid position is relative to the image, not relative to this shard. + */ @Override public DataBlock getBlock(long... blockGridPosition) { - return blocks.get(Arrays.hashCode(blockGridPosition)); + return blocks.get(Position.wrap(blockGridPosition)); } @Override @@ -88,7 +96,7 @@ public List> getBlocks( int[] blockIndexes ) { long[] position = new long[ getSize().length ]; for( int idx : blockIndexes ) { GridIterator.indexToPosition(idx, blocksPerShard, position); - DataBlock blk = blocks.get(Arrays.hashCode(position)); + DataBlock blk = getBlock(position); if( blk != null ); out.add(blk); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java index d1952b4b0..ddc3ba28b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java @@ -8,15 +8,12 @@ import java.util.Spliterator; import java.util.Spliterators; import java.util.TreeMap; -import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.janelia.saalfeldlab.n5.BlockParameters; import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.janelia.saalfeldlab.n5.util.Position; From 9224215e811bf9f0f637291d86d68ace92a1353a Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 21 Jan 2025 09:31:28 -0500 Subject: [PATCH 120/423] wip/feat: N5Reader.readShard --- .../java/org/janelia/saalfeldlab/n5/N5Reader.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 932768a6f..04c9d8892 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -43,6 +43,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.ShardParameters; + /** * A simple structured container for hierarchies of chunked * n-dimensional datasets and attributes. @@ -293,6 +296,18 @@ DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception; + /** + * Reads the {@link Shard} at the corresponding grid position. + * + * @param + * @param datasetPath + * @param datasetAttributes + * @param shardGridPosition + * @return the shard + */ + public Shard readShard(final String datasetPath, + final A datasetAttributes, long... shardGridPosition); + /** * Reads multiple {@link DataBlock}s. *

      From a74d3437d0b5520076e4e2629de4dc50a11db787 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 21 Jan 2025 10:08:02 -0500 Subject: [PATCH 121/423] feat: add ShardIndex.isEmpty * use it to return early for getBlocks --- .../org/janelia/saalfeldlab/n5/shard/ShardIndex.java | 11 +++++++++++ .../janelia/saalfeldlab/n5/shard/VirtualShard.java | 6 +++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 1a4836b7e..10433c101 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -25,6 +25,7 @@ import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.util.Arrays; +import java.util.stream.IntStream; public class ShardIndex extends LongArrayDataBlock { @@ -66,6 +67,16 @@ public boolean exists(int blockNum) { data[blockNum * 2 + 1] != EMPTY_INDEX_NBYTES; } + public int getNumBlocks() { + + return Arrays.stream(getSize()).reduce(1, (x, y) -> x * y); + } + + public boolean isEmpty() { + + return !IntStream.range(0, getNumBlocks()).anyMatch(i -> exists(i)); + } + public IndexLocation getLocation() { return location; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index f0586c8cb..88aa6b086 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -63,12 +63,12 @@ public List> getBlocks() { public List> getBlocks(final int[] blockIndexes) { // will not contain nulls - final ShardIndex index = getIndex(); - // TODO if the index is completely empty, can return right away - final ArrayList> blocks = new ArrayList<>(); + if (index.isEmpty()) + return blocks; + // sort index offsets // and keep track of relevant positions final long[] indexData = index.getData(); From ce50fa9a746179e4f741ef0c62939586f44fcc82 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 24 Jan 2025 13:12:19 -0500 Subject: [PATCH 122/423] test: add a test for nested sharding codecs * but ignore it for now --- .../saalfeldlab/n5/shard/ShardTest.java | 53 +++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 4ace0546b..6848b9df1 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -15,12 +15,16 @@ import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; +import org.janelia.saalfeldlab.n5.util.GridIterator; import org.junit.After; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import static org.junit.Assert.assertArrayEquals; + import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Arrays; @@ -121,10 +125,6 @@ public void writeReadBlocksTest() { final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); - final String shardKey = ((N5KeyValueWriter)writer).absoluteDataBlockPath("shard", 2, 2); - final VirtualShard vs = new VirtualShard<>(datasetAttributes, new long[]{2, 2}, kva, shardKey); - final List> blocks = vs.getBlocks(); - final String[][] keys = new String[][]{ {"shard", "0", "0"}, {"shard", "1", "0"}, @@ -272,4 +272,49 @@ public void writeReadShardTest() { } } + @Test + @Ignore + public void writeReadNestedShards() { + + int[] blockSize = new int[]{4, 4}; + int N = Arrays.stream(blockSize).reduce(1, (x,y) -> x*y); + + final N5Writer writer = tempN5Factory.createTempN5Writer(); + final ShardedDatasetAttributes datasetAttributes = getNestedShardCodecsAttributes(blockSize); + writer.createDataset("nestedShards", datasetAttributes); + + final byte[] data = new byte[N]; + Arrays.fill(data, (byte)4); + + writer.writeBlocks("nestedShards", datasetAttributes, + new ByteArrayDataBlock(blockSize, new long[] { 1, 1 }, data), + new ByteArrayDataBlock(blockSize, new long[] { 0, 2 }, data), + new ByteArrayDataBlock(blockSize, new long[] { 2, 1 }, data)); + + assertArrayEquals(data, (byte[]) writer.readBlock("nestedShards", datasetAttributes, 1, 1).getData()); + assertArrayEquals(data, (byte[]) writer.readBlock("nestedShards", datasetAttributes, 0, 2).getData()); + assertArrayEquals(data, (byte[]) writer.readBlock("nestedShards", datasetAttributes, 2, 1).getData()); + } + + private ShardedDatasetAttributes getNestedShardCodecsAttributes(int[] blockSize) { + + final int[] innerShardSize = new int[] { 2 * blockSize[0], 2 * blockSize[1] }; + final int[] shardSize = new int[] { 4 * blockSize[0], 4 * blockSize[1] }; + final long[] dimensions = GridIterator.int2long(shardSize); + + // TODO: its not even clear how we build this given + // this constructor. Is the block size of the sharded dataset attributes + // the innermost (block) size or the intermediate shard size? + // probably better to forget about this class - only use DatasetAttributes + // and detect shading in another way + final ShardingCodec innerShard = new ShardingCodec(innerShardSize, + new Codec[] { new BytesCodec() }, + new DeterministicSizeCodec[] { new BytesCodec(indexByteOrder), new Crc32cChecksumCodec() }, + IndexLocation.START); + + return new ShardedDatasetAttributes(dimensions, shardSize, blockSize, DataType.UINT8, + new Codec[] { innerShard }, + new DeterministicSizeCodec[] { new BytesCodec(indexByteOrder), new Crc32cChecksumCodec() }, + IndexLocation.END); + } } From 9518d00b8a54bf1bc014f96137d991901c908be3 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Jan 2025 18:11:47 -0500 Subject: [PATCH 123/423] Add N5ReadBenchmark --- pom.xml | 12 ++ .../saalfeldlab/n5/N5ReadBenchmark.java | 121 ++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java diff --git a/pom.xml b/pom.xml index 5567ec39f..6bbfb5e33 100644 --- a/pom.xml +++ b/pom.xml @@ -202,6 +202,18 @@ org.apache.commons commons-compress + + + + org.openjdk.jmh + jmh-core + test + + + org.openjdk.jmh + jmh-generator-annprocess + test + diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java new file mode 100644 index 000000000..f26b25e9f --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java @@ -0,0 +1,121 @@ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.GsonBuilder; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.TimeValue; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertTrue; + +@State( Scope.Thread ) +@Fork( 1 ) +public class N5ReadBenchmark { + + + + private static String tempN5PathName() { + try { + final File tmpFile = Files.createTempDirectory("n5-test-").toFile(); + tmpFile.deleteOnExit(); + return tmpFile.getCanonicalPath(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + private static final String basePath = tempN5PathName(); + private static final String datasetName = "/test/group/dataset"; + private static final long[] dimensions = new long[]{640, 640, 640}; + private static final int[] blockSize = new int[]{64, 64, 64}; + + private static byte[] byteBlock; + private static short[] shortBlock; + private static int[] intBlock; + private static long[] longBlock; + private static float[] floatBlock; + private static double[] doubleBlock; + + private static void createData() { + final Random rnd = new Random(); + final int blockNumElements = DataBlock.getNumElements(blockSize); + byteBlock = new byte[blockNumElements]; + shortBlock = new short[blockNumElements]; + intBlock = new int[blockNumElements]; + longBlock = new long[blockNumElements]; + floatBlock = new float[blockNumElements]; + doubleBlock = new double[blockNumElements]; + rnd.nextBytes(byteBlock); + for (int i = 0; i < blockNumElements; ++i) { + shortBlock[i] = (short)rnd.nextInt(); + intBlock[i] = rnd.nextInt(); + longBlock[i] = rnd.nextLong(); + floatBlock[i] = Float.intBitsToFloat(rnd.nextInt()); + doubleBlock[i] = Double.longBitsToDouble(rnd.nextLong()); + } + } + + private static N5Writer createTempN5Writer() { + return new N5FSWriter(basePath, new GsonBuilder()); + } + + private static N5Reader n5; + private static DatasetAttributes attributes; + + @Setup(Level.Trial) + public void setup() { + createData(); + System.out.println("basePath = " + basePath); + + try (final N5Writer n5 = createTempN5Writer()) { + final Compression compression = new Lz4Compression(); + n5.createDataset(datasetName, dimensions, blockSize, DataType.FLOAT64, compression); + final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + final DoubleArrayDataBlock dataBlock = new DoubleArrayDataBlock(blockSize, new long[] {0, 0, 0}, doubleBlock); + n5.writeBlock(datasetName, attributes, dataBlock); + } + + n5 = new N5FSReader(basePath, new GsonBuilder()); + attributes = n5.getDatasetAttributes(datasetName); + } + +// @TearDown(Level.Trial) +// public void teardown() { +// } + + @Benchmark + @BenchmarkMode( Mode.AverageTime ) + @OutputTimeUnit( TimeUnit.MILLISECONDS ) + public void bench() { + n5.readBlock(datasetName, attributes, 0, 0, 0); + } + + public static void main( final String... args ) throws RunnerException, IOException + { + final Options opt = new OptionsBuilder() + .include( N5ReadBenchmark.class.getSimpleName() ) + .warmupIterations( 4 ) + .measurementIterations( 8 ) + .warmupTime( TimeValue.milliseconds( 500 ) ) + .measurementTime( TimeValue.milliseconds( 500 ) ) + .build(); + new Runner( opt ).run(); + } +} From 3401fcf7cb78874ff37149342a35b93e8cb5fe59 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Jan 2025 09:40:30 -0500 Subject: [PATCH 124/423] remove redundant modifiers --- .../java/org/janelia/saalfeldlab/n5/DataBlock.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 3d9dc92a1..9ef5387a2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -47,7 +47,7 @@ public interface DataBlock { * * @return size of the data block */ - public int[] getSize(); + int[] getSize(); /** * Returns the position of this data block on the block grid. @@ -57,14 +57,14 @@ public interface DataBlock { * * @return position on the block grid */ - public long[] getGridPosition(); + long[] getGridPosition(); /** * Returns the data object held by this data block. * * @return data object */ - public T getData(); + T getData(); /** * Creates a {@link ByteBuffer} that contains the data object of this data @@ -78,7 +78,7 @@ public interface DataBlock { * * @return {@link ByteBuffer} containing data */ - public ByteBuffer toByteBuffer(); + ByteBuffer toByteBuffer(); /** * Reads the data object of this data block from a {@link ByteBuffer}. @@ -92,7 +92,7 @@ public interface DataBlock { * @param buffer * the byte buffer */ - public void readData(final ByteBuffer buffer); + void readData(final ByteBuffer buffer); /** * Returns the number of elements in this {@link DataBlock}. This number is @@ -101,7 +101,7 @@ public interface DataBlock { * * @return the number of elements */ - public int getNumElements(); + int getNumElements(); /** * Returns the number of elements in a box of given size. @@ -110,7 +110,7 @@ public interface DataBlock { * the size * @return the number of elements */ - public static int getNumElements(final int[] size) { + static int getNumElements(final int[] size) { int n = size[0]; for (int i = 1; i < size.length; ++i) From 9151207e2604339650e09c82e2c10e7a17327244 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Jan 2025 10:07:03 -0500 Subject: [PATCH 125/423] WIP: remove @Override annotations for ByteBuffer methods to be able to turn those methods on/off in the DataBlock interface without causing compile errors in the implementations. --- .../java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java | 2 -- .../java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java | 2 -- .../java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java | 2 -- src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java | 2 -- .../java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java | 2 -- .../java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java | 2 -- src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java | 2 -- 7 files changed, 14 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index 5717ad2ef..fb106b89c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -34,13 +34,11 @@ public ByteArrayDataBlock(final int[] size, final long[] gridPosition, final byt super(size, gridPosition, data); } - @Override public ByteBuffer toByteBuffer() { return ByteBuffer.wrap(getData()); } - @Override public void readData(final ByteBuffer buffer) { if (buffer.array() != getData()) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 426c7944d..ba7dc606a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -34,7 +34,6 @@ public DoubleArrayDataBlock(final int[] size, final long[] gridPosition, final d super(size, gridPosition, data); } - @Override public ByteBuffer toByteBuffer() { final ByteBuffer buffer = ByteBuffer.allocate(data.length * 8); @@ -42,7 +41,6 @@ public ByteBuffer toByteBuffer() { return buffer; } - @Override public void readData(final ByteBuffer buffer) { buffer.asDoubleBuffer().get(data); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index b8d309998..2730e5d7c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -34,7 +34,6 @@ public FloatArrayDataBlock(final int[] size, final long[] gridPosition, final fl super(size, gridPosition, data); } - @Override public ByteBuffer toByteBuffer() { final ByteBuffer buffer = ByteBuffer.allocate(data.length * 4); @@ -42,7 +41,6 @@ public ByteBuffer toByteBuffer() { return buffer; } - @Override public void readData(final ByteBuffer buffer) { buffer.asFloatBuffer().get(data); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 98c5577d4..797002be5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -34,7 +34,6 @@ public IntArrayDataBlock(final int[] size, final long[] gridPosition, final int[ super(size, gridPosition, data); } - @Override public ByteBuffer toByteBuffer() { final ByteBuffer buffer = ByteBuffer.allocate(data.length * 4); @@ -42,7 +41,6 @@ public ByteBuffer toByteBuffer() { return buffer; } - @Override public void readData(final ByteBuffer buffer) { buffer.asIntBuffer().get(data); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index d3f3fc9c5..7c8a812f2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -34,7 +34,6 @@ public LongArrayDataBlock(final int[] size, final long[] gridPosition, final lon super(size, gridPosition, data); } - @Override public ByteBuffer toByteBuffer() { final ByteBuffer buffer = ByteBuffer.allocate(data.length * 8); @@ -42,7 +41,6 @@ public ByteBuffer toByteBuffer() { return buffer; } - @Override public void readData(final ByteBuffer buffer) { buffer.asLongBuffer().get(data); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 2dbf6b176..61b5a81f2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -34,7 +34,6 @@ public ShortArrayDataBlock(final int[] size, final long[] gridPosition, final sh super(size, gridPosition, data); } - @Override public ByteBuffer toByteBuffer() { final ByteBuffer buffer = ByteBuffer.allocate(data.length * 2); @@ -42,7 +41,6 @@ public ByteBuffer toByteBuffer() { return buffer; } - @Override public void readData(final ByteBuffer buffer) { buffer.asShortBuffer().get(data); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index a64f3b9ca..e4ad904e2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -46,14 +46,12 @@ public StringDataBlock(final int[] size, final long[] gridPosition, final byte[] serializedData = data; } - @Override public ByteBuffer toByteBuffer() { if (serializedData == null) serializedData = serialize(actualData); return ByteBuffer.wrap(serializedData); } - @Override public void readData(final ByteBuffer buffer) { if (buffer.hasArray()) { From 27b6057b51bba7b1ee1830931d669e1e030d13cc Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Jan 2025 10:07:41 -0500 Subject: [PATCH 126/423] Fix benchmark image URL --- src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java index 8f72edaf9..4355e028e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java @@ -100,7 +100,7 @@ public static void setUpBeforeClass() throws Exception { throw new IOException("Could not create benchmark directory for HDF5Utils benchmark."); data = new short[64 * 64 * 64]; - final ImagePlus imp = new Opener().openURL("https://imagej.nih.gov/ij/images/t1-head-raw.zip"); + final ImagePlus imp = new Opener().openURL("https://imagej.net/ij/images/t1-head-raw.zip"); final ImagePlusImg img = (ImagePlusImg)(Object)ImagePlusImgs.from(imp); final Cursor cursor = Views.flatIterable(Views.interval(img, new long[]{100, 100, 30}, new long[]{163, 163, 93})).cursor(); for (int i = 0; i < data.length; ++i) From 08940ee6668863a906f72a8a403262e547bde912 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Jan 2025 15:17:14 -0500 Subject: [PATCH 127/423] Add byte[] DataType.createSerializeArray(numElements) --- .../org/janelia/saalfeldlab/n5/DataType.java | 50 ++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index a6d60cb81..356c816de 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -34,6 +34,7 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import java.util.function.IntFunction; /** * Enumerates available data types. @@ -47,82 +48,97 @@ public enum DataType { (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, - new byte[numElements])), + new byte[numElements]), + numElements -> new byte[Byte.BYTES * numElements]), UINT16( "uint16", (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( blockSize, gridPosition, - new short[numElements])), + new short[numElements]), + numElements -> new byte[Short.BYTES * numElements]), UINT32( "uint32", (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( blockSize, gridPosition, - new int[numElements])), + new int[numElements]), + numElements -> new byte[Integer.BYTES * numElements]), UINT64( "uint64", (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( blockSize, gridPosition, - new long[numElements])), + new long[numElements]), + numElements -> new byte[Long.BYTES * numElements]), INT8( "int8", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, - new byte[numElements])), + new byte[numElements]), + numElements -> new byte[Byte.BYTES * numElements]), INT16( "int16", (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( blockSize, gridPosition, - new short[numElements])), + new short[numElements]), + numElements -> new byte[Short.BYTES * numElements]), INT32( "int32", (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( blockSize, gridPosition, - new int[numElements])), + new int[numElements]), + numElements -> new byte[Integer.BYTES * numElements]), INT64( "int64", (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( blockSize, gridPosition, - new long[numElements])), + new long[numElements]), + numElements -> new byte[Long.BYTES * numElements]), FLOAT32( "float32", (blockSize, gridPosition, numElements) -> new FloatArrayDataBlock( blockSize, gridPosition, - new float[numElements])), + new float[numElements]), + numElements -> new byte[Float.BYTES * numElements]), FLOAT64( "float64", (blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock( blockSize, gridPosition, - new double[numElements])), + new double[numElements]), + numElements -> new byte[Double.BYTES * numElements]), STRING( "string", (blockSize, gridPosition, numElements) -> new StringDataBlock( blockSize, gridPosition, - new byte[numElements])), + new byte[numElements]), + numElements -> new byte[numElements]), OBJECT( "object", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, - new byte[numElements])); + new byte[numElements]), + numElements -> new byte[numElements]); private final String label; private final DataBlockFactory dataBlockFactory; - DataType(final String label, final DataBlockFactory dataBlockFactory) { + private final IntFunction serializeArrayFactory; + + DataType(final String label, final DataBlockFactory dataBlockFactory, final IntFunction serializeArrayFactory) { this.label = label; this.dataBlockFactory = dataBlockFactory; + this.serializeArrayFactory = serializeArrayFactory; } @Override @@ -171,6 +187,14 @@ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosi return dataBlockFactory.createDataBlock(blockSize, gridPosition, DataBlock.getNumElements(blockSize)); } + /** + * TODO: javadoc + */ + public byte[] createSerializeArray(final int numElements) { + + return serializeArrayFactory.apply(numElements); + } + private interface DataBlockFactory { DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements); From ad30b381c74ddefa9fcfd33206ca1d658961c97c Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Jan 2025 15:19:15 -0500 Subject: [PATCH 128/423] WIP Compression / Codec API --- .../janelia/saalfeldlab/n5/Compression.java | 48 +++++++++++ .../saalfeldlab/n5/Java9StreamMethods.java | 84 +++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index df0ca49e1..e60fa0898 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -25,6 +25,11 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.io.Serializable; import java.lang.annotation.ElementType; import java.lang.annotation.Inherited; @@ -75,4 +80,47 @@ public default String getType() { public BlockReader getReader(); public BlockWriter getWriter(); + + + // --------------------------------------------------------------------------------- + // TODO. clean up interface hierarchy. + // getInputStream and getOutputStream are duplicated here from DefaultBlockReader/Writer + // to allow for default implementation of decode/encode (which are copied from wip/codecsShards). + InputStream getInputStream(final InputStream in) throws IOException; + + OutputStream getOutputStream(final OutputStream out) throws IOException; + + /** + * Decode an {@link InputStream}. + * + * @param in + * input stream + * @return the decoded input stream + */ + default InputStream decode(InputStream in) throws IOException { + return getInputStream(in); + } + + /** + * Encode an {@link OutputStream}. + * + * @param out + * the output stream + * @return the encoded output stream + */ + default OutputStream encode(OutputStream out) throws IOException { + return getOutputStream(out); + } + + default byte[] encode(byte[] data) throws IOException { + final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + final OutputStream encodedStream = encode(byteStream); + encodedStream.write(data); + encodedStream.close(); + return byteStream.toByteArray(); + } + + default byte[] decode(byte[] data) throws IOException { + return Java9StreamMethods.readAllBytes(decode(new ByteArrayInputStream(data))); + } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java b/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java new file mode 100644 index 000000000..613e1d4e5 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java @@ -0,0 +1,84 @@ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Methods from InputStream interface in Java9+ (copied and modified to static methods). + */ +public final class Java9StreamMethods { + + private Java9StreamMethods() {} + + public static byte[] readAllBytes(final InputStream in) throws IOException { + return readNBytes(in, Integer.MAX_VALUE); + } + + private static final int DEFAULT_BUFFER_SIZE = 8192; + + private static final int MAX_BUFFER_SIZE = Integer.MAX_VALUE - 8; + + private static byte[] readNBytes(final InputStream in, int len) throws IOException { + if (len < 0) { + throw new IllegalArgumentException("len < 0"); + } + + List bufs = null; + byte[] result = null; + int total = 0; + int remaining = len; + int n; + do { + byte[] buf = new byte[Math.min(remaining, DEFAULT_BUFFER_SIZE)]; + int nread = 0; + + // read to EOF which may read more or less than buffer size + while ((n = in.read(buf, nread, + Math.min(buf.length - nread, remaining))) > 0) { + nread += n; + remaining -= n; + } + + if (nread > 0) { + if (MAX_BUFFER_SIZE - total < nread) { + throw new OutOfMemoryError("Required array size too large"); + } + total += nread; + if (result == null) { + result = buf; + } else { + if (bufs == null) { + bufs = new ArrayList<>(); + bufs.add(result); + } + bufs.add(buf); + } + } + // if the last call to read returned -1 or the number of bytes + // requested have been read then break + } while (n >= 0 && remaining > 0); + + if (bufs == null) { + if (result == null) { + return new byte[0]; + } + return result.length == total ? + result : Arrays.copyOf(result, total); + } + + result = new byte[total]; + int offset = 0; + remaining = total; + for (byte[] b : bufs) { + int count = Math.min(b.length, remaining); + System.arraycopy(b, 0, result, offset, count); + offset += count; + remaining -= count; + } + + return result; + } +} From 17f197612afb58b1fc89aefbab63f4bd4308f2f5 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Jan 2025 17:17:15 -0500 Subject: [PATCH 129/423] WIP encode/decode implementations --- .../janelia/saalfeldlab/n5/BlockReader.java | 1 + .../janelia/saalfeldlab/n5/BlockWriter.java | 1 + .../saalfeldlab/n5/ByteArrayDataBlock.java | 11 +++++ .../janelia/saalfeldlab/n5/Compression.java | 3 +- .../org/janelia/saalfeldlab/n5/DataBlock.java | 47 +++++++++---------- .../saalfeldlab/n5/DefaultBlockReader.java | 22 +++++++-- .../saalfeldlab/n5/DefaultBlockWriter.java | 15 +++++- .../saalfeldlab/n5/DoubleArrayDataBlock.java | 14 ++++++ .../saalfeldlab/n5/FloatArrayDataBlock.java | 13 +++++ .../saalfeldlab/n5/IntArrayDataBlock.java | 14 ++++++ .../saalfeldlab/n5/LongArrayDataBlock.java | 13 +++++ .../org/janelia/saalfeldlab/n5/N5Reader.java | 2 +- .../saalfeldlab/n5/RawCompression.java | 10 ++++ .../saalfeldlab/n5/ShortArrayDataBlock.java | 13 +++++ .../saalfeldlab/n5/StringDataBlock.java | 23 +++++++-- 15 files changed, 164 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java index b788a3f40..f1ff749e9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java @@ -50,5 +50,6 @@ public interface BlockReader { * @throws IOException * the exception */ + @Deprecated public > void read(final B dataBlock, final InputStream in) throws IOException; } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java index 5120e7457..0fc7455ce 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java @@ -48,5 +48,6 @@ public interface BlockWriter { * @throws IOException * the exception */ + @Deprecated public void write(final DataBlock dataBlock, final OutputStream out) throws IOException; } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index fb106b89c..69316f5e6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.nio.ByteBuffer; +import java.nio.ByteOrder; public class ByteArrayDataBlock extends AbstractDataBlock { @@ -45,6 +46,16 @@ public void readData(final ByteBuffer buffer) { buffer.get(getData()); } + @Override + public byte[] serialize(final ByteOrder byteOrder) { + return data; + } + + @Override + public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { + System.arraycopy(serialized, 0, data, 0, data.length); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index e60fa0898..a9ce678db 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -36,7 +36,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; - import org.scijava.annotations.Indexable; /** @@ -123,4 +122,4 @@ default byte[] encode(byte[] data) throws IOException { default byte[] decode(byte[] data) throws IOException { return Java9StreamMethods.readAllBytes(decode(new ByteArrayInputStream(data))); } -} \ No newline at end of file +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 9ef5387a2..3f37b2813 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.nio.ByteBuffer; +import java.nio.ByteOrder; /** * Interface for data blocks. A data block has data, a position on the block @@ -66,33 +67,27 @@ public interface DataBlock { */ T getData(); - /** - * Creates a {@link ByteBuffer} that contains the data object of this data - * block. - * - * The {@link ByteBuffer} may or may not map directly to the data - * object of this data block. I.e. modifying the {@link ByteBuffer} after - * calling this method may or may not change the data of this data block. - * modifying the data object of this data block after calling this method - * may or may not change the content of the {@link ByteBuffer}. - * - * @return {@link ByteBuffer} containing data - */ - ByteBuffer toByteBuffer(); + default byte[] serialize() { + // TODO: LITTLE_ENDIAN would be preferable, but BIG_ENDIAN is backwards-compatible. + return serialize(ByteOrder.BIG_ENDIAN); + } - /** - * Reads the data object of this data block from a {@link ByteBuffer}. - * - * The {@link ByteBuffer} may or may not map directly to the data - * object of this data block. I.e. modifying the {@link ByteBuffer} after - * calling this method may or may not change the data of this data block. - * modifying the data object of this data block after calling this method - * may or may not change the content of the {@link ByteBuffer}. - * - * @param buffer - * the byte buffer - */ - void readData(final ByteBuffer buffer); + default byte[] serialize(ByteOrder byteOrder) { + throw new UnsupportedOperationException("TODO: implement " + this.getClass().getName() + ".serialize(ByteOrder)"); + } + + default void deserialize(byte[] serialized) { + // TODO: LITTLE_ENDIAN would be preferable, but BIG_ENDIAN is backwards-compatible. + deserialize(ByteOrder.BIG_ENDIAN, serialized); + } + + default void deserialize(ByteOrder byteOrder, byte[] serizalized) { + throw new UnsupportedOperationException("TODO: implement " + this.getClass().getName() + ".deserialize(ByteOrder, byte[])"); + } + + @Deprecated ByteBuffer toByteBuffer(); + + @Deprecated void readData(final ByteBuffer buffer); /** * Returns the number of elements in this {@link DataBlock}. This number is diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 58c59780c..b7b9fe7b1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -38,8 +38,10 @@ */ public interface DefaultBlockReader extends BlockReader { + @Deprecated public InputStream getInputStream(final InputStream in) throws IOException; + @Deprecated @Override public default > void read( final B dataBlock, @@ -71,6 +73,7 @@ public static DataBlock readBlock( final DatasetAttributes datasetAttributes, final long[] gridPosition) throws IOException { + final DataType dataType = datasetAttributes.getDataType(); final DataInputStream dis = new DataInputStream(in); final short mode = dis.readShort(); final int numElements; @@ -85,14 +88,25 @@ public static DataBlock readBlock( } else { numElements = dis.readInt(); } - dataBlock = datasetAttributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); + dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); } else { numElements = dis.readInt(); - dataBlock = datasetAttributes.getDataType().createDataBlock(null, gridPosition, numElements); + dataBlock = dataType.createDataBlock(null, gridPosition, numElements); } - final BlockReader reader = datasetAttributes.getCompression().getReader(); - reader.read(dataBlock, in); + // variant 1 +// final byte[] data = dataType.createSerializeArray(numElements); +// datasetAttributes.getCompression().decode(in).read(data); +// dataBlock.deserialize(data); + + // variant 2 + final byte[] data = datasetAttributes.getCompression().decode(Java9StreamMethods.readAllBytes(in)); + dataBlock.deserialize(data); + + // old +// final BlockReader reader = datasetAttributes.getCompression().getReader(); +// reader.read(dataBlock, in); + return dataBlock; } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index c53aae2df..60b934c30 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -38,8 +38,10 @@ */ public interface DefaultBlockWriter extends BlockWriter { + @Deprecated public OutputStream getOutputStream(final OutputStream out) throws IOException; + @Deprecated @Override public default void write( final DataBlock dataBlock, @@ -92,7 +94,16 @@ else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSiz dos.flush(); - final BlockWriter writer = datasetAttributes.getCompression().getWriter(); - writer.write(dataBlock, out); + // variant 1 +// final byte[] data = dataBlock.serialize(); +// datasetAttributes.getCompression().getOutputStream(out).write(data); + + // variant 2 + final byte[] data = datasetAttributes.getCompression().encode(dataBlock.serialize()); + out.write(data); + + // old +// final BlockWriter writer = datasetAttributes.getCompression().getWriter(); +// writer.write(dataBlock, out); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index ba7dc606a..b48cfbf91 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.nio.ByteBuffer; +import java.nio.ByteOrder; public class DoubleArrayDataBlock extends AbstractDataBlock { @@ -46,6 +47,19 @@ public void readData(final ByteBuffer buffer) { buffer.asDoubleBuffer().get(data); } + + @Override + public byte[] serialize(final ByteOrder byteOrder) { + final ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES * data.length); + buffer.order(byteOrder).asDoubleBuffer().put(data); + return buffer.array(); + } + + @Override + public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { + ByteBuffer.wrap(serialized).order(byteOrder).asDoubleBuffer().get(data); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index 2730e5d7c..7f5a4f0b2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.nio.ByteBuffer; +import java.nio.ByteOrder; public class FloatArrayDataBlock extends AbstractDataBlock { @@ -46,6 +47,18 @@ public void readData(final ByteBuffer buffer) { buffer.asFloatBuffer().get(data); } + @Override + public byte[] serialize(final ByteOrder byteOrder) { + final ByteBuffer buffer = ByteBuffer.allocate(Float.BYTES * data.length); + buffer.order(byteOrder).asFloatBuffer().put(data); + return buffer.array(); + } + + @Override + public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { + ByteBuffer.wrap(serialized).order(byteOrder).asFloatBuffer().get(data); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 797002be5..ad808d3e8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -26,6 +26,8 @@ package org.janelia.saalfeldlab.n5; import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; public class IntArrayDataBlock extends AbstractDataBlock { @@ -46,6 +48,18 @@ public void readData(final ByteBuffer buffer) { buffer.asIntBuffer().get(data); } + @Override + public byte[] serialize(final ByteOrder byteOrder) { + final ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES * data.length); + buffer.order(byteOrder).asIntBuffer().put(data); + return buffer.array(); + } + + @Override + public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { + ByteBuffer.wrap(serialized).order(byteOrder).asIntBuffer().get(data); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index 7c8a812f2..f71543f3f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.nio.ByteBuffer; +import java.nio.ByteOrder; public class LongArrayDataBlock extends AbstractDataBlock { @@ -46,6 +47,18 @@ public void readData(final ByteBuffer buffer) { buffer.asLongBuffer().get(data); } + @Override + public byte[] serialize(final ByteOrder byteOrder) { + final ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * data.length); + buffer.order(byteOrder).asLongBuffer().put(data); + return buffer.array(); + } + + @Override + public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { + ByteBuffer.wrap(serialized).order(byteOrder).asLongBuffer().get(data); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 010e757f6..a7102d57b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -322,7 +322,7 @@ default T readSerializedBlock( if (block == null) return null; - final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.toByteBuffer().array()); + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.serialize()); try (ObjectInputStream in = new ObjectInputStream(byteArrayInputStream)) { return (T)in.readObject(); } catch (final IOException | UncheckedIOException e) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index ffa674fc4..0db2d79d1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -48,6 +48,16 @@ public OutputStream getOutputStream(final OutputStream out) throws IOException { return out; } + @Override + public byte[] encode(final byte[] data) throws IOException { + return data; + } + + @Override + public byte[] decode(final byte[] data) throws IOException { + return data; + } + @Override public RawCompression getReader() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 61b5a81f2..2a6133400 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.nio.ByteBuffer; +import java.nio.ByteOrder; public class ShortArrayDataBlock extends AbstractDataBlock { @@ -46,6 +47,18 @@ public void readData(final ByteBuffer buffer) { buffer.asShortBuffer().get(data); } + @Override + public byte[] serialize(final ByteOrder byteOrder) { + final ByteBuffer buffer = ByteBuffer.allocate(Short.BYTES * data.length); + buffer.order(byteOrder).asShortBuffer().put(data); + return buffer.array(); + } + + @Override + public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { + ByteBuffer.wrap(serialized).order(byteOrder).asShortBuffer().get(data); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index e4ad904e2..3e9326201 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -57,7 +58,7 @@ public void readData(final ByteBuffer buffer) { if (buffer.hasArray()) { if (buffer.array() != serializedData) buffer.get(serializedData); - actualData = deserialize(buffer.array()); + actualData = _deserialize(buffer.array()); } else actualData = ENCODING.decode(buffer).toString().split(NULLCHAR); } @@ -67,7 +68,7 @@ protected byte[] serialize(String[] strings) { return flattenedArray.getBytes(ENCODING); } - protected String[] deserialize(byte[] rawBytes) { + protected String[] _deserialize(byte[] rawBytes) { final String rawChars = new String(rawBytes, ENCODING); return rawChars.split(NULLCHAR); } @@ -82,7 +83,23 @@ public int getNumElements() { @Override public String[] getData() { if (actualData == null) - actualData = deserialize(serializedData); + actualData = _deserialize(serializedData); return actualData; } + + + + + + @Override + public byte[] serialize(final ByteOrder byteOrder) { + final String flattenedArray = String.join(NULLCHAR, actualData) + NULLCHAR; + return flattenedArray.getBytes(ENCODING); + } + + @Override + public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { + final String rawChars = new String(serialized, ENCODING); + actualData = rawChars.split(NULLCHAR); + } } From e8dde834dcd6234069df70951aeda5f64c19b8d2 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Jan 2025 18:25:57 -0500 Subject: [PATCH 130/423] bugfix --- .../java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index b7b9fe7b1..4a6bc73a8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -96,7 +96,10 @@ public static DataBlock readBlock( // variant 1 // final byte[] data = dataType.createSerializeArray(numElements); -// datasetAttributes.getCompression().decode(in).read(data); +// try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { +// final DataInputStream dis2 = new DataInputStream(inflater); +// dis2.readFully(data); +// } // dataBlock.deserialize(data); // variant 2 From 63f51ad6c08461fa0912896e18c3871493671688 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Jan 2025 18:29:53 -0500 Subject: [PATCH 131/423] cleanup --- src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 3f37b2813..9bcde5df0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -72,18 +72,14 @@ default byte[] serialize() { return serialize(ByteOrder.BIG_ENDIAN); } - default byte[] serialize(ByteOrder byteOrder) { - throw new UnsupportedOperationException("TODO: implement " + this.getClass().getName() + ".serialize(ByteOrder)"); - } + byte[] serialize(ByteOrder byteOrder); default void deserialize(byte[] serialized) { // TODO: LITTLE_ENDIAN would be preferable, but BIG_ENDIAN is backwards-compatible. deserialize(ByteOrder.BIG_ENDIAN, serialized); } - default void deserialize(ByteOrder byteOrder, byte[] serizalized) { - throw new UnsupportedOperationException("TODO: implement " + this.getClass().getName() + ".deserialize(ByteOrder, byte[])"); - } + void deserialize(ByteOrder byteOrder, byte[] serialized); @Deprecated ByteBuffer toByteBuffer(); From eba1fd750322e1d8b0072e6b2a5297dfbe9cb00b Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 24 Jan 2025 10:11:00 -0500 Subject: [PATCH 132/423] wip --- .../saalfeldlab/n5/DefaultBlockReader.java | 17 +++++++++-------- .../saalfeldlab/n5/Java9StreamMethods.java | 1 + .../janelia/saalfeldlab/n5/N5ReadBenchmark.java | 5 +++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 4a6bc73a8..c8bf4a008 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -25,6 +25,7 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.BufferedInputStream; import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; @@ -95,16 +96,16 @@ public static DataBlock readBlock( } // variant 1 -// final byte[] data = dataType.createSerializeArray(numElements); -// try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { -// final DataInputStream dis2 = new DataInputStream(inflater); -// dis2.readFully(data); -// } -// dataBlock.deserialize(data); + final byte[] data = dataType.createSerializeArray(numElements); + try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { + final DataInputStream dis2 = new DataInputStream(inflater); + dis2.readFully(data); + } + dataBlock.deserialize(data); // variant 2 - final byte[] data = datasetAttributes.getCompression().decode(Java9StreamMethods.readAllBytes(in)); - dataBlock.deserialize(data); +// final byte[] data = datasetAttributes.getCompression().decode(Java9StreamMethods.readAllBytes(in)); +// dataBlock.deserialize(data); // old // final BlockReader reader = datasetAttributes.getCompression().getReader(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java b/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java index 613e1d4e5..a0daf978f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java @@ -14,6 +14,7 @@ public final class Java9StreamMethods { private Java9StreamMethods() {} public static byte[] readAllBytes(final InputStream in) throws IOException { +// return in.readAllBytes(); return readNBytes(in, Integer.MAX_VALUE); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java index f26b25e9f..659e17d8e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java @@ -85,7 +85,8 @@ public void setup() { System.out.println("basePath = " + basePath); try (final N5Writer n5 = createTempN5Writer()) { - final Compression compression = new Lz4Compression(); +// final Compression compression = new Lz4Compression(); + final Compression compression = new RawCompression(); n5.createDataset(datasetName, dimensions, blockSize, DataType.FLOAT64, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final DoubleArrayDataBlock dataBlock = new DoubleArrayDataBlock(blockSize, new long[] {0, 0, 0}, doubleBlock); @@ -111,7 +112,7 @@ public static void main( final String... args ) throws RunnerException, IOExcept { final Options opt = new OptionsBuilder() .include( N5ReadBenchmark.class.getSimpleName() ) - .warmupIterations( 4 ) + .warmupIterations( 8 ) .measurementIterations( 8 ) .warmupTime( TimeValue.milliseconds( 500 ) ) .measurementTime( TimeValue.milliseconds( 500 ) ) From b94ba8b10b7c8024380987043ac5c6d1780d01ae Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 3 Sep 2024 16:48:17 -0400 Subject: [PATCH 133/423] feat: DataBlock methods to read/write directly from DataInput/Output --- .../saalfeldlab/n5/AbstractDataBlock.java | 21 +++++++++++++++++++ .../saalfeldlab/n5/ByteArrayDataBlock.java | 8 +++++++ .../org/janelia/saalfeldlab/n5/DataBlock.java | 7 +++++++ .../saalfeldlab/n5/DoubleArrayDataBlock.java | 9 ++++++++ .../saalfeldlab/n5/FloatArrayDataBlock.java | 9 ++++++++ .../saalfeldlab/n5/IntArrayDataBlock.java | 20 ++++++++++++++++-- .../saalfeldlab/n5/LongArrayDataBlock.java | 17 +++++++++++++++ .../saalfeldlab/n5/ShortArrayDataBlock.java | 9 ++++++++ 8 files changed, 98 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index f1cbc3529..59208fcfe 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -25,6 +25,11 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.nio.ByteBuffer; + /** * Abstract base class for {@link DataBlock} implementations. * @@ -63,4 +68,20 @@ public T getData() { return data; } + + @Override + public void readData(final DataInput input) throws IOException { + + final ByteBuffer buffer = toByteBuffer(); + input.readFully(buffer.array()); + readData(buffer); + } + + @Override + public void writeData(final DataOutput output) throws IOException { + + final ByteBuffer buffer = toByteBuffer(); + output.write(buffer.array()); + } + } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index 69316f5e6..2e454c1ec 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -25,6 +25,8 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -46,6 +48,12 @@ public void readData(final ByteBuffer buffer) { buffer.get(getData()); } + @Override + public void readData(final DataInput inputStream) throws IOException { + + inputStream.readFully(data); + } + @Override public byte[] serialize(final ByteOrder byteOrder) { return data; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 9bcde5df0..e13b8f736 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -25,6 +25,9 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -85,6 +88,10 @@ default void deserialize(byte[] serialized) { @Deprecated void readData(final ByteBuffer buffer); + public void readData(final DataInput inputStream) throws IOException; + + public void writeData(final DataOutput output) throws IOException; + /** * Returns the number of elements in this {@link DataBlock}. This number is * not necessarily equal {@link #getNumElements(int[]) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index b48cfbf91..58edcafb3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -25,6 +25,8 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -60,6 +62,13 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { ByteBuffer.wrap(serialized).order(byteOrder).asDoubleBuffer().get(data); } + @Override + public void readData(final DataInput inputStream) throws IOException { + + for (int i = 0; i < data.length; i++) + data[i] = inputStream.readDouble(); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index 7f5a4f0b2..87b986aeb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -25,6 +25,8 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -47,6 +49,13 @@ public void readData(final ByteBuffer buffer) { buffer.asFloatBuffer().get(data); } + @Override + public void readData(final DataInput inputStream) throws IOException { + + for (int i = 0; i < data.length; i++) + data[i] = inputStream.readFloat(); + } + @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Float.BYTES * data.length); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index ad808d3e8..a9db9e42a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -25,12 +25,14 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.FloatBuffer; -public class IntArrayDataBlock extends AbstractDataBlock { +public class IntArrayDataBlock extends AbstractDataBlock { public IntArrayDataBlock(final int[] size, final long[] gridPosition, final int[] data) { super(size, gridPosition, data); @@ -48,6 +50,20 @@ public void readData(final ByteBuffer buffer) { buffer.asIntBuffer().get(data); } + @Override + public void readData(final DataInput input) throws IOException { + + for (int i = 0; i < data.length; i++) + data[i] = input.readInt(); + } + + @Override + public void writeData(final DataOutput output) throws IOException { + + for (int i = 0; i < data.length; i++) + output.writeInt(data[i]); + } + @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES * data.length); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index f71543f3f..fc041cf0b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -25,6 +25,9 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -47,6 +50,20 @@ public void readData(final ByteBuffer buffer) { buffer.asLongBuffer().get(data); } + @Override + public void readData(final DataInput inputStream) throws IOException { + + for (int i = 0; i < data.length; i++) + data[i] = inputStream.readLong(); + } + + @Override + public void writeData(final DataOutput output) throws IOException { + + for (int i = 0; i < data.length; i++) + output.writeLong(data[i]); + } + @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * data.length); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 2a6133400..671417e09 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -25,6 +25,8 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInput; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -47,6 +49,13 @@ public void readData(final ByteBuffer buffer) { buffer.asShortBuffer().get(data); } + @Override + public void readData(final DataInput dataInput) throws IOException { + + for (int i = 0; i < data.length; i++) + data[i] = dataInput.readShort(); + } + @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Short.BYTES * data.length); From 953715bcf55ef376dcc8995ce9b9bdd57937324d Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 24 Jan 2025 20:22:33 -0500 Subject: [PATCH 134/423] WIP: SplittableReadData --- .../janelia/saalfeldlab/n5/Splittable.java | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/Splittable.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java b/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java new file mode 100644 index 000000000..1a0a9df6d --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java @@ -0,0 +1,171 @@ +package org.janelia.saalfeldlab.n5; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class Splittable { + + + public interface ReadData { + + /** + * Returns number of bytes in this {@link ReadData}, if known. Otherwise + * {@code -1}. + * + * @return number of bytes, if known, or -1 + * + * @throws IOException + * if an I/O error occurs while trying to get the length + */ + default long length() throws IOException { + return -1; + } + + /** + * Open a {@code InputStream} on this data. + *

      + * Repeatedly calling this method may or may not work, depending on how + * the underlying data is stored. For example, if the underlying data is + * stored as a {@code byte[]} array, multiple streams can be opened. If + * the underlying data is just an {@code InputStream} then this will be + * returned on the first call. + * + * @return an InputStream on this data + * + * @throws IOException + * if any I/O error occurs + * @throws IllegalStateException + * if this method was already called once and cannot be called again. + */ + InputStream inputStream() throws IOException, IllegalStateException; + + /** + * If this {@code ReadData} is a {@code SplittableReadData}, just returns {@code this}. + *

      + * Otherwise, if the underlying data is an {@code InputStream}, all data is read and + * wrapped as a {@code ByteArraySplittableReadData}. + *

      + * The returned {@code SplittableReadData} has a known {@link #length} + * and multiple {@link #inputStream}s can be opened on it. + */ + SplittableReadData splittable() throws IOException; + } + + public interface SplittableReadData extends ReadData { + + ReadData split(final long offset, final long length) throws IOException; + } + + public static class ByteArraySplittableReadData implements SplittableReadData { + + private final byte[] data; + private final int offset; + private final int length; + + ByteArraySplittableReadData(final byte[] data, final int offset, final int length) { + this.data = data; + this.offset = offset; + this.length = length; + } + + @Override + public long length() { + return length; + } + + @Override + public InputStream inputStream() throws IOException { + return new ByteArrayInputStream(data, offset, length); + } + + @Override + public SplittableReadData splittable() throws IOException { + return this; + } + + @Override + public SplittableReadData split(final long offset, final long length) throws IOException { + if (offset < 0 || offset > this.length || length < 0) { + throw new IndexOutOfBoundsException(); + } + final int o = this.offset + (int) offset; + final int l = Math.min((int) length, this.length - o); + return new ByteArraySplittableReadData(data, o, l); + } + } + + public static class InputStreamReadData implements ReadData { + + private final InputStream inputStream; + private final int length; + private ByteArraySplittableReadData bytes; + + InputStreamReadData(final InputStream inputStream, final int length) { + this.inputStream = inputStream; + this.length = length; + } + + InputStreamReadData(final InputStream inputStream) { + this(inputStream, -1); + } + + @Override + public long length() { + return length; + } + + @Override + public SplittableReadData splittable() throws IOException { + if (bytes == null) { + final byte[] data; + if (length >= 0) { + data = new byte[length]; + new DataInputStream(inputStream()).readFully(data); + } else { + data = Java9StreamMethods.readAllBytes(inputStream()); + } + bytes = new ByteArraySplittableReadData(data, 0, data.length); + } + return bytes; + } + + private boolean inputStreamCalled = false; + + @Override + public InputStream inputStream() throws IOException { + if ( inputStreamCalled ) { + throw new IOException("InputStream() already called"); + } else { + inputStreamCalled = true; + return inputStream; + } + } + } + + public static class KeyValueAccessReadData implements ReadData { + + private final KeyValueAccess keyValueAccess; + private final String normalPath; + + KeyValueAccessReadData(final KeyValueAccess keyValueAccess, final String normalPath) { + this.keyValueAccess = keyValueAccess; + this.normalPath = normalPath; + } + + @Override + public InputStream inputStream() throws IOException, IllegalStateException { + final LockedChannel lockedChannel = keyValueAccess.lockForReading(normalPath); + return lockedChannel.newInputStream(); + // TODO bind close() of the InputStream to close of the lockedChannel + } + + @Override + public SplittableReadData splittable() throws IOException { + return null; + } + } + + +} From 22ece34ecb5cdd98dc2f4fb5ca0fb2c1796e49e2 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 24 Jan 2025 20:23:17 -0500 Subject: [PATCH 135/423] WIP DataBlock readData variants --- .../saalfeldlab/n5/AbstractDataBlock.java | 6 ++---- .../saalfeldlab/n5/DefaultBlockReader.java | 18 ++++++++++++------ .../saalfeldlab/n5/DoubleArrayDataBlock.java | 10 +++++++++- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index 59208fcfe..c79cf720c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -69,9 +69,9 @@ public T getData() { return data; } + @Deprecated @Override public void readData(final DataInput input) throws IOException { - final ByteBuffer buffer = toByteBuffer(); input.readFully(buffer.array()); readData(buffer); @@ -79,9 +79,7 @@ public void readData(final DataInput input) throws IOException { @Override public void writeData(final DataOutput output) throws IOException { - - final ByteBuffer buffer = toByteBuffer(); - output.write(buffer.array()); + output.write(serialize()); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index c8bf4a008..213aef42b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -96,17 +96,23 @@ public static DataBlock readBlock( } // variant 1 - final byte[] data = dataType.createSerializeArray(numElements); - try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { - final DataInputStream dis2 = new DataInputStream(inflater); - dis2.readFully(data); - } - dataBlock.deserialize(data); +// final byte[] data = dataType.createSerializeArray(numElements); +// try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { +// final DataInputStream dis2 = new DataInputStream(inflater); +// dis2.readFully(data); +// } +// dataBlock.deserialize(data); // variant 2 // final byte[] data = datasetAttributes.getCompression().decode(Java9StreamMethods.readAllBytes(in)); // dataBlock.deserialize(data); + // variant 3 +// try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { +// final DataInputStream dis2 = new DataInputStream(inflater); +// dataBlock.readData(dis2); +// } + // old // final BlockReader reader = datasetAttributes.getCompression().getReader(); // reader.read(dataBlock, in); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 58edcafb3..e7f1ebcb0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -26,9 +26,11 @@ package org.janelia.saalfeldlab.n5; import java.io.DataInput; +import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.Arrays; public class DoubleArrayDataBlock extends AbstractDataBlock { @@ -49,7 +51,6 @@ public void readData(final ByteBuffer buffer) { buffer.asDoubleBuffer().get(data); } - @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES * data.length); @@ -69,6 +70,13 @@ public void readData(final DataInput inputStream) throws IOException { data[i] = inputStream.readDouble(); } + @Override + public void writeData(final DataOutput output) throws IOException { + + for (int i = 0; i < data.length; i++) + output.writeDouble(data[i]); + } + @Override public int getNumElements() { From a18d137e179674e22d542d038bde571389d64b92 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 24 Jan 2025 21:22:58 -0500 Subject: [PATCH 136/423] WIP SplittableReadData --- .../janelia/saalfeldlab/n5/Splittable.java | 84 +++++++++++-------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java b/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java index 1a0a9df6d..e91b674b2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java @@ -2,12 +2,12 @@ import java.io.ByteArrayInputStream; import java.io.DataInputStream; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; public class Splittable { - public interface ReadData { /** @@ -96,30 +96,16 @@ public SplittableReadData split(final long offset, final long length) throws IOE } } - public static class InputStreamReadData implements ReadData { + // not thread-safe + private abstract static class AbstractInputStreamReadData implements ReadData { - private final InputStream inputStream; - private final int length; private ByteArraySplittableReadData bytes; - InputStreamReadData(final InputStream inputStream, final int length) { - this.inputStream = inputStream; - this.length = length; - } - - InputStreamReadData(final InputStream inputStream) { - this(inputStream, -1); - } - - @Override - public long length() { - return length; - } - @Override public SplittableReadData splittable() throws IOException { if (bytes == null) { final byte[] data; + final int length = (int) length(); if (length >= 0) { data = new byte[length]; new DataInputStream(inputStream()).readFully(data); @@ -130,13 +116,34 @@ public SplittableReadData splittable() throws IOException { } return bytes; } + } + + // not thread-safe + public static class InputStreamReadData extends AbstractInputStreamReadData { + + private final InputStream inputStream; + private final int length; + + public InputStreamReadData(final InputStream inputStream, final int length) { + this.inputStream = inputStream; + this.length = length; + } + + public InputStreamReadData(final InputStream inputStream) { + this(inputStream, -1); + } + + @Override + public long length() { + return length; + } private boolean inputStreamCalled = false; @Override - public InputStream inputStream() throws IOException { - if ( inputStreamCalled ) { - throw new IOException("InputStream() already called"); + public InputStream inputStream() throws IllegalStateException { + if (inputStreamCalled) { + throw new IllegalStateException("InputStream() already called"); } else { inputStreamCalled = true; return inputStream; @@ -144,28 +151,39 @@ public InputStream inputStream() throws IOException { } } - public static class KeyValueAccessReadData implements ReadData { + public static class KeyValueAccessReadData extends AbstractInputStreamReadData { private final KeyValueAccess keyValueAccess; private final String normalPath; - KeyValueAccessReadData(final KeyValueAccess keyValueAccess, final String normalPath) { + public KeyValueAccessReadData(final KeyValueAccess keyValueAccess, final String normalPath) { this.keyValueAccess = keyValueAccess; this.normalPath = normalPath; } + /** + * Open a {@code InputStream} on this data. + *

      + * This will open a {@code LockedChannel} on the underlying {@code + * KeyValueAccess}. Make sure to {@code close()} the returned {@code + * InputStream} to release the underlying {@code LockedChannel}. + * + * @return an InputStream on this data + * + * @throws IOException + * if any I/O error occurs + */ @Override - public InputStream inputStream() throws IOException, IllegalStateException { - final LockedChannel lockedChannel = keyValueAccess.lockForReading(normalPath); - return lockedChannel.newInputStream(); - // TODO bind close() of the InputStream to close of the lockedChannel - } - - @Override - public SplittableReadData splittable() throws IOException { - return null; + public InputStream inputStream() throws IOException { + final LockedChannel channel = keyValueAccess.lockForReading(normalPath); + return new FilterInputStream(channel.newInputStream()) { + @Override + public void close() throws IOException { + in.close(); + channel.close(); + } + }; } } - } From 8811f1d570817fbd2b1e35ae81b0e065bbece606 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 25 Jan 2025 10:58:37 -0500 Subject: [PATCH 137/423] WIP clean up Compression interface --- .../janelia/saalfeldlab/n5/Bzip2Compression.java | 12 ------------ .../org/janelia/saalfeldlab/n5/Compression.java | 13 +++++-------- .../janelia/saalfeldlab/n5/GzipCompression.java | 12 ------------ .../janelia/saalfeldlab/n5/Lz4Compression.java | 12 ------------ .../janelia/saalfeldlab/n5/RawCompression.java | 12 ------------ .../org/janelia/saalfeldlab/n5/XzCompression.java | 15 +-------------- 6 files changed, 6 insertions(+), 70 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 5d3d61614..b57bb16d9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -63,18 +63,6 @@ public OutputStream getOutputStream(final OutputStream out) throws IOException { return new BZip2CompressorOutputStream(out, blockSize); } - @Override - public Bzip2Compression getReader() { - - return this; - } - - @Override - public Bzip2Compression getWriter() { - - return this; - } - @Override public boolean equals(final Object other) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index a9ce678db..9f7667ba9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -53,7 +53,7 @@ public interface Compression extends Serializable { @Inherited @Target(ElementType.TYPE) @Indexable - public static @interface CompressionType { + @interface CompressionType { String value(); } @@ -65,9 +65,9 @@ public interface Compression extends Serializable { @Retention(RetentionPolicy.RUNTIME) @Inherited @Target(ElementType.FIELD) - public static @interface CompressionParameter {} + @interface CompressionParameter {} - public default String getType() { + default String getType() { final CompressionType compressionType = getClass().getAnnotation(CompressionType.class); if (compressionType == null) @@ -76,11 +76,6 @@ public default String getType() { return compressionType.value(); } - public BlockReader getReader(); - - public BlockWriter getWriter(); - - // --------------------------------------------------------------------------------- // TODO. clean up interface hierarchy. // getInputStream and getOutputStream are duplicated here from DefaultBlockReader/Writer @@ -111,6 +106,7 @@ default OutputStream encode(OutputStream out) throws IOException { return getOutputStream(out); } + // TODO probably remove? default byte[] encode(byte[] data) throws IOException { final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); final OutputStream encodedStream = encode(byteStream); @@ -119,6 +115,7 @@ default byte[] encode(byte[] data) throws IOException { return byteStream.toByteArray(); } + // TODO probably remove? default byte[] decode(byte[] data) throws IOException { return Java9StreamMethods.readAllBytes(decode(new ByteArrayInputStream(data))); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index b691a6d3e..f5a850e87 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -88,18 +88,6 @@ public OutputStream getOutputStream(final OutputStream out) throws IOException { } } - @Override - public GzipCompression getReader() { - - return this; - } - - @Override - public GzipCompression getWriter() { - - return this; - } - private void readObject(final ObjectInputStream in) throws Exception { in.defaultReadObject(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index d76e4fe53..241944088 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -64,18 +64,6 @@ public OutputStream getOutputStream(final OutputStream out) throws IOException { return new LZ4BlockOutputStream(out, blockSize); } - @Override - public Lz4Compression getReader() { - - return this; - } - - @Override - public Lz4Compression getWriter() { - - return this; - } - @Override public boolean equals(final Object other) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index 0db2d79d1..8e06eecf9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -58,18 +58,6 @@ public byte[] decode(final byte[] data) throws IOException { return data; } - @Override - public RawCompression getReader() { - - return this; - } - - @Override - public RawCompression getWriter() { - - return this; - } - @Override public boolean equals(final Object other) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index 5204e799f..91f586247 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -34,7 +34,7 @@ import org.janelia.saalfeldlab.n5.Compression.CompressionType; @CompressionType("xz") -public class XzCompression implements DefaultBlockReader, DefaultBlockWriter, Compression { +public class XzCompression implements Compression { private static final long serialVersionUID = -7272153943564743774L; @@ -63,18 +63,6 @@ public OutputStream getOutputStream(final OutputStream out) throws IOException { return new XZCompressorOutputStream(out, preset); } - @Override - public XzCompression getReader() { - - return this; - } - - @Override - public XzCompression getWriter() { - - return this; - } - @Override public boolean equals(final Object other) { @@ -83,5 +71,4 @@ public boolean equals(final Object other) { else return preset == ((XzCompression)other).preset; } - } From 069f3e2755c1adad561b61223452a5143d85482f Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 25 Jan 2025 10:59:29 -0500 Subject: [PATCH 138/423] WIP speed up DataBlock.readData(InputStream), and clean up --- .../saalfeldlab/n5/AbstractDataBlock.java | 21 +--------- .../saalfeldlab/n5/ByteArrayDataBlock.java | 20 ++++------ .../org/janelia/saalfeldlab/n5/DataBlock.java | 15 +++---- .../saalfeldlab/n5/DefaultBlockReader.java | 39 +++---------------- .../saalfeldlab/n5/DoubleArrayDataBlock.java | 31 +++++---------- .../saalfeldlab/n5/FloatArrayDataBlock.java | 21 +++++----- .../saalfeldlab/n5/IntArrayDataBlock.java | 23 +++++------ .../saalfeldlab/n5/LongArrayDataBlock.java | 28 +++++-------- .../saalfeldlab/n5/ShortArrayDataBlock.java | 25 ++++++------ .../saalfeldlab/n5/StringDataBlock.java | 15 ++++--- 10 files changed, 77 insertions(+), 161 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index c79cf720c..822ceddab 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -25,11 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.nio.ByteBuffer; - /** * Abstract base class for {@link DataBlock} implementations. * @@ -68,18 +63,4 @@ public T getData() { return data; } - - @Deprecated - @Override - public void readData(final DataInput input) throws IOException { - final ByteBuffer buffer = toByteBuffer(); - input.readFully(buffer.array()); - readData(buffer); - } - - @Override - public void writeData(final DataOutput output) throws IOException { - output.write(serialize()); - } - -} \ No newline at end of file +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index 2e454c1ec..6956cec99 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -25,8 +25,9 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; +import java.io.DataInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -42,18 +43,6 @@ public ByteBuffer toByteBuffer() { return ByteBuffer.wrap(getData()); } - public void readData(final ByteBuffer buffer) { - - if (buffer.array() != getData()) - buffer.get(getData()); - } - - @Override - public void readData(final DataInput inputStream) throws IOException { - - inputStream.readFully(data); - } - @Override public byte[] serialize(final ByteOrder byteOrder) { return data; @@ -64,6 +53,11 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { System.arraycopy(serialized, 0, data, 0, data.length); } + @Override + public void readData(final InputStream inputStream) throws IOException { + new DataInputStream(inputStream).readFully(data); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index e13b8f736..e3d04ec40 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -28,6 +28,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -44,7 +45,7 @@ public interface DataBlock { /** * Returns the size of this data block. - * + *

      * The size of a data block is expected to be smaller than or equal to the * spacing of the block grid. The dimensionality of size is expected to be * equal to the dimensionality of the dataset. Consistency is not enforced. @@ -55,7 +56,7 @@ public interface DataBlock { /** * Returns the position of this data block on the block grid. - * + *

      * The dimensionality of the grid position is expected to be equal to the * dimensionality of the dataset. Consistency is not enforced. * @@ -71,26 +72,22 @@ public interface DataBlock { T getData(); default byte[] serialize() { - // TODO: LITTLE_ENDIAN would be preferable, but BIG_ENDIAN is backwards-compatible. return serialize(ByteOrder.BIG_ENDIAN); } byte[] serialize(ByteOrder byteOrder); default void deserialize(byte[] serialized) { - // TODO: LITTLE_ENDIAN would be preferable, but BIG_ENDIAN is backwards-compatible. deserialize(ByteOrder.BIG_ENDIAN, serialized); } void deserialize(ByteOrder byteOrder, byte[] serialized); - @Deprecated ByteBuffer toByteBuffer(); + void readData(final InputStream inputStream) throws IOException; - @Deprecated void readData(final ByteBuffer buffer); - - public void readData(final DataInput inputStream) throws IOException; + @Deprecated ByteBuffer toByteBuffer(); - public void writeData(final DataOutput output) throws IOException; +// void writeData(final DataOutput output) throws IOException; /** * Returns the number of elements in this {@link DataBlock}. This number is diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 213aef42b..1873e4939 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -37,24 +37,7 @@ * @author Stephan Saalfeld * @author Igor Pisarev */ -public interface DefaultBlockReader extends BlockReader { - - @Deprecated - public InputStream getInputStream(final InputStream in) throws IOException; - - @Deprecated - @Override - public default > void read( - final B dataBlock, - final InputStream in) throws IOException { - - final ByteBuffer buffer = dataBlock.toByteBuffer(); - try (final InputStream inflater = getInputStream(in)) { - final DataInputStream dis = new DataInputStream(inflater); - dis.readFully(buffer.array()); - } - dataBlock.readData(buffer); - } +public interface DefaultBlockReader { /** * Reads a {@link DataBlock} from an {@link InputStream}. @@ -69,7 +52,7 @@ public default > void read( * @throws IOException * the exception */ - public static DataBlock readBlock( + static DataBlock readBlock( final InputStream in, final DatasetAttributes datasetAttributes, final long[] gridPosition) throws IOException { @@ -95,7 +78,6 @@ public static DataBlock readBlock( dataBlock = dataType.createDataBlock(null, gridPosition, numElements); } - // variant 1 // final byte[] data = dataType.createSerializeArray(numElements); // try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { // final DataInputStream dis2 = new DataInputStream(inflater); @@ -103,19 +85,10 @@ public static DataBlock readBlock( // } // dataBlock.deserialize(data); - // variant 2 -// final byte[] data = datasetAttributes.getCompression().decode(Java9StreamMethods.readAllBytes(in)); -// dataBlock.deserialize(data); - - // variant 3 -// try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { -// final DataInputStream dis2 = new DataInputStream(inflater); -// dataBlock.readData(dis2); -// } - - // old -// final BlockReader reader = datasetAttributes.getCompression().getReader(); -// reader.read(dataBlock, in); + try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { + final DataInputStream dis2 = new DataInputStream(inflater); + dataBlock.readData(dis2); + } return dataBlock; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index e7f1ebcb0..bfa159775 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -25,12 +25,11 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; -import java.io.DataOutput; +import java.io.DataInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.Arrays; public class DoubleArrayDataBlock extends AbstractDataBlock { @@ -39,18 +38,14 @@ public DoubleArrayDataBlock(final int[] size, final long[] gridPosition, final d super(size, gridPosition, data); } + @Deprecated + @Override public ByteBuffer toByteBuffer() { - final ByteBuffer buffer = ByteBuffer.allocate(data.length * 8); buffer.asDoubleBuffer().put(data); return buffer; } - public void readData(final ByteBuffer buffer) { - - buffer.asDoubleBuffer().get(data); - } - @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES * data.length); @@ -64,22 +59,14 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { } @Override - public void readData(final DataInput inputStream) throws IOException { - - for (int i = 0; i < data.length; i++) - data[i] = inputStream.readDouble(); - } - - @Override - public void writeData(final DataOutput output) throws IOException { - - for (int i = 0; i < data.length; i++) - output.writeDouble(data[i]); + public void readData(final InputStream inputStream) throws IOException { + final byte[] bytes = DataType.FLOAT64.createSerializeArray(getNumElements()); + new DataInputStream(inputStream).readFully(bytes); + deserialize(bytes); } @Override public int getNumElements() { - return data.length; } -} \ No newline at end of file +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index 87b986aeb..1653c8315 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -26,7 +26,9 @@ package org.janelia.saalfeldlab.n5; import java.io.DataInput; +import java.io.DataInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -44,18 +46,6 @@ public ByteBuffer toByteBuffer() { return buffer; } - public void readData(final ByteBuffer buffer) { - - buffer.asFloatBuffer().get(data); - } - - @Override - public void readData(final DataInput inputStream) throws IOException { - - for (int i = 0; i < data.length; i++) - data[i] = inputStream.readFloat(); - } - @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Float.BYTES * data.length); @@ -68,6 +58,13 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { ByteBuffer.wrap(serialized).order(byteOrder).asFloatBuffer().get(data); } + @Override + public void readData(final InputStream inputStream) throws IOException { + final byte[] bytes = DataType.FLOAT32.createSerializeArray(data.length); + new DataInputStream(inputStream).readFully(bytes); + deserialize(bytes); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index a9db9e42a..ee490e848 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -26,8 +26,10 @@ package org.janelia.saalfeldlab.n5; import java.io.DataInput; +import java.io.DataInputStream; import java.io.DataOutput; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -50,20 +52,6 @@ public void readData(final ByteBuffer buffer) { buffer.asIntBuffer().get(data); } - @Override - public void readData(final DataInput input) throws IOException { - - for (int i = 0; i < data.length; i++) - data[i] = input.readInt(); - } - - @Override - public void writeData(final DataOutput output) throws IOException { - - for (int i = 0; i < data.length; i++) - output.writeInt(data[i]); - } - @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES * data.length); @@ -76,6 +64,13 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { ByteBuffer.wrap(serialized).order(byteOrder).asIntBuffer().get(data); } + @Override + public void readData(final InputStream inputStream) throws IOException { + final byte[] bytes = DataType.INT32.createSerializeArray(data.length); + new DataInputStream(inputStream).readFully(bytes); + deserialize(bytes); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index fc041cf0b..93aa55a0b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -26,8 +26,10 @@ package org.janelia.saalfeldlab.n5; import java.io.DataInput; +import java.io.DataInputStream; import java.io.DataOutput; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -45,25 +47,6 @@ public ByteBuffer toByteBuffer() { return buffer; } - public void readData(final ByteBuffer buffer) { - - buffer.asLongBuffer().get(data); - } - - @Override - public void readData(final DataInput inputStream) throws IOException { - - for (int i = 0; i < data.length; i++) - data[i] = inputStream.readLong(); - } - - @Override - public void writeData(final DataOutput output) throws IOException { - - for (int i = 0; i < data.length; i++) - output.writeLong(data[i]); - } - @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * data.length); @@ -76,6 +59,13 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { ByteBuffer.wrap(serialized).order(byteOrder).asLongBuffer().get(data); } + @Override + public void readData(final InputStream inputStream) throws IOException { + final byte[] bytes = DataType.INT64.createSerializeArray(data.length); + new DataInputStream(inputStream).readFully(bytes); + deserialize(bytes); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 671417e09..7a7dabb78 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -25,8 +25,9 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; +import java.io.DataInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -37,25 +38,14 @@ public ShortArrayDataBlock(final int[] size, final long[] gridPosition, final sh super(size, gridPosition, data); } + @Deprecated + @Override public ByteBuffer toByteBuffer() { - final ByteBuffer buffer = ByteBuffer.allocate(data.length * 2); buffer.asShortBuffer().put(data); return buffer; } - public void readData(final ByteBuffer buffer) { - - buffer.asShortBuffer().get(data); - } - - @Override - public void readData(final DataInput dataInput) throws IOException { - - for (int i = 0; i < data.length; i++) - data[i] = dataInput.readShort(); - } - @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Short.BYTES * data.length); @@ -68,6 +58,13 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { ByteBuffer.wrap(serialized).order(byteOrder).asShortBuffer().get(data); } + @Override + public void readData(final InputStream inputStream) throws IOException { + final byte[] bytes = DataType.INT16.createSerializeArray(data.length); + new DataInputStream(inputStream).readFully(bytes); + deserialize(bytes); + } + @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 3e9326201..830b6aab0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -25,6 +25,9 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.Charset; @@ -76,7 +79,7 @@ protected String[] _deserialize(byte[] rawBytes) { @Override public int getNumElements() { if (serializedData == null) - serializedData = serialize(actualData); + serializedData = serialize(); return serializedData.length; } @@ -87,10 +90,6 @@ public String[] getData() { return actualData; } - - - - @Override public byte[] serialize(final ByteOrder byteOrder) { final String flattenedArray = String.join(NULLCHAR, actualData) + NULLCHAR; @@ -102,4 +101,10 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { final String rawChars = new String(serialized, ENCODING); actualData = rawChars.split(NULLCHAR); } + + @Override + public void readData(final InputStream inputStream) throws IOException { + new DataInputStream(inputStream).readFully(serializedData); + deserialize(serializedData); + } } From 4fca725c2a81ec04a51e0778663e8e3e1f028d62 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 26 Jan 2025 19:52:16 -0500 Subject: [PATCH 139/423] WIP DataBlock.writeData(OutputStream) --- .../janelia/saalfeldlab/n5/BlockReader.java | 1 + .../janelia/saalfeldlab/n5/BlockWriter.java | 1 + .../saalfeldlab/n5/ByteArrayDataBlock.java | 6 +--- .../janelia/saalfeldlab/n5/BytesCodec.java | 26 ++++++++++++++ .../janelia/saalfeldlab/n5/Compression.java | 4 ++- .../org/janelia/saalfeldlab/n5/DataBlock.java | 11 +++--- .../saalfeldlab/n5/DefaultBlockReader.java | 5 ++- .../saalfeldlab/n5/DefaultBlockWriter.java | 36 ++++--------------- .../saalfeldlab/n5/DoubleArrayDataBlock.java | 8 ----- .../saalfeldlab/n5/FloatArrayDataBlock.java | 7 ---- .../saalfeldlab/n5/IntArrayDataBlock.java | 11 +----- .../saalfeldlab/n5/Java9StreamMethods.java | 1 + .../saalfeldlab/n5/LongArrayDataBlock.java | 7 ---- .../saalfeldlab/n5/RawCompression.java | 2 +- .../saalfeldlab/n5/ShortArrayDataBlock.java | 8 ----- .../saalfeldlab/n5/StringDataBlock.java | 36 +++++++++++-------- 16 files changed, 71 insertions(+), 99 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java index f1ff749e9..8d0103b74 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java @@ -34,6 +34,7 @@ * * @author Stephan Saalfeld */ +@Deprecated public interface BlockReader { /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java index 0fc7455ce..884200d58 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java @@ -34,6 +34,7 @@ * * @author Stephan Saalfeld */ +@Deprecated public interface BlockWriter { /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index 6956cec99..78e87c544 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -28,6 +28,7 @@ import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -38,11 +39,6 @@ public ByteArrayDataBlock(final int[] size, final long[] gridPosition, final byt super(size, gridPosition, data); } - public ByteBuffer toByteBuffer() { - - return ByteBuffer.wrap(getData()); - } - @Override public byte[] serialize(final ByteOrder byteOrder) { return data; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java new file mode 100644 index 000000000..9057c8e9a --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java @@ -0,0 +1,26 @@ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public interface BytesCodec { + + /** + * Decode an {@link InputStream}. + * + * @param in + * input stream + * @return the decoded input stream + */ + InputStream decode(InputStream in) throws IOException; + + /** + * Encode an {@link OutputStream}. + * + * @param out + * the output stream + * @return the encoded output stream + */ + OutputStream encode(OutputStream out) throws IOException; +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index 9f7667ba9..30179f286 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -43,7 +43,7 @@ * * @author Stephan Saalfeld */ -public interface Compression extends Serializable { +public interface Compression extends BytesCodec, Serializable { /** * Annotation for runtime discovery of compression schemes. @@ -91,6 +91,7 @@ default String getType() { * input stream * @return the decoded input stream */ + @Override default InputStream decode(InputStream in) throws IOException { return getInputStream(in); } @@ -102,6 +103,7 @@ default InputStream decode(InputStream in) throws IOException { * the output stream * @return the encoded output stream */ + @Override default OutputStream encode(OutputStream out) throws IOException { return getOutputStream(out); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index e3d04ec40..592fedac7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -25,10 +25,9 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; -import java.io.DataOutput; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -83,11 +82,13 @@ default void deserialize(byte[] serialized) { void deserialize(ByteOrder byteOrder, byte[] serialized); + // TODO should have ByteOrder argument void readData(final InputStream inputStream) throws IOException; - @Deprecated ByteBuffer toByteBuffer(); - -// void writeData(final DataOutput output) throws IOException; + // TODO should have ByteOrder argument + default void writeData(final OutputStream outputStream) throws IOException { + outputStream.write(serialize()); + }; /** * Returns the number of elements in this {@link DataBlock}. This number is diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 1873e4939..e36f3a324 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -32,7 +32,7 @@ import java.nio.ByteBuffer; /** - * Default implementation of {@link BlockReader}. + * Default implementation of block reading (N5 format). * * @author Stephan Saalfeld * @author Igor Pisarev @@ -86,8 +86,7 @@ static DataBlock readBlock( // dataBlock.deserialize(data); try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { - final DataInputStream dis2 = new DataInputStream(inflater); - dataBlock.readData(dis2); + dataBlock.readData(inflater); } return dataBlock; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index 60b934c30..9b10fcbff 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -31,28 +31,12 @@ import java.nio.ByteBuffer; /** - * Default implementation of {@link BlockWriter}. + * Default implementation of block writing (N5 format). * * @author Stephan Saalfeld * @author Igor Pisarev */ -public interface DefaultBlockWriter extends BlockWriter { - - @Deprecated - public OutputStream getOutputStream(final OutputStream out) throws IOException; - - @Deprecated - @Override - public default void write( - final DataBlock dataBlock, - final OutputStream out) throws IOException { - - final ByteBuffer buffer = dataBlock.toByteBuffer(); - try (final OutputStream deflater = getOutputStream(out)) { - deflater.write(buffer.array()); - deflater.flush(); - } - } +public interface DefaultBlockWriter { /** * Writes a {@link DataBlock} into an {@link OutputStream}. @@ -67,7 +51,7 @@ public default void write( * @throws IOException * the exception */ - public static void writeBlock( + static void writeBlock( final OutputStream out, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { @@ -94,16 +78,8 @@ else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSiz dos.flush(); - // variant 1 -// final byte[] data = dataBlock.serialize(); -// datasetAttributes.getCompression().getOutputStream(out).write(data); - - // variant 2 - final byte[] data = datasetAttributes.getCompression().encode(dataBlock.serialize()); - out.write(data); - - // old -// final BlockWriter writer = datasetAttributes.getCompression().getWriter(); -// writer.write(dataBlock, out); + try (final OutputStream deflater = datasetAttributes.getCompression().getOutputStream(out)) { + dataBlock.writeData(deflater); + } } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index bfa159775..75573085e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -38,14 +38,6 @@ public DoubleArrayDataBlock(final int[] size, final long[] gridPosition, final d super(size, gridPosition, data); } - @Deprecated - @Override - public ByteBuffer toByteBuffer() { - final ByteBuffer buffer = ByteBuffer.allocate(data.length * 8); - buffer.asDoubleBuffer().put(data); - return buffer; - } - @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES * data.length); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index 1653c8315..626611610 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -39,13 +39,6 @@ public FloatArrayDataBlock(final int[] size, final long[] gridPosition, final fl super(size, gridPosition, data); } - public ByteBuffer toByteBuffer() { - - final ByteBuffer buffer = ByteBuffer.allocate(data.length * 4); - buffer.asFloatBuffer().put(data); - return buffer; - } - @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Float.BYTES * data.length); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index ee490e848..4e5fe100c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -25,9 +25,7 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; import java.io.DataInputStream; -import java.io.DataOutput; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; @@ -40,13 +38,6 @@ public IntArrayDataBlock(final int[] size, final long[] gridPosition, final int[ super(size, gridPosition, data); } - public ByteBuffer toByteBuffer() { - - final ByteBuffer buffer = ByteBuffer.allocate(data.length * 4); - buffer.asIntBuffer().put(data); - return buffer; - } - public void readData(final ByteBuffer buffer) { buffer.asIntBuffer().get(data); @@ -65,7 +56,7 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { } @Override - public void readData(final InputStream inputStream) throws IOException { + public void readData(final InputStream inputStream) throws IOException { final byte[] bytes = DataType.INT32.createSerializeArray(data.length); new DataInputStream(inputStream).readFully(bytes); deserialize(bytes); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java b/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java index a0daf978f..bc118222a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java @@ -13,6 +13,7 @@ public final class Java9StreamMethods { private Java9StreamMethods() {} + // TODO replace usage with IOUtils.toByteArray(in) and remove this class public static byte[] readAllBytes(final InputStream in) throws IOException { // return in.readAllBytes(); return readNBytes(in, Integer.MAX_VALUE); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index 93aa55a0b..a9cf46a0b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -40,13 +40,6 @@ public LongArrayDataBlock(final int[] size, final long[] gridPosition, final lon super(size, gridPosition, data); } - public ByteBuffer toByteBuffer() { - - final ByteBuffer buffer = ByteBuffer.allocate(data.length * 8); - buffer.asLongBuffer().put(data); - return buffer; - } - @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * data.length); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index 8e06eecf9..85ad908c5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -32,7 +32,7 @@ import org.janelia.saalfeldlab.n5.Compression.CompressionType; @CompressionType("raw") -public class RawCompression implements DefaultBlockReader, DefaultBlockWriter, Compression { +public class RawCompression implements DefaultBlockWriter, Compression { private static final long serialVersionUID = 7526445806847086477L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 7a7dabb78..b86e83e91 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -38,14 +38,6 @@ public ShortArrayDataBlock(final int[] size, final long[] gridPosition, final sh super(size, gridPosition, data); } - @Deprecated - @Override - public ByteBuffer toByteBuffer() { - final ByteBuffer buffer = ByteBuffer.allocate(data.length * 2); - buffer.asShortBuffer().put(data); - return buffer; - } - @Override public byte[] serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Short.BYTES * data.length); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 830b6aab0..795fd33c3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -28,6 +28,7 @@ import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.Charset; @@ -50,21 +51,21 @@ public StringDataBlock(final int[] size, final long[] gridPosition, final byte[] serializedData = data; } - public ByteBuffer toByteBuffer() { - if (serializedData == null) - serializedData = serialize(actualData); - return ByteBuffer.wrap(serializedData); - } +// public ByteBuffer toByteBuffer() { +// if (serializedData == null) +// serializedData = serialize(actualData); +// return ByteBuffer.wrap(serializedData); +// } - public void readData(final ByteBuffer buffer) { - - if (buffer.hasArray()) { - if (buffer.array() != serializedData) - buffer.get(serializedData); - actualData = _deserialize(buffer.array()); - } else - actualData = ENCODING.decode(buffer).toString().split(NULLCHAR); - } +// public void readData(final ByteBuffer buffer) { +// +// if (buffer.hasArray()) { +// if (buffer.array() != serializedData) +// buffer.get(serializedData); +// actualData = _deserialize(buffer.array()); +// } else +// actualData = ENCODING.decode(buffer).toString().split(NULLCHAR); +// } protected byte[] serialize(String[] strings) { final String flattenedArray = String.join(NULLCHAR, strings) + NULLCHAR; @@ -107,4 +108,11 @@ public void readData(final InputStream inputStream) throws IOException { new DataInputStream(inputStream).readFully(serializedData); deserialize(serializedData); } + + @Override + public void writeData(final OutputStream outputStream) throws IOException { + if (serializedData == null) + serializedData = serialize(); + outputStream.write(serializedData); + } } From 701d4218cdeb1d30f0474466dc4460d3a191ebbf Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 27 Jan 2025 14:55:50 -0500 Subject: [PATCH 140/423] WIP use Splittable.ReadData in DefaultBlockReader --- .../saalfeldlab/n5/ByteArrayDataBlock.java | 8 +- .../org/janelia/saalfeldlab/n5/DataBlock.java | 17 +++-- .../org/janelia/saalfeldlab/n5/DataType.java | 73 +++++++++++-------- .../saalfeldlab/n5/DefaultBlockReader.java | 14 ++-- .../saalfeldlab/n5/DoubleArrayDataBlock.java | 10 --- .../saalfeldlab/n5/FloatArrayDataBlock.java | 11 --- .../saalfeldlab/n5/IntArrayDataBlock.java | 10 --- .../saalfeldlab/n5/LongArrayDataBlock.java | 12 --- .../saalfeldlab/n5/ShortArrayDataBlock.java | 10 --- .../janelia/saalfeldlab/n5/Splittable.java | 34 +++++++++ .../saalfeldlab/n5/StringDataBlock.java | 9 +-- 11 files changed, 100 insertions(+), 108 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index 78e87c544..465630a4f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -27,10 +27,8 @@ import java.io.DataInputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; import java.nio.ByteOrder; +import org.janelia.saalfeldlab.n5.Splittable.ReadData; public class ByteArrayDataBlock extends AbstractDataBlock { @@ -50,8 +48,8 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { } @Override - public void readData(final InputStream inputStream) throws IOException { - new DataInputStream(inputStream).readFully(data); + public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { + new DataInputStream(readData.inputStream()).readFully(data); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 592fedac7..0709bb815 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -30,6 +30,7 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import org.janelia.saalfeldlab.n5.Splittable.ReadData; /** * Interface for data blocks. A data block has data, a position on the block @@ -70,20 +71,24 @@ public interface DataBlock { */ T getData(); + // TODO: Remove this later? default byte[] serialize() { return serialize(ByteOrder.BIG_ENDIAN); } byte[] serialize(ByteOrder byteOrder); - default void deserialize(byte[] serialized) { - deserialize(ByteOrder.BIG_ENDIAN, serialized); - } - void deserialize(ByteOrder byteOrder, byte[] serialized); - // TODO should have ByteOrder argument - void readData(final InputStream inputStream) throws IOException; + // TODO: Remove this? readData() is not called in many places, so maybe it + // is not worth cluttering the interface with this overload? + default void readData(ReadData readData) throws IOException { + readData(ByteOrder.BIG_ENDIAN, readData); + } + + default void readData(ByteOrder byteOrder, ReadData readData) throws IOException { + deserialize(byteOrder, readData.allBytes()); + } // TODO should have ByteOrder argument default void writeData(final OutputStream outputStream) throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 356c816de..28238dd43 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -34,7 +34,6 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; -import java.util.function.IntFunction; /** * Enumerates available data types. @@ -45,100 +44,112 @@ public enum DataType { UINT8( "uint8", + Byte.BYTES, (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, - new byte[numElements]), - numElements -> new byte[Byte.BYTES * numElements]), + new byte[numElements]) + ), UINT16( "uint16", + Short.BYTES, (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( blockSize, gridPosition, - new short[numElements]), - numElements -> new byte[Short.BYTES * numElements]), + new short[numElements]) + ), UINT32( "uint32", + Integer.BYTES, (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( blockSize, gridPosition, - new int[numElements]), - numElements -> new byte[Integer.BYTES * numElements]), + new int[numElements]) + ), UINT64( "uint64", + Long.BYTES, (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( blockSize, gridPosition, - new long[numElements]), - numElements -> new byte[Long.BYTES * numElements]), + new long[numElements]) + ), INT8( "int8", + Byte.BYTES, (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, - new byte[numElements]), - numElements -> new byte[Byte.BYTES * numElements]), + new byte[numElements]) + ), INT16( "int16", + Short.BYTES, (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( blockSize, gridPosition, - new short[numElements]), - numElements -> new byte[Short.BYTES * numElements]), + new short[numElements]) + ), INT32( "int32", + Integer.BYTES, (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( blockSize, gridPosition, - new int[numElements]), - numElements -> new byte[Integer.BYTES * numElements]), + new int[numElements]) + ), INT64( "int64", + Long.BYTES, (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( blockSize, gridPosition, - new long[numElements]), - numElements -> new byte[Long.BYTES * numElements]), + new long[numElements]) + ), FLOAT32( "float32", + Float.BYTES, (blockSize, gridPosition, numElements) -> new FloatArrayDataBlock( blockSize, gridPosition, - new float[numElements]), - numElements -> new byte[Float.BYTES * numElements]), + new float[numElements]) + ), FLOAT64( "float64", + Double.BYTES, (blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock( blockSize, gridPosition, - new double[numElements]), - numElements -> new byte[Double.BYTES * numElements]), + new double[numElements]) + ), STRING( "string", + 1, (blockSize, gridPosition, numElements) -> new StringDataBlock( blockSize, gridPosition, - new byte[numElements]), - numElements -> new byte[numElements]), + new byte[numElements]) + ), OBJECT( "object", + 1, (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, - new byte[numElements]), - numElements -> new byte[numElements]); + new byte[numElements]) + ); private final String label; private final DataBlockFactory dataBlockFactory; - private final IntFunction serializeArrayFactory; + private final int bytesPerElement; - DataType(final String label, final DataBlockFactory dataBlockFactory, final IntFunction serializeArrayFactory) { + DataType(final String label, final int bytesPerElement, final DataBlockFactory dataBlockFactory) { this.label = label; this.dataBlockFactory = dataBlockFactory; - this.serializeArrayFactory = serializeArrayFactory; + this.bytesPerElement = bytesPerElement; } @Override @@ -189,10 +200,10 @@ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosi /** * TODO: javadoc + * explain that STRING and OBJECT are a bit weird ... */ - public byte[] createSerializeArray(final int numElements) { - - return serializeArrayFactory.apply(numElements); + public int bytesPerElement() { + return bytesPerElement; } private interface DataBlockFactory { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index e36f3a324..027283abb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -30,6 +30,9 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import org.janelia.saalfeldlab.n5.Splittable.InputStreamReadData; +import org.janelia.saalfeldlab.n5.Splittable.ReadData; /** * Default implementation of block reading (N5 format). @@ -78,15 +81,10 @@ static DataBlock readBlock( dataBlock = dataType.createDataBlock(null, gridPosition, numElements); } -// final byte[] data = dataType.createSerializeArray(numElements); -// try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { -// final DataInputStream dis2 = new DataInputStream(inflater); -// dis2.readFully(data); -// } -// dataBlock.deserialize(data); - try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { - dataBlock.readData(inflater); + final int numBytes = numElements * dataType.bytesPerElement(); + ReadData data = new InputStreamReadData(inflater, numBytes); + dataBlock.readData(ByteOrder.BIG_ENDIAN, data); } return dataBlock; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 75573085e..27a88e067 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -25,9 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -50,13 +47,6 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { ByteBuffer.wrap(serialized).order(byteOrder).asDoubleBuffer().get(data); } - @Override - public void readData(final InputStream inputStream) throws IOException { - final byte[] bytes = DataType.FLOAT64.createSerializeArray(getNumElements()); - new DataInputStream(inputStream).readFully(bytes); - deserialize(bytes); - } - @Override public int getNumElements() { return data.length; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index 626611610..9e93f6c1a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -25,10 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -51,13 +47,6 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { ByteBuffer.wrap(serialized).order(byteOrder).asFloatBuffer().get(data); } - @Override - public void readData(final InputStream inputStream) throws IOException { - final byte[] bytes = DataType.FLOAT32.createSerializeArray(data.length); - new DataInputStream(inputStream).readFully(bytes); - deserialize(bytes); - } - @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 4e5fe100c..4aefb2a8f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -25,9 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -55,13 +52,6 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { ByteBuffer.wrap(serialized).order(byteOrder).asIntBuffer().get(data); } - @Override - public void readData(final InputStream inputStream) throws IOException { - final byte[] bytes = DataType.INT32.createSerializeArray(data.length); - new DataInputStream(inputStream).readFully(bytes); - deserialize(bytes); - } - @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index a9cf46a0b..7e15b64fd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -25,11 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.DataOutput; -import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -52,13 +47,6 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { ByteBuffer.wrap(serialized).order(byteOrder).asLongBuffer().get(data); } - @Override - public void readData(final InputStream inputStream) throws IOException { - final byte[] bytes = DataType.INT64.createSerializeArray(data.length); - new DataInputStream(inputStream).readFully(bytes); - deserialize(bytes); - } - @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index b86e83e91..8ea50a7eb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -25,9 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -50,13 +47,6 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { ByteBuffer.wrap(serialized).order(byteOrder).asShortBuffer().get(data); } - @Override - public void readData(final InputStream inputStream) throws IOException { - final byte[] bytes = DataType.INT16.createSerializeArray(data.length); - new DataInputStream(inputStream).readFully(bytes); - deserialize(bytes); - } - @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java b/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java index e91b674b2..ff7833578 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java @@ -5,6 +5,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; public class Splittable { @@ -41,6 +42,25 @@ default long length() throws IOException { */ InputStream inputStream() throws IOException, IllegalStateException; + /** + * Return the contained data as a {@code byte[]} array. + *

      + * This may use {@link #inputStream()} to read the data. + * Because repeatedly calling {@link #inputStream()} may not work, + *

        + *
      1. this method may fail with {@code IllegalStateException} if {@code inputStream()} was already called
        2. , + *
      2. subsequent {@code inputStream()} calls may fail with {@code IllegalStateException}
        3. , + *
      + * + * @return all contained data as a byte[] array + * + * @throws IOException + * if any I/O error occurs + * @throws IllegalStateException + * if {@link #inputStream()} was already called once and cannot be called again. + */ + byte[] allBytes() throws IOException, IllegalStateException; + /** * If this {@code ReadData} is a {@code SplittableReadData}, just returns {@code this}. *

      @@ -80,6 +100,15 @@ public InputStream inputStream() throws IOException { return new ByteArrayInputStream(data, offset, length); } + @Override + public byte[] allBytes() { + if (offset == 0 && data.length == length) { + return data; + } else { + return Arrays.copyOfRange(data, offset, offset + length); + } + } + @Override public SplittableReadData splittable() throws IOException { return this; @@ -116,6 +145,11 @@ public SplittableReadData splittable() throws IOException { } return bytes; } + + @Override + public byte[] allBytes() throws IOException, IllegalStateException { + return splittable().allBytes(); + } } // not thread-safe diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 795fd33c3..241667cb0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -27,12 +27,11 @@ import java.io.DataInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import org.janelia.saalfeldlab.n5.Splittable.ReadData; public class StringDataBlock extends AbstractDataBlock { @@ -104,9 +103,9 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { } @Override - public void readData(final InputStream inputStream) throws IOException { - new DataInputStream(inputStream).readFully(serializedData); - deserialize(serializedData); + public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { + new DataInputStream(readData.inputStream()).readFully(serializedData); + deserialize(byteOrder, serializedData); } @Override From 8cffeed489cd75c8d1bd5a6f59eb1b5d4c2df07d Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 27 Jan 2025 15:21:48 -0500 Subject: [PATCH 141/423] WIP revise StringDataBlock (doesn't work yet) --- .../org/janelia/saalfeldlab/n5/DataType.java | 2 +- .../saalfeldlab/n5/StringDataBlock.java | 78 ++++--------------- 2 files changed, 15 insertions(+), 65 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 28238dd43..2926de754 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -128,7 +128,7 @@ public enum DataType { (blockSize, gridPosition, numElements) -> new StringDataBlock( blockSize, gridPosition, - new byte[numElements]) + new String[numElements]) ), OBJECT( "object", diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 241667cb0..66c42d3cd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -25,93 +25,43 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteOrder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import org.janelia.saalfeldlab.n5.Splittable.ReadData; public class StringDataBlock extends AbstractDataBlock { - protected static final Charset ENCODING = StandardCharsets.UTF_8; - protected static final String NULLCHAR = "\0"; - protected byte[] serializedData = null; - protected String[] actualData = null; + protected static final Charset ENCODING = StandardCharsets.UTF_8; + protected static final String NULLCHAR = "\0"; + protected byte[] serializedData = null; - public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { - super(size, gridPosition, new String[0]); - actualData = data; - } - - public StringDataBlock(final int[] size, final long[] gridPosition, final byte[] data) { - super(size, gridPosition, new String[0]); - serializedData = data; - } - -// public ByteBuffer toByteBuffer() { -// if (serializedData == null) -// serializedData = serialize(actualData); -// return ByteBuffer.wrap(serializedData); -// } - -// public void readData(final ByteBuffer buffer) { -// -// if (buffer.hasArray()) { -// if (buffer.array() != serializedData) -// buffer.get(serializedData); -// actualData = _deserialize(buffer.array()); -// } else -// actualData = ENCODING.decode(buffer).toString().split(NULLCHAR); -// } - - protected byte[] serialize(String[] strings) { - final String flattenedArray = String.join(NULLCHAR, strings) + NULLCHAR; - return flattenedArray.getBytes(ENCODING); - } - - protected String[] _deserialize(byte[] rawBytes) { - final String rawChars = new String(rawBytes, ENCODING); - return rawChars.split(NULLCHAR); - } - - @Override - public int getNumElements() { - if (serializedData == null) - serializedData = serialize(); - return serializedData.length; - } - - @Override - public String[] getData() { - if (actualData == null) - actualData = _deserialize(serializedData); - return actualData; - } + public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { + super(size, gridPosition, data); + } @Override public byte[] serialize(final ByteOrder byteOrder) { - final String flattenedArray = String.join(NULLCHAR, actualData) + NULLCHAR; + final String flattenedArray = String.join(NULLCHAR, data) + NULLCHAR; return flattenedArray.getBytes(ENCODING); } @Override public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { + // TODO: write benchmark, make more efficient final String rawChars = new String(serialized, ENCODING); - actualData = rawChars.split(NULLCHAR); + final String[] actualData = rawChars.split(NULLCHAR); + System.arraycopy(actualData, 0, data, 0, actualData.length); } @Override - public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - new DataInputStream(readData.inputStream()).readFully(serializedData); - deserialize(byteOrder, serializedData); + public void writeData(final OutputStream outputStream) throws IOException { + outputStream.write(serialize()); } @Override - public void writeData(final OutputStream outputStream) throws IOException { - if (serializedData == null) - serializedData = serialize(); - outputStream.write(serializedData); + public int getNumElements() { + return data.length; } } From e47ba4c50d1614b33e9eab39fb6f63875be9a903 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 27 Jan 2025 16:10:03 -0500 Subject: [PATCH 142/423] WIP revise StringDataBlock revert to holding serialized and actual data. otherwise we can't be compatible with existing N5 datasets. --- .../org/janelia/saalfeldlab/n5/DataType.java | 9 ++++-- .../saalfeldlab/n5/DefaultBlockReader.java | 8 +++--- .../saalfeldlab/n5/StringDataBlock.java | 28 ++++++++++++++----- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 2926de754..c3144a51f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -124,11 +124,11 @@ public enum DataType { ), STRING( "string", - 1, + -1, (blockSize, gridPosition, numElements) -> new StringDataBlock( blockSize, gridPosition, - new String[numElements]) + null) ), OBJECT( "object", @@ -201,11 +201,16 @@ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosi /** * TODO: javadoc * explain that STRING and OBJECT are a bit weird ... + * -1 means varlength */ public int bytesPerElement() { return bytesPerElement; } + public boolean isVarLength() { + return bytesPerElement < 0; + } + private interface DataBlockFactory { DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 027283abb..8f6f59d51 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -25,11 +25,9 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.BufferedInputStream; import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import org.janelia.saalfeldlab.n5.Splittable.InputStreamReadData; import org.janelia.saalfeldlab.n5.Splittable.ReadData; @@ -82,8 +80,10 @@ static DataBlock readBlock( } try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { - final int numBytes = numElements * dataType.bytesPerElement(); - ReadData data = new InputStreamReadData(inflater, numBytes); + final int numBytes = dataType.isVarLength() + ? numElements + : (numElements * dataType.bytesPerElement()); + final ReadData data = new InputStreamReadData(inflater, numBytes); dataBlock.readData(ByteOrder.BIG_ENDIAN, data); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 66c42d3cd..a1b243139 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -30,38 +30,52 @@ import java.nio.ByteOrder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.util.Arrays; public class StringDataBlock extends AbstractDataBlock { protected static final Charset ENCODING = StandardCharsets.UTF_8; protected static final String NULLCHAR = "\0"; - protected byte[] serializedData = null; + protected byte[] serializedData; + protected String[] actualData; public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { - super(size, gridPosition, data); + super(size, gridPosition, null); + actualData = data; } @Override public byte[] serialize(final ByteOrder byteOrder) { - final String flattenedArray = String.join(NULLCHAR, data) + NULLCHAR; + final String flattenedArray = String.join(NULLCHAR, actualData) + NULLCHAR; return flattenedArray.getBytes(ENCODING); } @Override public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { // TODO: write benchmark, make more efficient + serializedData = serialized; final String rawChars = new String(serialized, ENCODING); - final String[] actualData = rawChars.split(NULLCHAR); - System.arraycopy(actualData, 0, data, 0, actualData.length); + actualData = rawChars.split(NULLCHAR); } @Override public void writeData(final OutputStream outputStream) throws IOException { - outputStream.write(serialize()); + if (serializedData == null) { + serializedData = serialize(ByteOrder.BIG_ENDIAN); + } + outputStream.write(serializedData); } @Override public int getNumElements() { - return data.length; + if (serializedData == null) { + serializedData = serialize(ByteOrder.BIG_ENDIAN); + } + return serializedData.length; + } + + @Override + public String[] getData() { + return actualData; } } From 8732a0b5a31189f11f47736120fffeb93785e1fe Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 27 Jan 2025 21:10:05 -0500 Subject: [PATCH 143/423] Add ByteOrder to DataBlock.writeData() --- src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java | 7 +++---- .../org/janelia/saalfeldlab/n5/DefaultBlockWriter.java | 3 ++- .../java/org/janelia/saalfeldlab/n5/StringDataBlock.java | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 0709bb815..91acdd776 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -86,13 +86,12 @@ default void readData(ReadData readData) throws IOException { readData(ByteOrder.BIG_ENDIAN, readData); } - default void readData(ByteOrder byteOrder, ReadData readData) throws IOException { + default void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { deserialize(byteOrder, readData.allBytes()); } - // TODO should have ByteOrder argument - default void writeData(final OutputStream outputStream) throws IOException { - outputStream.write(serialize()); + default void writeData(final ByteOrder byteOrder, final OutputStream outputStream) throws IOException { + outputStream.write(serialize(byteOrder)); }; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index 9b10fcbff..fa4912f87 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.nio.ByteOrder; /** * Default implementation of block writing (N5 format). @@ -79,7 +80,7 @@ else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSiz dos.flush(); try (final OutputStream deflater = datasetAttributes.getCompression().getOutputStream(out)) { - dataBlock.writeData(deflater); + dataBlock.writeData(ByteOrder.BIG_ENDIAN, deflater); } } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index a1b243139..7878a920e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -59,9 +59,9 @@ public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { } @Override - public void writeData(final OutputStream outputStream) throws IOException { + public void writeData(final ByteOrder byteOrder, final OutputStream outputStream) throws IOException { if (serializedData == null) { - serializedData = serialize(ByteOrder.BIG_ENDIAN); + serializedData = serialize(byteOrder); } outputStream.write(serializedData); } From 190a8ce38805dff0a084395be693355afd35a28f Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 27 Jan 2025 21:19:11 -0500 Subject: [PATCH 144/423] Clean up --- .../java/org/janelia/saalfeldlab/n5/DataBlock.java | 11 ----------- .../java/org/janelia/saalfeldlab/n5/N5Reader.java | 3 ++- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 91acdd776..ed9bf5ed2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -71,21 +71,10 @@ public interface DataBlock { */ T getData(); - // TODO: Remove this later? - default byte[] serialize() { - return serialize(ByteOrder.BIG_ENDIAN); - } - byte[] serialize(ByteOrder byteOrder); void deserialize(ByteOrder byteOrder, byte[] serialized); - // TODO: Remove this? readData() is not called in many places, so maybe it - // is not worth cluttering the interface with this overload? - default void readData(ReadData readData) throws IOException { - readData(ByteOrder.BIG_ENDIAN, readData); - } - default void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { deserialize(byteOrder, readData.allBytes()); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index a7102d57b..527015c62 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -32,6 +32,7 @@ import java.io.UncheckedIOException; import java.lang.reflect.Type; import java.net.URI; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -322,7 +323,7 @@ default T readSerializedBlock( if (block == null) return null; - final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.serialize()); + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.serialize(ByteOrder.BIG_ENDIAN)); try (ObjectInputStream in = new ObjectInputStream(byteArrayInputStream)) { return (T)in.readObject(); } catch (final IOException | UncheckedIOException e) { From 3df2b0b4f6c650a82da3aba78898abb1452752ec Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 27 Jan 2025 21:23:59 -0500 Subject: [PATCH 145/423] Clean up --- .../janelia/saalfeldlab/n5/Compression.java | 18 +--- .../saalfeldlab/n5/DefaultBlockWriter.java | 2 +- .../saalfeldlab/n5/Java9StreamMethods.java | 86 ------------------- .../saalfeldlab/n5/RawCompression.java | 10 --- .../janelia/saalfeldlab/n5/Splittable.java | 3 +- .../saalfeldlab/n5/StringDataBlock.java | 1 - 6 files changed, 5 insertions(+), 115 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index 30179f286..1236e9c86 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -92,7 +92,7 @@ default String getType() { * @return the decoded input stream */ @Override - default InputStream decode(InputStream in) throws IOException { + default InputStream decode(final InputStream in) throws IOException { return getInputStream(in); } @@ -104,21 +104,7 @@ default InputStream decode(InputStream in) throws IOException { * @return the encoded output stream */ @Override - default OutputStream encode(OutputStream out) throws IOException { + default OutputStream encode(final OutputStream out) throws IOException { return getOutputStream(out); } - - // TODO probably remove? - default byte[] encode(byte[] data) throws IOException { - final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - final OutputStream encodedStream = encode(byteStream); - encodedStream.write(data); - encodedStream.close(); - return byteStream.toByteArray(); - } - - // TODO probably remove? - default byte[] decode(byte[] data) throws IOException { - return Java9StreamMethods.readAllBytes(decode(new ByteArrayInputStream(data))); - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index fa4912f87..b2852c3c5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -79,7 +79,7 @@ else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSiz dos.flush(); - try (final OutputStream deflater = datasetAttributes.getCompression().getOutputStream(out)) { + try (final OutputStream deflater = datasetAttributes.getCompression().encode(out)) { dataBlock.writeData(ByteOrder.BIG_ENDIAN, deflater); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java b/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java deleted file mode 100644 index bc118222a..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/Java9StreamMethods.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * Methods from InputStream interface in Java9+ (copied and modified to static methods). - */ -public final class Java9StreamMethods { - - private Java9StreamMethods() {} - - // TODO replace usage with IOUtils.toByteArray(in) and remove this class - public static byte[] readAllBytes(final InputStream in) throws IOException { -// return in.readAllBytes(); - return readNBytes(in, Integer.MAX_VALUE); - } - - private static final int DEFAULT_BUFFER_SIZE = 8192; - - private static final int MAX_BUFFER_SIZE = Integer.MAX_VALUE - 8; - - private static byte[] readNBytes(final InputStream in, int len) throws IOException { - if (len < 0) { - throw new IllegalArgumentException("len < 0"); - } - - List bufs = null; - byte[] result = null; - int total = 0; - int remaining = len; - int n; - do { - byte[] buf = new byte[Math.min(remaining, DEFAULT_BUFFER_SIZE)]; - int nread = 0; - - // read to EOF which may read more or less than buffer size - while ((n = in.read(buf, nread, - Math.min(buf.length - nread, remaining))) > 0) { - nread += n; - remaining -= n; - } - - if (nread > 0) { - if (MAX_BUFFER_SIZE - total < nread) { - throw new OutOfMemoryError("Required array size too large"); - } - total += nread; - if (result == null) { - result = buf; - } else { - if (bufs == null) { - bufs = new ArrayList<>(); - bufs.add(result); - } - bufs.add(buf); - } - } - // if the last call to read returned -1 or the number of bytes - // requested have been read then break - } while (n >= 0 && remaining > 0); - - if (bufs == null) { - if (result == null) { - return new byte[0]; - } - return result.length == total ? - result : Arrays.copyOf(result, total); - } - - result = new byte[total]; - int offset = 0; - remaining = total; - for (byte[] b : bufs) { - int count = Math.min(b.length, remaining); - System.arraycopy(b, 0, result, offset, count); - offset += count; - remaining -= count; - } - - return result; - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index 85ad908c5..998bf2655 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -48,16 +48,6 @@ public OutputStream getOutputStream(final OutputStream out) throws IOException { return out; } - @Override - public byte[] encode(final byte[] data) throws IOException { - return data; - } - - @Override - public byte[] decode(final byte[] data) throws IOException { - return data; - } - @Override public boolean equals(final Object other) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java b/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java index ff7833578..55c5c407a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.Arrays; +import org.apache.commons.io.IOUtils; public class Splittable { @@ -139,7 +140,7 @@ public SplittableReadData splittable() throws IOException { data = new byte[length]; new DataInputStream(inputStream()).readFully(data); } else { - data = Java9StreamMethods.readAllBytes(inputStream()); + data = IOUtils.toByteArray(inputStream()); } bytes = new ByteArraySplittableReadData(data, 0, data.length); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 7878a920e..25f646cb9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -52,7 +52,6 @@ public byte[] serialize(final ByteOrder byteOrder) { @Override public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { - // TODO: write benchmark, make more efficient serializedData = serialized; final String rawChars = new String(serialized, ENCODING); actualData = rawChars.split(NULLCHAR); From 705ffb329e1644198948bfde75d95d1ae070d5ad Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 29 Jan 2025 13:58:09 -0500 Subject: [PATCH 146/423] fix javadoc error --- src/main/java/org/janelia/saalfeldlab/n5/Splittable.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java b/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java index 55c5c407a..6e0e49f8d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java @@ -49,8 +49,8 @@ default long length() throws IOException { * This may use {@link #inputStream()} to read the data. * Because repeatedly calling {@link #inputStream()} may not work, *

        - *
      1. this method may fail with {@code IllegalStateException} if {@code inputStream()} was already called
        2. , - *
      2. subsequent {@code inputStream()} calls may fail with {@code IllegalStateException}
        3. , + *
      3. this method may fail with {@code IllegalStateException} if {@code inputStream()} was already called
        4. + *
      4. subsequent {@code inputStream()} calls may fail with {@code IllegalStateException}
        5. *
      * * @return all contained data as a byte[] array From 5c43f5bc1897729f1a74fbb327fb230beb112457 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 30 Jan 2025 08:44:01 +0100 Subject: [PATCH 147/423] switch from byte[] to ByteBuffer for DataBlock.de/serialize() --- .../saalfeldlab/n5/ByteArrayDataBlock.java | 9 +++++---- .../org/janelia/saalfeldlab/n5/DataBlock.java | 8 ++++---- .../saalfeldlab/n5/DoubleArrayDataBlock.java | 8 ++++---- .../saalfeldlab/n5/FloatArrayDataBlock.java | 8 ++++---- .../saalfeldlab/n5/IntArrayDataBlock.java | 13 ++++--------- .../saalfeldlab/n5/LongArrayDataBlock.java | 8 ++++---- .../org/janelia/saalfeldlab/n5/N5Reader.java | 2 +- .../saalfeldlab/n5/ShortArrayDataBlock.java | 8 ++++---- .../janelia/saalfeldlab/n5/StringDataBlock.java | 16 ++++++++-------- 9 files changed, 38 insertions(+), 42 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index 465630a4f..a05f38dbb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -27,6 +27,7 @@ import java.io.DataInputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.ByteOrder; import org.janelia.saalfeldlab.n5.Splittable.ReadData; @@ -38,13 +39,13 @@ public ByteArrayDataBlock(final int[] size, final long[] gridPosition, final byt } @Override - public byte[] serialize(final ByteOrder byteOrder) { - return data; + public ByteBuffer serialize(final ByteOrder byteOrder) { + return ByteBuffer.wrap(data); } @Override - public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { - System.arraycopy(serialized, 0, data, 0, data.length); + public void deserialize(final ByteBuffer serialized) { + System.arraycopy(serialized.array(), 0, data, 0, data.length); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index ed9bf5ed2..e58c720cc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -71,16 +71,16 @@ public interface DataBlock { */ T getData(); - byte[] serialize(ByteOrder byteOrder); + ByteBuffer serialize(ByteOrder byteOrder); - void deserialize(ByteOrder byteOrder, byte[] serialized); + void deserialize(ByteBuffer serialized); default void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - deserialize(byteOrder, readData.allBytes()); + deserialize(ByteBuffer.wrap(readData.allBytes()).order(byteOrder)); } default void writeData(final ByteOrder byteOrder, final OutputStream outputStream) throws IOException { - outputStream.write(serialize(byteOrder)); + outputStream.write(serialize(byteOrder).array()); }; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 27a88e067..5781f7da5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -36,15 +36,15 @@ public DoubleArrayDataBlock(final int[] size, final long[] gridPosition, final d } @Override - public byte[] serialize(final ByteOrder byteOrder) { + public ByteBuffer serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES * data.length); buffer.order(byteOrder).asDoubleBuffer().put(data); - return buffer.array(); + return buffer; } @Override - public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { - ByteBuffer.wrap(serialized).order(byteOrder).asDoubleBuffer().get(data); + public void deserialize(final ByteBuffer serialized) { + serialized.asDoubleBuffer().get(data); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index 9e93f6c1a..7211177d7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -36,15 +36,15 @@ public FloatArrayDataBlock(final int[] size, final long[] gridPosition, final fl } @Override - public byte[] serialize(final ByteOrder byteOrder) { + public ByteBuffer serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Float.BYTES * data.length); buffer.order(byteOrder).asFloatBuffer().put(data); - return buffer.array(); + return buffer; } @Override - public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { - ByteBuffer.wrap(serialized).order(byteOrder).asFloatBuffer().get(data); + public void deserialize(final ByteBuffer serialized) { + serialized.asFloatBuffer().get(data); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 4aefb2a8f..a7175bb1b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -35,21 +35,16 @@ public IntArrayDataBlock(final int[] size, final long[] gridPosition, final int[ super(size, gridPosition, data); } - public void readData(final ByteBuffer buffer) { - - buffer.asIntBuffer().get(data); - } - @Override - public byte[] serialize(final ByteOrder byteOrder) { + public ByteBuffer serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES * data.length); buffer.order(byteOrder).asIntBuffer().put(data); - return buffer.array(); + return buffer; } @Override - public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { - ByteBuffer.wrap(serialized).order(byteOrder).asIntBuffer().get(data); + public void deserialize(final ByteBuffer serialized) { + serialized.asIntBuffer().get(data); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index 7e15b64fd..e974c2bce 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -36,15 +36,15 @@ public LongArrayDataBlock(final int[] size, final long[] gridPosition, final lon } @Override - public byte[] serialize(final ByteOrder byteOrder) { + public ByteBuffer serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * data.length); buffer.order(byteOrder).asLongBuffer().put(data); - return buffer.array(); + return buffer; } @Override - public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { - ByteBuffer.wrap(serialized).order(byteOrder).asLongBuffer().get(data); + public void deserialize(final ByteBuffer serialized) { + serialized.asLongBuffer().get(data); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 527015c62..cbbd401ba 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -323,7 +323,7 @@ default T readSerializedBlock( if (block == null) return null; - final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.serialize(ByteOrder.BIG_ENDIAN)); + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.serialize(ByteOrder.BIG_ENDIAN).array()); try (ObjectInputStream in = new ObjectInputStream(byteArrayInputStream)) { return (T)in.readObject(); } catch (final IOException | UncheckedIOException e) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 8ea50a7eb..08e3f3b8f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -36,15 +36,15 @@ public ShortArrayDataBlock(final int[] size, final long[] gridPosition, final sh } @Override - public byte[] serialize(final ByteOrder byteOrder) { + public ByteBuffer serialize(final ByteOrder byteOrder) { final ByteBuffer buffer = ByteBuffer.allocate(Short.BYTES * data.length); buffer.order(byteOrder).asShortBuffer().put(data); - return buffer.array(); + return buffer; } @Override - public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { - ByteBuffer.wrap(serialized).order(byteOrder).asShortBuffer().get(data); + public void deserialize(final ByteBuffer serialized) { + serialized.asShortBuffer().get(data); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 25f646cb9..2faff64fe 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -27,10 +27,10 @@ import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.Arrays; public class StringDataBlock extends AbstractDataBlock { @@ -45,22 +45,22 @@ public StringDataBlock(final int[] size, final long[] gridPosition, final String } @Override - public byte[] serialize(final ByteOrder byteOrder) { + public ByteBuffer serialize(final ByteOrder byteOrder) { final String flattenedArray = String.join(NULLCHAR, actualData) + NULLCHAR; - return flattenedArray.getBytes(ENCODING); + return ByteBuffer.wrap(flattenedArray.getBytes(ENCODING)); } @Override - public void deserialize(final ByteOrder byteOrder, final byte[] serialized) { - serializedData = serialized; - final String rawChars = new String(serialized, ENCODING); + public void deserialize(final ByteBuffer serialized) { + serializedData = serialized.array(); + final String rawChars = new String(serializedData, ENCODING); actualData = rawChars.split(NULLCHAR); } @Override public void writeData(final ByteOrder byteOrder, final OutputStream outputStream) throws IOException { if (serializedData == null) { - serializedData = serialize(byteOrder); + serializedData = serialize(byteOrder).array(); } outputStream.write(serializedData); } @@ -68,7 +68,7 @@ public void writeData(final ByteOrder byteOrder, final OutputStream outputStream @Override public int getNumElements() { if (serializedData == null) { - serializedData = serialize(ByteOrder.BIG_ENDIAN); + serializedData = serialize(ByteOrder.BIG_ENDIAN).array(); } return serializedData.length; } From fbee9db8091ad4fdf08bdcfa387509feb864234b Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 30 Jan 2025 08:44:33 +0100 Subject: [PATCH 148/423] Add explicit commons-io dependency --- pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pom.xml b/pom.xml index 6bbfb5e33..a1b0632f8 100644 --- a/pom.xml +++ b/pom.xml @@ -202,6 +202,10 @@ org.apache.commons commons-compress + + commons-io + commons-io + From 4c348baae207f7b91ab3357ad52f9801bed3b69b Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 31 Jan 2025 22:38:22 +0100 Subject: [PATCH 149/423] Lean more on ReadData * Move ReadData etc to separate classes * Add ReadData.writeTo(OutputStream) * Add EncodedReadData that wraps a ReadData and an OutputStreamEncoder * Compression (BytesCodec) can encode ReadData. (This might happen immediately or later when the ReadData is written to OutputStream). --- .../saalfeldlab/n5/ByteArrayDataBlock.java | 2 +- .../janelia/saalfeldlab/n5/BytesCodec.java | 19 ++ .../saalfeldlab/n5/Bzip2Compression.java | 10 +- .../org/janelia/saalfeldlab/n5/DataBlock.java | 11 +- .../saalfeldlab/n5/DefaultBlockReader.java | 4 +- .../saalfeldlab/n5/DefaultBlockWriter.java | 8 +- .../saalfeldlab/n5/GzipCompression.java | 20 +- .../saalfeldlab/n5/Lz4Compression.java | 11 + .../saalfeldlab/n5/RawCompression.java | 9 +- .../janelia/saalfeldlab/n5/Splittable.java | 224 ------------------ .../janelia/saalfeldlab/n5/XzCompression.java | 11 +- .../readdata/AbstractInputStreamReadData.java | 32 +++ .../readdata/ByteArraySplittableReadData.java | 57 +++++ .../n5/readdata/EncodedReadData.java | 84 +++++++ .../n5/readdata/InputStreamReadData.java | 36 +++ .../n5/readdata/KeyValueAccessReadData.java | 43 ++++ .../saalfeldlab/n5/readdata/ReadData.java | 96 ++++++++ .../n5/readdata/SplittableReadData.java | 8 + 18 files changed, 444 insertions(+), 241 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/Splittable.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index a05f38dbb..b3465aadc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -29,7 +29,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.Splittable.ReadData; +import org.janelia.saalfeldlab.n5.readdata.ReadData; public class ByteArrayDataBlock extends AbstractDataBlock { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java index 9057c8e9a..42228c50d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import org.janelia.saalfeldlab.n5.readdata.ReadData; public interface BytesCodec { @@ -23,4 +24,22 @@ public interface BytesCodec { * @return the encoded output stream */ OutputStream encode(OutputStream out) throws IOException; + + /** + * TODO javadoc + * + * @param readData + * @return + */ + ReadData encode(ReadData readData) throws IOException; + + /** + * TODO javadoc + * + * @param readData + * @return + */ + default ReadData decode(ReadData readData) { + throw new UnsupportedOperationException(); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index b57bb16d9..600577182 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -28,10 +28,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; - import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.readdata.EncodedReadData; +import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("bzip2") public class Bzip2Compression implements DefaultBlockReader, DefaultBlockWriter, Compression { @@ -72,4 +73,11 @@ public boolean equals(final Object other) { return blockSize == ((Bzip2Compression)other).blockSize; } + @Override + public ReadData encode(final ReadData readData) { + return new EncodedReadData(readData, out -> { + final BZip2CompressorOutputStream deflater = new BZip2CompressorOutputStream(out, blockSize); + return new EncodedReadData.EncodedOutputStream(deflater, deflater::finish); + }); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index e58c720cc..44395e2e1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -26,11 +26,10 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.Splittable.ReadData; +import org.janelia.saalfeldlab.n5.readdata.ByteArraySplittableReadData; +import org.janelia.saalfeldlab.n5.readdata.ReadData; /** * Interface for data blocks. A data block has data, a position on the block @@ -79,9 +78,9 @@ default void readData(final ByteOrder byteOrder, final ReadData readData) throws deserialize(ByteBuffer.wrap(readData.allBytes()).order(byteOrder)); } - default void writeData(final ByteOrder byteOrder, final OutputStream outputStream) throws IOException { - outputStream.write(serialize(byteOrder).array()); - }; + default ReadData writeData(final ByteOrder byteOrder) { + return new ByteArraySplittableReadData(serialize(byteOrder).array()); + } /** * Returns the number of elements in this {@link DataBlock}. This number is diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 8f6f59d51..caa69798a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -29,8 +29,8 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.Splittable.InputStreamReadData; -import org.janelia.saalfeldlab.n5.Splittable.ReadData; +import org.janelia.saalfeldlab.n5.readdata.InputStreamReadData; +import org.janelia.saalfeldlab.n5.readdata.ReadData; /** * Default implementation of block reading (N5 format). diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index b2852c3c5..4d94745fb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -28,7 +28,6 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.nio.ByteOrder; /** @@ -79,8 +78,9 @@ else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSiz dos.flush(); - try (final OutputStream deflater = datasetAttributes.getCompression().encode(out)) { - dataBlock.writeData(ByteOrder.BIG_ENDIAN, deflater); - } + dataBlock.writeData(ByteOrder.BIG_ENDIAN) + .encode(datasetAttributes.getCompression()) + .writeTo(out); + out.flush(); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index f5a850e87..1db23d239 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -32,11 +32,13 @@ import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; - import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; import org.apache.commons.compress.compressors.gzip.GzipParameters; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.readdata.EncodedReadData; +import org.janelia.saalfeldlab.n5.readdata.EncodedReadData.EncodedOutputStream; +import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("gzip") public class GzipCompression implements DefaultBlockReader, DefaultBlockWriter, Compression { @@ -104,4 +106,20 @@ public boolean equals(final Object other) { return useZlib == gz.useZlib && level == gz.level; } } + + @Override + public ReadData encode(final ReadData readData) { + if (useZlib) { + return new EncodedReadData(readData, out -> { + final DeflaterOutputStream deflater = new DeflaterOutputStream(out, new Deflater(level)); + return new EncodedOutputStream(deflater, deflater::finish); + }); + } else { + return new EncodedReadData(readData, out -> { + parameters.setCompressionLevel(level); + final GzipCompressorOutputStream deflater = new GzipCompressorOutputStream(out, parameters); + return new EncodedOutputStream(deflater, deflater::finish); + }); + } + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index 241944088..c5db729c7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -33,6 +33,9 @@ import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; +import org.janelia.saalfeldlab.n5.readdata.EncodedReadData; +import org.janelia.saalfeldlab.n5.readdata.EncodedReadData.EncodedOutputStream; +import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("lz4") public class Lz4Compression implements DefaultBlockReader, DefaultBlockWriter, Compression { @@ -72,4 +75,12 @@ public boolean equals(final Object other) { else return blockSize == ((Lz4Compression)other).blockSize; } + + @Override + public ReadData encode(final ReadData readData) { + return new EncodedReadData(readData, out -> { + final LZ4BlockOutputStream deflater = new LZ4BlockOutputStream(out, blockSize); + return new EncodedOutputStream(deflater, deflater::finish); + }); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index 998bf2655..ac67bafed 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -28,8 +28,10 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; - import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.readdata.EncodedReadData; +import org.janelia.saalfeldlab.n5.readdata.EncodedReadData.EncodedOutputStream; +import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("raw") public class RawCompression implements DefaultBlockWriter, Compression { @@ -56,4 +58,9 @@ public boolean equals(final Object other) { else return true; } + + @Override + public ReadData encode(final ReadData readData) { + return new EncodedReadData(readData, out -> new EncodedOutputStream(out, out::flush)); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java b/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java deleted file mode 100644 index 6e0e49f8d..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/Splittable.java +++ /dev/null @@ -1,224 +0,0 @@ -package org.janelia.saalfeldlab.n5; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import org.apache.commons.io.IOUtils; - -public class Splittable { - - public interface ReadData { - - /** - * Returns number of bytes in this {@link ReadData}, if known. Otherwise - * {@code -1}. - * - * @return number of bytes, if known, or -1 - * - * @throws IOException - * if an I/O error occurs while trying to get the length - */ - default long length() throws IOException { - return -1; - } - - /** - * Open a {@code InputStream} on this data. - *

      - * Repeatedly calling this method may or may not work, depending on how - * the underlying data is stored. For example, if the underlying data is - * stored as a {@code byte[]} array, multiple streams can be opened. If - * the underlying data is just an {@code InputStream} then this will be - * returned on the first call. - * - * @return an InputStream on this data - * - * @throws IOException - * if any I/O error occurs - * @throws IllegalStateException - * if this method was already called once and cannot be called again. - */ - InputStream inputStream() throws IOException, IllegalStateException; - - /** - * Return the contained data as a {@code byte[]} array. - *

      - * This may use {@link #inputStream()} to read the data. - * Because repeatedly calling {@link #inputStream()} may not work, - *

        - *
      1. this method may fail with {@code IllegalStateException} if {@code inputStream()} was already called
        2. - *
      2. subsequent {@code inputStream()} calls may fail with {@code IllegalStateException}
        3. - *
      - * - * @return all contained data as a byte[] array - * - * @throws IOException - * if any I/O error occurs - * @throws IllegalStateException - * if {@link #inputStream()} was already called once and cannot be called again. - */ - byte[] allBytes() throws IOException, IllegalStateException; - - /** - * If this {@code ReadData} is a {@code SplittableReadData}, just returns {@code this}. - *

      - * Otherwise, if the underlying data is an {@code InputStream}, all data is read and - * wrapped as a {@code ByteArraySplittableReadData}. - *

      - * The returned {@code SplittableReadData} has a known {@link #length} - * and multiple {@link #inputStream}s can be opened on it. - */ - SplittableReadData splittable() throws IOException; - } - - public interface SplittableReadData extends ReadData { - - ReadData split(final long offset, final long length) throws IOException; - } - - public static class ByteArraySplittableReadData implements SplittableReadData { - - private final byte[] data; - private final int offset; - private final int length; - - ByteArraySplittableReadData(final byte[] data, final int offset, final int length) { - this.data = data; - this.offset = offset; - this.length = length; - } - - @Override - public long length() { - return length; - } - - @Override - public InputStream inputStream() throws IOException { - return new ByteArrayInputStream(data, offset, length); - } - - @Override - public byte[] allBytes() { - if (offset == 0 && data.length == length) { - return data; - } else { - return Arrays.copyOfRange(data, offset, offset + length); - } - } - - @Override - public SplittableReadData splittable() throws IOException { - return this; - } - - @Override - public SplittableReadData split(final long offset, final long length) throws IOException { - if (offset < 0 || offset > this.length || length < 0) { - throw new IndexOutOfBoundsException(); - } - final int o = this.offset + (int) offset; - final int l = Math.min((int) length, this.length - o); - return new ByteArraySplittableReadData(data, o, l); - } - } - - // not thread-safe - private abstract static class AbstractInputStreamReadData implements ReadData { - - private ByteArraySplittableReadData bytes; - - @Override - public SplittableReadData splittable() throws IOException { - if (bytes == null) { - final byte[] data; - final int length = (int) length(); - if (length >= 0) { - data = new byte[length]; - new DataInputStream(inputStream()).readFully(data); - } else { - data = IOUtils.toByteArray(inputStream()); - } - bytes = new ByteArraySplittableReadData(data, 0, data.length); - } - return bytes; - } - - @Override - public byte[] allBytes() throws IOException, IllegalStateException { - return splittable().allBytes(); - } - } - - // not thread-safe - public static class InputStreamReadData extends AbstractInputStreamReadData { - - private final InputStream inputStream; - private final int length; - - public InputStreamReadData(final InputStream inputStream, final int length) { - this.inputStream = inputStream; - this.length = length; - } - - public InputStreamReadData(final InputStream inputStream) { - this(inputStream, -1); - } - - @Override - public long length() { - return length; - } - - private boolean inputStreamCalled = false; - - @Override - public InputStream inputStream() throws IllegalStateException { - if (inputStreamCalled) { - throw new IllegalStateException("InputStream() already called"); - } else { - inputStreamCalled = true; - return inputStream; - } - } - } - - public static class KeyValueAccessReadData extends AbstractInputStreamReadData { - - private final KeyValueAccess keyValueAccess; - private final String normalPath; - - public KeyValueAccessReadData(final KeyValueAccess keyValueAccess, final String normalPath) { - this.keyValueAccess = keyValueAccess; - this.normalPath = normalPath; - } - - /** - * Open a {@code InputStream} on this data. - *

      - * This will open a {@code LockedChannel} on the underlying {@code - * KeyValueAccess}. Make sure to {@code close()} the returned {@code - * InputStream} to release the underlying {@code LockedChannel}. - * - * @return an InputStream on this data - * - * @throws IOException - * if any I/O error occurs - */ - @Override - public InputStream inputStream() throws IOException { - final LockedChannel channel = keyValueAccess.lockForReading(normalPath); - return new FilterInputStream(channel.newInputStream()) { - @Override - public void close() throws IOException { - in.close(); - channel.close(); - } - }; - } - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index 91f586247..e52ed90ce 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -28,10 +28,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; - import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.readdata.EncodedReadData; +import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("xz") public class XzCompression implements Compression { @@ -71,4 +72,12 @@ public boolean equals(final Object other) { else return preset == ((XzCompression)other).preset; } + + @Override + public ReadData encode(final ReadData readData) { + return new EncodedReadData(readData, out -> { + final XZCompressorOutputStream deflater = new XZCompressorOutputStream(out, preset); + return new EncodedReadData.EncodedOutputStream(deflater, deflater::finish); + }); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java new file mode 100644 index 000000000..0340e70e0 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java @@ -0,0 +1,32 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.DataInputStream; +import java.io.IOException; +import org.apache.commons.io.IOUtils; + +// not thread-safe +abstract class AbstractInputStreamReadData implements ReadData { + + private ByteArraySplittableReadData bytes; + + @Override + public SplittableReadData splittable() throws IOException { + if (bytes == null) { + final byte[] data; + final int length = (int) length(); + if (length >= 0) { + data = new byte[length]; + new DataInputStream(inputStream()).readFully(data); + } else { + data = IOUtils.toByteArray(inputStream()); + } + bytes = new ByteArraySplittableReadData(data); + } + return bytes; + } + + @Override + public byte[] allBytes() throws IOException, IllegalStateException { + return splittable().allBytes(); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java new file mode 100644 index 000000000..9256ea8e7 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java @@ -0,0 +1,57 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +public class ByteArraySplittableReadData implements SplittableReadData { + + private final byte[] data; + private final int offset; + private final int length; + + public ByteArraySplittableReadData(final byte[] data) { + this(data, 0, data.length); + } + + public ByteArraySplittableReadData(final byte[] data, final int offset, final int length) { + this.data = data; + this.offset = offset; + this.length = length; + } + + @Override + public long length() { + return length; + } + + @Override + public InputStream inputStream() throws IOException { + return new ByteArrayInputStream(data, offset, length); + } + + @Override + public byte[] allBytes() { + if (offset == 0 && data.length == length) { + return data; + } else { + return Arrays.copyOfRange(data, offset, offset + length); + } + } + + @Override + public SplittableReadData splittable() throws IOException { + return this; + } + + @Override + public SplittableReadData split(final long offset, final long length) throws IOException { + if (offset < 0 || offset > this.length || length < 0) { + throw new IndexOutOfBoundsException(); + } + final int o = this.offset + (int) offset; + final int l = Math.min((int) length, this.length - o); + return new ByteArraySplittableReadData(data, o, l); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java new file mode 100644 index 000000000..3e60c0190 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java @@ -0,0 +1,84 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public class EncodedReadData implements ReadData { + + public interface OutputStreamEncoder { + + EncodedOutputStream encode(OutputStream out) throws IOException; + } + + public interface Finisher { + void finish() throws IOException; + } + + public static final class EncodedOutputStream { + + private final OutputStream outputStream; + private final Finisher finisher; + + public EncodedOutputStream(final OutputStream outputStream, final Finisher finisher) { + this.outputStream = outputStream; + this.finisher = finisher; + } + + OutputStream outputStream() { + return outputStream; + } + + void finish() throws IOException { + finisher.finish(); + } + } + + private final ReadData source; + + private final OutputStreamEncoder encoder; + + public EncodedReadData(final ReadData data, final OutputStreamEncoder encoder) { + this.source = data; + this.encoder = encoder; + } + + private ByteArraySplittableReadData bytes; + + @Override + public SplittableReadData splittable() throws IOException { + if (bytes == null) { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); + writeTo(baos); + bytes = new ByteArraySplittableReadData(baos.toByteArray()); + } + return bytes; + } + + @Override + public long length() throws IOException { + return splittable().length(); + } + + @Override + public InputStream inputStream() throws IOException, IllegalStateException { + return splittable().inputStream(); + } + + @Override + public byte[] allBytes() throws IOException, IllegalStateException { + return splittable().allBytes(); + } + + @Override + public void writeTo(final OutputStream outputStream) throws IOException, IllegalStateException { + if (bytes != null) { + outputStream.write(bytes.allBytes()); + } else { + final EncodedOutputStream deflater = encoder.encode(outputStream); + source.writeTo(deflater.outputStream()); + deflater.finish(); + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java new file mode 100644 index 000000000..5eaefa598 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java @@ -0,0 +1,36 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.InputStream; + +// not thread-safe +public class InputStreamReadData extends AbstractInputStreamReadData { + + private final InputStream inputStream; + private final int length; + + public InputStreamReadData(final InputStream inputStream, final int length) { + this.inputStream = inputStream; + this.length = length; + } + + public InputStreamReadData(final InputStream inputStream) { + this(inputStream, -1); + } + + @Override + public long length() { + return length; + } + + private boolean inputStreamCalled = false; + + @Override + public InputStream inputStream() throws IllegalStateException { + if (inputStreamCalled) { + throw new IllegalStateException("InputStream() already called"); + } else { + inputStreamCalled = true; + return inputStream; + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java new file mode 100644 index 000000000..948b82d51 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java @@ -0,0 +1,43 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; + +public class KeyValueAccessReadData extends AbstractInputStreamReadData { + + private final KeyValueAccess keyValueAccess; + private final String normalPath; + + public KeyValueAccessReadData(final KeyValueAccess keyValueAccess, final String normalPath) { + this.keyValueAccess = keyValueAccess; + this.normalPath = normalPath; + } + + /** + * Open a {@code InputStream} on this data. + *

      + * This will open a {@code LockedChannel} on the underlying {@code + * KeyValueAccess}. Make sure to {@code close()} the returned {@code + * InputStream} to release the underlying {@code LockedChannel}. + * + * @return an InputStream on this data + * + * @throws IOException + * if any I/O error occurs + */ + @Override + public InputStream inputStream() throws IOException { + final LockedChannel channel = keyValueAccess.lockForReading(normalPath); + return new FilterInputStream(channel.newInputStream()) { + + @Override + public void close() throws IOException { + in.close(); + channel.close(); + } + }; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java new file mode 100644 index 000000000..9c80e9775 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -0,0 +1,96 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import org.janelia.saalfeldlab.n5.BytesCodec; + +public interface ReadData { + + /** + * Returns number of bytes in this {@link ReadData}, if known. Otherwise + * {@code -1}. + * + * @return number of bytes, if known, or -1 + * + * @throws IOException + * if an I/O error occurs while trying to get the length + */ + default long length() throws IOException { + return -1; + } + + /** + * Open a {@code InputStream} on this data. + *

      + * Repeatedly calling this method may or may not work, depending on how + * the underlying data is stored. For example, if the underlying data is + * stored as a {@code byte[]} array, multiple streams can be opened. If + * the underlying data is just an {@code InputStream} then this will be + * returned on the first call. + * + * @return an InputStream on this data + * + * @throws IOException + * if any I/O error occurs + * @throws IllegalStateException + * if this method was already called once and cannot be called again. + */ + InputStream inputStream() throws IOException, IllegalStateException; + + /** + * Return the contained data as a {@code byte[]} array. + *

      + * This may use {@link #inputStream()} to read the data. + * Because repeatedly calling {@link #inputStream()} may not work, + *

        + *
      1. this method may fail with {@code IllegalStateException} if {@code inputStream()} was already called
        2. + *
      2. subsequent {@code inputStream()} calls may fail with {@code IllegalStateException}
        3. + *
      + * + * @return all contained data as a byte[] array + * + * @throws IOException + * if any I/O error occurs + * @throws IllegalStateException + * if {@link #inputStream()} was already called once and cannot be called again. + */ + byte[] allBytes() throws IOException, IllegalStateException; + + /** + * If this {@code ReadData} is a {@code SplittableReadData}, just returns {@code this}. + *

      + * Otherwise, if the underlying data is an {@code InputStream}, all data is read and + * wrapped as a {@code ByteArraySplittableReadData}. + *

      + * The returned {@code SplittableReadData} has a known {@link #length} + * and multiple {@link #inputStream}s can be opened on it. + */ + SplittableReadData splittable() throws IOException; + + + /** + * Write the contained data into an {@code OutputStream}. + *

      + * This may use {@link #inputStream()} to read the data. + * Because repeatedly calling {@link #inputStream()} may not work, + *

        + *
      1. this method may fail with {@code IllegalStateException} if {@code inputStream()} was already called
        2. + *
      2. subsequent {@code inputStream()} calls may fail with {@code IllegalStateException}
        3. + *
      + * + * @param outputStream destination to write to + * @throws IOException + * if any I/O error occurs + * @throws IllegalStateException + * if {@link #inputStream()} was already called once and cannot be called again. + */ + default void writeTo(OutputStream outputStream) throws IOException, IllegalStateException { + outputStream.write(allBytes()); + } + + // TODO: WIP, exploring API options... + default ReadData encode(BytesCodec codec) throws IOException { + return codec.encode(this); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java new file mode 100644 index 000000000..396359c98 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java @@ -0,0 +1,8 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.IOException; + +public interface SplittableReadData extends ReadData { + + ReadData split(final long offset, final long length) throws IOException; +} From 4b6ee4ff12962a78d22b7fc4a073b498c1808d94 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 1 Feb 2025 22:08:46 +0100 Subject: [PATCH 150/423] WIP ReadData.decode(Codec) --- .../janelia/saalfeldlab/n5/BytesCodec.java | 10 +++++---- .../saalfeldlab/n5/DefaultBlockReader.java | 21 ++++++++++++------- .../saalfeldlab/n5/RawCompression.java | 16 +++++++------- .../saalfeldlab/n5/StringDataBlock.java | 10 ++------- .../saalfeldlab/n5/readdata/ReadData.java | 5 +++++ 5 files changed, 35 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java index 42228c50d..72575535c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import org.janelia.saalfeldlab.n5.readdata.InputStreamReadData; import org.janelia.saalfeldlab.n5.readdata.ReadData; public interface BytesCodec { @@ -31,7 +32,10 @@ public interface BytesCodec { * @param readData * @return */ - ReadData encode(ReadData readData) throws IOException; + // TODO add variant that knows the length of the decoded ReadData + default ReadData decode(ReadData readData) throws IOException { + return new InputStreamReadData(decode(readData.inputStream())); + } /** * TODO javadoc @@ -39,7 +43,5 @@ public interface BytesCodec { * @param readData * @return */ - default ReadData decode(ReadData readData) { - throw new UnsupportedOperationException(); - } + ReadData encode(ReadData readData) throws IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index caa69798a..93ccfaef8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -79,13 +79,20 @@ static DataBlock readBlock( dataBlock = dataType.createDataBlock(null, gridPosition, numElements); } - try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { - final int numBytes = dataType.isVarLength() - ? numElements - : (numElements * dataType.bytesPerElement()); - final ReadData data = new InputStreamReadData(inflater, numBytes); - dataBlock.readData(ByteOrder.BIG_ENDIAN, data); - } + final int numBytes = dataType.isVarLength() + ? numElements + : (numElements * dataType.bytesPerElement()); + ReadData data = new InputStreamReadData(in); + data = new InputStreamReadData(data.decode(datasetAttributes.getCompression()).inputStream(), numBytes); + dataBlock.readData(ByteOrder.BIG_ENDIAN, data); + +// try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { +// final int numBytes = dataType.isVarLength() +// ? numElements +// : (numElements * dataType.bytesPerElement()); +// final ReadData data = new InputStreamReadData(inflater, numBytes); +// dataBlock.readData(ByteOrder.BIG_ENDIAN, data); +// } return dataBlock; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index ac67bafed..030a23a3f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -29,8 +29,6 @@ import java.io.InputStream; import java.io.OutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; -import org.janelia.saalfeldlab.n5.readdata.EncodedReadData; -import org.janelia.saalfeldlab.n5.readdata.EncodedReadData.EncodedOutputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("raw") @@ -39,20 +37,17 @@ public class RawCompression implements DefaultBlockWriter, Compression { private static final long serialVersionUID = 7526445806847086477L; @Override - public InputStream getInputStream(final InputStream in) throws IOException { - + public InputStream getInputStream(final InputStream in) { return in; } @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { - + public OutputStream getOutputStream(final OutputStream out) { return out; } @Override public boolean equals(final Object other) { - if (other == null || other.getClass() != RawCompression.class) return false; else @@ -61,6 +56,11 @@ public boolean equals(final Object other) { @Override public ReadData encode(final ReadData readData) { - return new EncodedReadData(readData, out -> new EncodedOutputStream(out, out::flush)); + return readData; + } + + @Override + public ReadData decode(final ReadData readData) { + return readData; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 2faff64fe..025e0024c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -31,6 +31,8 @@ import java.nio.ByteOrder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import org.janelia.saalfeldlab.n5.readdata.ByteArraySplittableReadData; +import org.janelia.saalfeldlab.n5.readdata.ReadData; public class StringDataBlock extends AbstractDataBlock { @@ -57,14 +59,6 @@ public void deserialize(final ByteBuffer serialized) { actualData = rawChars.split(NULLCHAR); } - @Override - public void writeData(final ByteOrder byteOrder, final OutputStream outputStream) throws IOException { - if (serializedData == null) { - serializedData = serialize(byteOrder).array(); - } - outputStream.write(serializedData); - } - @Override public int getNumElements() { if (serializedData == null) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 9c80e9775..ddbaa1b8d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -93,4 +93,9 @@ default void writeTo(OutputStream outputStream) throws IOException, IllegalState default ReadData encode(BytesCodec codec) throws IOException { return codec.encode(this); } + + // TODO: WIP, exploring API options... + default ReadData decode(BytesCodec codec) throws IOException { + return codec.decode(this); + } } From 47578e3ddd2c70566f20dfec4b910093daea7183 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 1 Feb 2025 22:20:51 +0100 Subject: [PATCH 151/423] Add decode(ReadData readData) variant that knows the length of the decoded data --- .../java/org/janelia/saalfeldlab/n5/BytesCodec.java | 7 ++++--- .../janelia/saalfeldlab/n5/DefaultBlockReader.java | 12 ++---------- .../org/janelia/saalfeldlab/n5/RawCompression.java | 2 +- .../janelia/saalfeldlab/n5/readdata/ReadData.java | 7 ++++++- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java index 72575535c..d9f9a8dc2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java @@ -30,11 +30,12 @@ public interface BytesCodec { * TODO javadoc * * @param readData + * @param decodedLength -1 if unknown * @return + * @throws IOException */ - // TODO add variant that knows the length of the decoded ReadData - default ReadData decode(ReadData readData) throws IOException { - return new InputStreamReadData(decode(readData.inputStream())); + default ReadData decode(ReadData readData, int decodedLength) throws IOException { + return new InputStreamReadData(decode(readData.inputStream()), decodedLength); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 93ccfaef8..3eb0273d9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -82,18 +82,10 @@ static DataBlock readBlock( final int numBytes = dataType.isVarLength() ? numElements : (numElements * dataType.bytesPerElement()); - ReadData data = new InputStreamReadData(in); - data = new InputStreamReadData(data.decode(datasetAttributes.getCompression()).inputStream(), numBytes); + final ReadData data = new InputStreamReadData(in) + .decode(datasetAttributes.getCompression(), numBytes); dataBlock.readData(ByteOrder.BIG_ENDIAN, data); -// try (final InputStream inflater = datasetAttributes.getCompression().decode(in)) { -// final int numBytes = dataType.isVarLength() -// ? numElements -// : (numElements * dataType.bytesPerElement()); -// final ReadData data = new InputStreamReadData(inflater, numBytes); -// dataBlock.readData(ByteOrder.BIG_ENDIAN, data); -// } - return dataBlock; } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index 030a23a3f..dc9cd4238 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -60,7 +60,7 @@ public ReadData encode(final ReadData readData) { } @Override - public ReadData decode(final ReadData readData) { + public ReadData decode(final ReadData readData, int decodedLength) { return readData; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index ddbaa1b8d..658277930 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -96,6 +96,11 @@ default ReadData encode(BytesCodec codec) throws IOException { // TODO: WIP, exploring API options... default ReadData decode(BytesCodec codec) throws IOException { - return codec.decode(this); + return decode(codec, -1); + } + + // TODO: WIP, exploring API options... + default ReadData decode(BytesCodec codec, int decodedLength) throws IOException { + return codec.decode(this, decodedLength); } } From 80fa6a87827381ec59f007701eb3e26715557cff Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 2 Feb 2025 20:21:20 +0100 Subject: [PATCH 152/423] Hide ReadData implementations, use static ReadData factory methods instead --- .../janelia/saalfeldlab/n5/BytesCodec.java | 3 +- .../saalfeldlab/n5/Bzip2Compression.java | 6 +-- .../org/janelia/saalfeldlab/n5/DataBlock.java | 3 +- .../saalfeldlab/n5/DefaultBlockReader.java | 3 +- .../saalfeldlab/n5/GzipCompression.java | 7 ++-- .../saalfeldlab/n5/Lz4Compression.java | 5 +-- .../saalfeldlab/n5/StringDataBlock.java | 4 -- .../janelia/saalfeldlab/n5/XzCompression.java | 6 +-- .../readdata/ByteArraySplittableReadData.java | 6 +-- .../n5/readdata/EncodedReadData.java | 34 ++-------------- .../n5/readdata/InputStreamReadData.java | 8 +--- .../n5/readdata/KeyValueAccessReadData.java | 4 +- .../n5/readdata/OutputStreamEncoder.java | 33 ++++++++++++++++ .../saalfeldlab/n5/readdata/ReadData.java | 39 +++++++++++++++++++ 14 files changed, 96 insertions(+), 65 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/OutputStreamEncoder.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java index d9f9a8dc2..1cd2ba256 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import org.janelia.saalfeldlab.n5.readdata.InputStreamReadData; import org.janelia.saalfeldlab.n5.readdata.ReadData; public interface BytesCodec { @@ -35,7 +34,7 @@ public interface BytesCodec { * @throws IOException */ default ReadData decode(ReadData readData, int decodedLength) throws IOException { - return new InputStreamReadData(decode(readData.inputStream()), decodedLength); + return ReadData.from(decode(readData.inputStream()), decodedLength); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 600577182..55b32ac95 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -31,7 +31,7 @@ import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; -import org.janelia.saalfeldlab.n5.readdata.EncodedReadData; +import org.janelia.saalfeldlab.n5.readdata.OutputStreamEncoder; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("bzip2") @@ -75,9 +75,9 @@ public boolean equals(final Object other) { @Override public ReadData encode(final ReadData readData) { - return new EncodedReadData(readData, out -> { + return readData.encode(out -> { final BZip2CompressorOutputStream deflater = new BZip2CompressorOutputStream(out, blockSize); - return new EncodedReadData.EncodedOutputStream(deflater, deflater::finish); + return new OutputStreamEncoder.EncodedOutputStream(deflater, deflater::finish); }); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 44395e2e1..00b77363d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -28,7 +28,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.readdata.ByteArraySplittableReadData; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** @@ -79,7 +78,7 @@ default void readData(final ByteOrder byteOrder, final ReadData readData) throws } default ReadData writeData(final ByteOrder byteOrder) { - return new ByteArraySplittableReadData(serialize(byteOrder).array()); + return ReadData.from(serialize(byteOrder)); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 3eb0273d9..241f30c8b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.readdata.InputStreamReadData; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** @@ -82,7 +81,7 @@ static DataBlock readBlock( final int numBytes = dataType.isVarLength() ? numElements : (numElements * dataType.bytesPerElement()); - final ReadData data = new InputStreamReadData(in) + final ReadData data = ReadData.from(in) .decode(datasetAttributes.getCompression(), numBytes); dataBlock.readData(ByteOrder.BIG_ENDIAN, data); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index 1db23d239..535def6d0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -36,8 +36,7 @@ import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; import org.apache.commons.compress.compressors.gzip.GzipParameters; import org.janelia.saalfeldlab.n5.Compression.CompressionType; -import org.janelia.saalfeldlab.n5.readdata.EncodedReadData; -import org.janelia.saalfeldlab.n5.readdata.EncodedReadData.EncodedOutputStream; +import org.janelia.saalfeldlab.n5.readdata.OutputStreamEncoder.EncodedOutputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("gzip") @@ -110,12 +109,12 @@ public boolean equals(final Object other) { @Override public ReadData encode(final ReadData readData) { if (useZlib) { - return new EncodedReadData(readData, out -> { + return readData.encode(out -> { final DeflaterOutputStream deflater = new DeflaterOutputStream(out, new Deflater(level)); return new EncodedOutputStream(deflater, deflater::finish); }); } else { - return new EncodedReadData(readData, out -> { + return readData.encode(out -> { parameters.setCompressionLevel(level); final GzipCompressorOutputStream deflater = new GzipCompressorOutputStream(out, parameters); return new EncodedOutputStream(deflater, deflater::finish); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index c5db729c7..e36ca5511 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -33,8 +33,7 @@ import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; -import org.janelia.saalfeldlab.n5.readdata.EncodedReadData; -import org.janelia.saalfeldlab.n5.readdata.EncodedReadData.EncodedOutputStream; +import org.janelia.saalfeldlab.n5.readdata.OutputStreamEncoder.EncodedOutputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("lz4") @@ -78,7 +77,7 @@ public boolean equals(final Object other) { @Override public ReadData encode(final ReadData readData) { - return new EncodedReadData(readData, out -> { + return readData.encode(out -> { final LZ4BlockOutputStream deflater = new LZ4BlockOutputStream(out, blockSize); return new EncodedOutputStream(deflater, deflater::finish); }); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 025e0024c..979a9ffce 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -25,14 +25,10 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import org.janelia.saalfeldlab.n5.readdata.ByteArraySplittableReadData; -import org.janelia.saalfeldlab.n5.readdata.ReadData; public class StringDataBlock extends AbstractDataBlock { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index e52ed90ce..8be2010b2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -31,7 +31,7 @@ import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; -import org.janelia.saalfeldlab.n5.readdata.EncodedReadData; +import org.janelia.saalfeldlab.n5.readdata.OutputStreamEncoder; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("xz") @@ -75,9 +75,9 @@ public boolean equals(final Object other) { @Override public ReadData encode(final ReadData readData) { - return new EncodedReadData(readData, out -> { + return readData.encode(out -> { final XZCompressorOutputStream deflater = new XZCompressorOutputStream(out, preset); - return new EncodedReadData.EncodedOutputStream(deflater, deflater::finish); + return new OutputStreamEncoder.EncodedOutputStream(deflater, deflater::finish); }); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java index 9256ea8e7..1fadedfff 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java @@ -5,17 +5,17 @@ import java.io.InputStream; import java.util.Arrays; -public class ByteArraySplittableReadData implements SplittableReadData { +class ByteArraySplittableReadData implements SplittableReadData { private final byte[] data; private final int offset; private final int length; - public ByteArraySplittableReadData(final byte[] data) { + ByteArraySplittableReadData(final byte[] data) { this(data, 0, data.length); } - public ByteArraySplittableReadData(final byte[] data, final int offset, final int length) { + ByteArraySplittableReadData(final byte[] data, final int offset, final int length) { this.data = data; this.offset = offset; this.length = length; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java index 3e60c0190..18a221419 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java @@ -5,41 +5,13 @@ import java.io.InputStream; import java.io.OutputStream; -public class EncodedReadData implements ReadData { - - public interface OutputStreamEncoder { - - EncodedOutputStream encode(OutputStream out) throws IOException; - } - - public interface Finisher { - void finish() throws IOException; - } - - public static final class EncodedOutputStream { - - private final OutputStream outputStream; - private final Finisher finisher; - - public EncodedOutputStream(final OutputStream outputStream, final Finisher finisher) { - this.outputStream = outputStream; - this.finisher = finisher; - } - - OutputStream outputStream() { - return outputStream; - } - - void finish() throws IOException { - finisher.finish(); - } - } +class EncodedReadData implements ReadData { private final ReadData source; private final OutputStreamEncoder encoder; - public EncodedReadData(final ReadData data, final OutputStreamEncoder encoder) { + EncodedReadData(final ReadData data, final OutputStreamEncoder encoder) { this.source = data; this.encoder = encoder; } @@ -76,7 +48,7 @@ public void writeTo(final OutputStream outputStream) throws IOException, Illegal if (bytes != null) { outputStream.write(bytes.allBytes()); } else { - final EncodedOutputStream deflater = encoder.encode(outputStream); + final OutputStreamEncoder.EncodedOutputStream deflater = encoder.encode(outputStream); source.writeTo(deflater.outputStream()); deflater.finish(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java index 5eaefa598..8d15bde13 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java @@ -3,20 +3,16 @@ import java.io.InputStream; // not thread-safe -public class InputStreamReadData extends AbstractInputStreamReadData { +class InputStreamReadData extends AbstractInputStreamReadData { private final InputStream inputStream; private final int length; - public InputStreamReadData(final InputStream inputStream, final int length) { + InputStreamReadData(final InputStream inputStream, final int length) { this.inputStream = inputStream; this.length = length; } - public InputStreamReadData(final InputStream inputStream) { - this(inputStream, -1); - } - @Override public long length() { return length; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java index 948b82d51..8ad582646 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java @@ -6,12 +6,12 @@ import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; -public class KeyValueAccessReadData extends AbstractInputStreamReadData { +class KeyValueAccessReadData extends AbstractInputStreamReadData { private final KeyValueAccess keyValueAccess; private final String normalPath; - public KeyValueAccessReadData(final KeyValueAccess keyValueAccess, final String normalPath) { + KeyValueAccessReadData(final KeyValueAccess keyValueAccess, final String normalPath) { this.keyValueAccess = keyValueAccess; this.normalPath = normalPath; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/OutputStreamEncoder.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/OutputStreamEncoder.java new file mode 100644 index 000000000..3eed7ada8 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/OutputStreamEncoder.java @@ -0,0 +1,33 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.IOException; +import java.io.OutputStream; + +public interface OutputStreamEncoder { + + EncodedOutputStream encode(OutputStream out) throws IOException; + + interface Finisher { + + void finish() throws IOException; + } + + final class EncodedOutputStream { + + private final OutputStream outputStream; + private final Finisher finisher; + + public EncodedOutputStream(final OutputStream outputStream, final Finisher finisher) { + this.outputStream = outputStream; + this.finisher = finisher; + } + + OutputStream outputStream() { + return outputStream; + } + + void finish() throws IOException { + finisher.finish(); + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 658277930..26b69e915 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -3,7 +3,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import org.janelia.saalfeldlab.n5.BytesCodec; +import org.janelia.saalfeldlab.n5.KeyValueAccess; public interface ReadData { @@ -94,6 +96,11 @@ default ReadData encode(BytesCodec codec) throws IOException { return codec.encode(this); } + // TODO: WIP, exploring API options... + default ReadData encode(OutputStreamEncoder encoder) { + return new EncodedReadData(this, encoder); + } + // TODO: WIP, exploring API options... default ReadData decode(BytesCodec codec) throws IOException { return decode(codec, -1); @@ -103,4 +110,36 @@ default ReadData decode(BytesCodec codec) throws IOException { default ReadData decode(BytesCodec codec, int decodedLength) throws IOException { return codec.decode(this, decodedLength); } + + + // --------------- Factory Methods ------------------ + + static ReadData from(final InputStream inputStream, final int length) { + return new InputStreamReadData(inputStream, length); + } + + static ReadData from(final InputStream inputStream) { + return from(inputStream, -1); + } + + static ReadData from(final KeyValueAccess keyValueAccess, final String normalPath) { + return new KeyValueAccessReadData(keyValueAccess, normalPath); + } + + static ReadData from(final byte[] data, final int offset, final int length) { + return new ByteArraySplittableReadData(data, offset, length); + } + + static ReadData from(final byte[] data) { + return from(data, 0, data.length); + } + + static ReadData from(final ByteBuffer data) { + if (data.hasArray()) { + return from(data.array(), 0, data.limit()); + } else { + throw new UnsupportedOperationException("TODO. Direct ByteBuffer not supported yet."); + } + } + } From a43bf40a8378526bfa7bad3921cc5d5cdd1ca1cd Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 2 Feb 2025 20:31:13 +0100 Subject: [PATCH 153/423] Remove deprecated BlockWriter/BlockReader interfaces --- .../janelia/saalfeldlab/n5/BlockReader.java | 56 ------------------- .../janelia/saalfeldlab/n5/BlockWriter.java | 54 ------------------ 2 files changed, 110 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java deleted file mode 100644 index 8d0103b74..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import java.io.InputStream; - -/** - * Interface for block readers that can read a {@link DataBlock} from an - * {@link InputStream}. - * - * @author Stephan Saalfeld - */ -@Deprecated -public interface BlockReader { - - /** - * Reads a {@link DataBlock} from an {@link InputStream}. - * - * @param dataBlock - * the data block - * @param in - * the input stream - * @param - * the block data type - * @param - * the block type - * @throws IOException - * the exception - */ - @Deprecated - public > void read(final B dataBlock, final InputStream in) throws IOException; -} \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java deleted file mode 100644 index 884200d58..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import java.io.OutputStream; - -/** - * Interface for block writers that can write a {@link DataBlock} into an - * {@link OutputStream}. - * - * @author Stephan Saalfeld - */ -@Deprecated -public interface BlockWriter { - - /** - * Writes a {@link DataBlock} into an {@link OutputStream}. - * - * @param dataBlock - * the data block - * @param out - * the output stream - * @param - * the block data type - * @throws IOException - * the exception - */ - @Deprecated - public void write(final DataBlock dataBlock, final OutputStream out) throws IOException; -} \ No newline at end of file From 89a6bfd805be826718a898f7fbdad3269c789cbd Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 2 Feb 2025 20:55:50 +0100 Subject: [PATCH 154/423] Clean up --- .../java/org/janelia/saalfeldlab/n5/GzipCompression.java | 2 +- .../java/org/janelia/saalfeldlab/n5/Lz4Compression.java | 6 +++--- .../java/org/janelia/saalfeldlab/n5/RawCompression.java | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index 535def6d0..48ba089c6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -40,7 +40,7 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("gzip") -public class GzipCompression implements DefaultBlockReader, DefaultBlockWriter, Compression { +public class GzipCompression implements Compression { private static final long serialVersionUID = 8630847239813334263L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index e36ca5511..4042e9a38 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -37,7 +37,7 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("lz4") -public class Lz4Compression implements DefaultBlockReader, DefaultBlockWriter, Compression { +public class Lz4Compression implements Compression { private static final long serialVersionUID = -9071316415067427256L; @@ -55,13 +55,13 @@ public Lz4Compression() { } @Override - public InputStream getInputStream(final InputStream in) throws IOException { + public InputStream getInputStream(final InputStream in) { return new LZ4BlockInputStream(in); } @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { + public OutputStream getOutputStream(final OutputStream out) { return new LZ4BlockOutputStream(out, blockSize); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index dc9cd4238..053041164 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -25,14 +25,13 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("raw") -public class RawCompression implements DefaultBlockWriter, Compression { +public class RawCompression implements Compression { private static final long serialVersionUID = 7526445806847086477L; From 5ff62a98e7f93d31306715ee675e5067e862ab88 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 2 Feb 2025 21:36:12 +0100 Subject: [PATCH 155/423] javacod --- .../saalfeldlab/n5/readdata/ReadData.java | 151 ++++++++++++++++-- 1 file changed, 137 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 26b69e915..f21f2f571 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -7,6 +7,23 @@ import org.janelia.saalfeldlab.n5.BytesCodec; import org.janelia.saalfeldlab.n5.KeyValueAccess; +/** + * An abstraction over {@code byte[]} data. + *

      + * The data may come from a {@code byte[]} array, a {@code ByteBuffer}, an + * {@code InputStream}, a {@code KeyValueAccess}. + *

      + * {@code ReadData} instances can be created via one of the static {@link #from} + * methods. For example, use {@link #from(InputStream, int)} to wrap an {@code + * InputStream}. + *

      + * {@code ReadData} may be lazy-loaded. For example, for {@code InputStream} and + * {@code KeyValueAccess} sources, loading is deferred until the data is + * accessed (e.g., {@link #allBytes()}, {@link #writeTo(OutputStream)}). + *

      + * {@code ReadData} can be {@code encoded} and {@code decoded} with a {@code + * Codec}, which will also be lazy if possible. + */ public interface ReadData { /** @@ -70,7 +87,6 @@ default long length() throws IOException { */ SplittableReadData splittable() throws IOException; - /** * Write the contained data into an {@code OutputStream}. *

      @@ -81,7 +97,9 @@ default long length() throws IOException { *

    2. subsequent {@code inputStream()} calls may fail with {@code IllegalStateException}
      3. *
    * - * @param outputStream destination to write to + * @param outputStream + * destination to write to + * * @throws IOException * if any I/O error occurs * @throws IllegalStateException @@ -91,55 +109,160 @@ default void writeTo(OutputStream outputStream) throws IOException, IllegalState outputStream.write(allBytes()); } + // + // + // ------------- Encoding / Decoding ---------------- + // + // TODO: WIP, exploring API options... + + /** + * Returns a new ReadData that uses the given {@code Codec} to encode this + * ReadData. + * + * @param codec + * Codec to use for encoding + * + * @return encoded ReadData + * + * @throws IOException + * if any I/O error occurs + */ default ReadData encode(BytesCodec codec) throws IOException { return codec.encode(this); } - // TODO: WIP, exploring API options... + /** + * Returns a new ReadData that uses the given {@code OutputStreamEncoder} to + * encode this ReadData. + * + * @param encoder + * OutputStreamEncoder to use for encoding + * + * @return encoded ReadData + * + * @throws IOException + * if any I/O error occurs + */ default ReadData encode(OutputStreamEncoder encoder) { return new EncodedReadData(this, encoder); } - // TODO: WIP, exploring API options... - default ReadData decode(BytesCodec codec) throws IOException { - return decode(codec, -1); - } - - // TODO: WIP, exploring API options... + /** + * Returns a new ReadData that uses the given {@code codec} to decode this + * ReadData. + *

    + * The returned ReadData reports {@link #length()}{@code == decodedLength}. + * Decoding may be lazy or eager, depending on the {@code BytesCodec}. + * + * @param codec + * Codec to use for decoding + * @param decodedLength + * length of the decoded data (-1 if unknown). + * + * @return decoded ReadData + * + * @throws IOException + * if any I/O error occurs + */ default ReadData decode(BytesCodec codec, int decodedLength) throws IOException { return codec.decode(this, decodedLength); } - + // + // // --------------- Factory Methods ------------------ + // + /** + * Create a new {@code ReadData} that loads lazily from {@code inputStream} + * and {@link #length() reports} the given {@code length}. + *

    + * No effort is made to ensure that the {@code inputStream} in fact contains + * exactly {@code length} bytes. + * + * @param inputStream + * InputStream to read from + * @param length + * reported length of the ReadData + * + * @return a new ReadData + */ static ReadData from(final InputStream inputStream, final int length) { return new InputStreamReadData(inputStream, length); } + /** + * Create a new {@code ReadData} that loads lazily from {@code inputStream} + * and reports {@link #length() length() == -1} (i.e., unknown length). + * + * @param inputStream + * InputStream to read from + * + * @return a new ReadData + */ static ReadData from(final InputStream inputStream) { return from(inputStream, -1); } + /** + * Create a new {@code ReadData} that loads lazily from {@code normalPath} + * in {@code keyValueAccess}. The returned ReadData reports {@link #length() + * length() == -1} (i.e., unknown length). + * + * @param keyValueAccess + * KeyValueAccess to read from + * @param normalPath + * path in the {@code keyValueAccess} to read from + * + * @return a new ReadData + */ static ReadData from(final KeyValueAccess keyValueAccess, final String normalPath) { return new KeyValueAccessReadData(keyValueAccess, normalPath); } - static ReadData from(final byte[] data, final int offset, final int length) { + /** + * Create a new {@code ReadData} that wraps the specified portion of a + * {@code byte[]} array. + * + * @param data + * array containing the data + * @param offset + * start offset of the ReadData in the data array + * @param length + * length of the ReadData (in bytes) + * + * @return a new ReadData + */ + static SplittableReadData from(final byte[] data, final int offset, final int length) { return new ByteArraySplittableReadData(data, offset, length); } - static ReadData from(final byte[] data) { + /** + * Create a new {@code ReadData} that wraps the given {@code byte[]} array. + * + * @param data + * array containing the data + * + * @return a new ReadData + */ + static SplittableReadData from(final byte[] data) { return from(data, 0, data.length); } - static ReadData from(final ByteBuffer data) { + /** + * Create a new {@code ReadData} that wraps the given {@code ByteBuffer}. + * + * @param data + * buffer containing the data + * + * @return a new ReadData + */ + static SplittableReadData from(final ByteBuffer data) { if (data.hasArray()) { return from(data.array(), 0, data.limit()); } else { throw new UnsupportedOperationException("TODO. Direct ByteBuffer not supported yet."); } } - } From decdafdada750047452e65027da9fa2ccb1219bc Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 3 Feb 2025 12:32:09 +0100 Subject: [PATCH 156/423] Remove old Compression methods. Everything goes through ReadData --- .../janelia/saalfeldlab/n5/BytesCodec.java | 49 +++++++++---------- .../saalfeldlab/n5/Bzip2Compression.java | 19 +++---- .../janelia/saalfeldlab/n5/Compression.java | 37 -------------- .../saalfeldlab/n5/GzipCompression.java | 39 +++++---------- .../saalfeldlab/n5/Lz4Compression.java | 21 +++----- .../saalfeldlab/n5/RawCompression.java | 12 ----- .../janelia/saalfeldlab/n5/XzCompression.java | 19 +++---- 7 files changed, 53 insertions(+), 143 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java index 1cd2ba256..2d69211c3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java @@ -1,47 +1,42 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; public interface BytesCodec { /** - * Decode an {@link InputStream}. + * Decode the given {@code readData}. + *

    + * The returned decoded {@code ReadData} reports {@link ReadData#length() + * length()}{@code == decodedLength}. Decoding may be lazy or eager, + * depending on the {@code BytesCodec} implementation. * - * @param in - * input stream - * @return the decoded input stream - */ - InputStream decode(InputStream in) throws IOException; - - /** - * Encode an {@link OutputStream}. + * @param readData + * data to decode + * @param decodedLength + * length of the decoded data (-1 if unknown) * - * @param out - * the output stream - * @return the encoded output stream - */ - OutputStream encode(OutputStream out) throws IOException; - - /** - * TODO javadoc + * @return decoded ReadData * - * @param readData - * @param decodedLength -1 if unknown - * @return * @throws IOException + * if any I/O error occurs */ - default ReadData decode(ReadData readData, int decodedLength) throws IOException { - return ReadData.from(decode(readData.inputStream()), decodedLength); - } + ReadData decode(ReadData readData, int decodedLength) throws IOException; /** - * TODO javadoc + * Encode the given {@code readData}. + *

    + * Encoding may be lazy or eager, depending on the {@code BytesCodec} + * implementation. * * @param readData - * @return + * data to encode + * + * @return encoded ReadData + * + * @throws IOException + * if any I/O error occurs */ ReadData encode(ReadData readData) throws IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 55b32ac95..9f084601c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -27,7 +27,6 @@ import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; @@ -52,18 +51,6 @@ public Bzip2Compression() { this(BZip2CompressorOutputStream.MAX_BLOCKSIZE); } - @Override - public InputStream getInputStream(final InputStream in) throws IOException { - - return new BZip2CompressorInputStream(in); - } - - @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { - - return new BZip2CompressorOutputStream(out, blockSize); - } - @Override public boolean equals(final Object other) { @@ -73,6 +60,12 @@ public boolean equals(final Object other) { return blockSize == ((Bzip2Compression)other).blockSize; } + @Override + public ReadData decode(final ReadData readData, final int decodedLength) throws IOException { + final InputStream inflater = new BZip2CompressorInputStream(readData.inputStream()); + return ReadData.from(inflater, decodedLength); + } + @Override public ReadData encode(final ReadData readData) { return readData.encode(out -> { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index 1236e9c86..d9cac2fb7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -25,11 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.io.Serializable; import java.lang.annotation.ElementType; import java.lang.annotation.Inherited; @@ -75,36 +70,4 @@ default String getType() { else return compressionType.value(); } - - // --------------------------------------------------------------------------------- - // TODO. clean up interface hierarchy. - // getInputStream and getOutputStream are duplicated here from DefaultBlockReader/Writer - // to allow for default implementation of decode/encode (which are copied from wip/codecsShards). - InputStream getInputStream(final InputStream in) throws IOException; - - OutputStream getOutputStream(final OutputStream out) throws IOException; - - /** - * Decode an {@link InputStream}. - * - * @param in - * input stream - * @return the decoded input stream - */ - @Override - default InputStream decode(final InputStream in) throws IOException { - return getInputStream(in); - } - - /** - * Encode an {@link OutputStream}. - * - * @param out - * the output stream - * @return the encoded output stream - */ - @Override - default OutputStream encode(final OutputStream out) throws IOException { - return getOutputStream(out); - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index 48ba089c6..949d54fa8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -27,8 +27,6 @@ import java.io.IOException; import java.io.InputStream; -import java.io.ObjectInputStream; -import java.io.OutputStream; import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; @@ -69,41 +67,28 @@ public GzipCompression(final int level, final boolean useZlib) { } @Override - public InputStream getInputStream(final InputStream in) throws IOException { + public boolean equals(final Object other) { - if (useZlib) { - return new InflaterInputStream(in); - } else { - return new GzipCompressorInputStream(in, true); + if (other == null || other.getClass() != GzipCompression.class) + return false; + else { + final GzipCompression gz = ((GzipCompression)other); + return useZlib == gz.useZlib && level == gz.level; } } - @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { - + private InputStream decode(final InputStream in) throws IOException { if (useZlib) { - return new DeflaterOutputStream(out, new Deflater(level)); + return new InflaterInputStream(in); } else { - parameters.setCompressionLevel(level); - return new GzipCompressorOutputStream(out, parameters); + return new GzipCompressorInputStream(in, true); } } - private void readObject(final ObjectInputStream in) throws Exception { - - in.defaultReadObject(); - ReflectionUtils.setFieldValue(this, "parameters", new GzipParameters()); - } - @Override - public boolean equals(final Object other) { - - if (other == null || other.getClass() != GzipCompression.class) - return false; - else { - final GzipCompression gz = ((GzipCompression)other); - return useZlib == gz.useZlib && level == gz.level; - } + public ReadData decode(final ReadData readData, final int decodedLength) throws IOException { + final InputStream inflater = decode(readData.inputStream()); + return ReadData.from(inflater, decodedLength); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index 4042e9a38..2a8e9c2c6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -26,9 +26,8 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.InputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; import net.jpountz.lz4.LZ4BlockInputStream; @@ -54,18 +53,6 @@ public Lz4Compression() { this(1 << 16); } - @Override - public InputStream getInputStream(final InputStream in) { - - return new LZ4BlockInputStream(in); - } - - @Override - public OutputStream getOutputStream(final OutputStream out) { - - return new LZ4BlockOutputStream(out, blockSize); - } - @Override public boolean equals(final Object other) { @@ -75,6 +62,12 @@ public boolean equals(final Object other) { return blockSize == ((Lz4Compression)other).blockSize; } + @Override + public ReadData decode(final ReadData readData, final int decodedLength) throws IOException { + final InputStream inflater = new LZ4BlockInputStream(readData.inputStream()); + return ReadData.from(inflater, decodedLength); + } + @Override public ReadData encode(final ReadData readData) { return readData.encode(out -> { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index 053041164..e895a6fb7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -25,8 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.InputStream; -import java.io.OutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -35,16 +33,6 @@ public class RawCompression implements Compression { private static final long serialVersionUID = 7526445806847086477L; - @Override - public InputStream getInputStream(final InputStream in) { - return in; - } - - @Override - public OutputStream getOutputStream(final OutputStream out) { - return out; - } - @Override public boolean equals(final Object other) { if (other == null || other.getClass() != RawCompression.class) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index 8be2010b2..bc32f93f3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -27,7 +27,6 @@ import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; @@ -52,18 +51,6 @@ public XzCompression() { this(6); } - @Override - public InputStream getInputStream(final InputStream in) throws IOException { - - return new XZCompressorInputStream(in); - } - - @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { - - return new XZCompressorOutputStream(out, preset); - } - @Override public boolean equals(final Object other) { @@ -73,6 +60,12 @@ public boolean equals(final Object other) { return preset == ((XzCompression)other).preset; } + @Override + public ReadData decode(final ReadData readData, final int decodedLength) throws IOException { + final InputStream inflater = new XZCompressorInputStream(readData.inputStream()); + return ReadData.from(inflater, decodedLength); + } + @Override public ReadData encode(final ReadData readData) { return readData.encode(out -> { From 3b54467a253f2ec255ea74248bbc10fc5cdb1570 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 3 Feb 2025 13:42:06 +0100 Subject: [PATCH 157/423] Remove DataBlock de/serialize() methods Maybe we'll rename readData()/writeData() later, the point is to remove the ByteBuffer methods from the API --- .../saalfeldlab/n5/ByteArrayDataBlock.java | 14 ++---- .../org/janelia/saalfeldlab/n5/DataBlock.java | 44 ++++++++++++++----- .../saalfeldlab/n5/DoubleArrayDataBlock.java | 15 ++++--- .../saalfeldlab/n5/FloatArrayDataBlock.java | 15 ++++--- .../saalfeldlab/n5/IntArrayDataBlock.java | 15 ++++--- .../saalfeldlab/n5/LongArrayDataBlock.java | 15 ++++--- .../org/janelia/saalfeldlab/n5/N5Reader.java | 7 +-- .../saalfeldlab/n5/ShortArrayDataBlock.java | 15 ++++--- .../saalfeldlab/n5/StringDataBlock.java | 29 +++++++----- 9 files changed, 104 insertions(+), 65 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index b3465aadc..a97c515d1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -27,7 +27,6 @@ import java.io.DataInputStream; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.ByteOrder; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -39,18 +38,13 @@ public ByteArrayDataBlock(final int[] size, final long[] gridPosition, final byt } @Override - public ByteBuffer serialize(final ByteOrder byteOrder) { - return ByteBuffer.wrap(data); - } - - @Override - public void deserialize(final ByteBuffer serialized) { - System.arraycopy(serialized.array(), 0, data, 0, data.length); + public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { + new DataInputStream(readData.inputStream()).readFully(data); } @Override - public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - new DataInputStream(readData.inputStream()).readFully(data); + public ReadData writeData(final ByteOrder byteOrder) { + return ReadData.from(data); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 00b77363d..5a93b4267 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -69,17 +69,41 @@ public interface DataBlock { */ T getData(); - ByteBuffer serialize(ByteOrder byteOrder); - - void deserialize(ByteBuffer serialized); - - default void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - deserialize(ByteBuffer.wrap(readData.allBytes()).order(byteOrder)); - } + /** + * Read (deserialize) the data object of this data block from a {@link ReadData}. + *

    + * The {@code ReadData} may or may not map directly to the data + * object of this data block. I.e. modifying the {@code ReadData} after + * calling this method may or may not change the data of this data block. + * modifying the data object of this data block after calling this method + * may or may not change the content of the {@code ReadData}. + * + * @param byteOrder + * ByteOrder to use for serialization + * @param readData + * data to deserialize + */ + // TODO: include ByteOrder in ReadData + // TODO: rename? "readFrom"? "deserializeFrom"? + void readData(ByteOrder byteOrder, ReadData readData) throws IOException; - default ReadData writeData(final ByteOrder byteOrder) { - return ReadData.from(serialize(byteOrder)); - } + /** + * Creates a {@link ReadData} that contains the serialized data object of + * this data block. + *

    + * The {@code ReadData} may or may not map directly to the data + * object of this data block. I.e. modifying the {@code ReadData} after + * calling this method may or may not change the data of this data block. + * modifying the data object of this data block after calling this method + * may or may not change the content of the {@code ReadData}. + * + * @param byteOrder + * ByteOrder to use for serialization + * + * @return serialized {@code ReadData} + */ + // TODO: rename? "serialize"? "write"? + ReadData writeData(ByteOrder byteOrder); /** * Returns the number of elements in this {@link DataBlock}. This number is diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 5781f7da5..9424b4617 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -25,8 +25,10 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import org.janelia.saalfeldlab.n5.readdata.ReadData; public class DoubleArrayDataBlock extends AbstractDataBlock { @@ -36,15 +38,16 @@ public DoubleArrayDataBlock(final int[] size, final long[] gridPosition, final d } @Override - public ByteBuffer serialize(final ByteOrder byteOrder) { - final ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES * data.length); - buffer.order(byteOrder).asDoubleBuffer().put(data); - return buffer; + public ReadData writeData(final ByteOrder byteOrder) { + final ByteBuffer serialized = ByteBuffer.allocate(Double.BYTES * data.length); + serialized.order(byteOrder).asDoubleBuffer().put(data); + return ReadData.from(serialized); } @Override - public void deserialize(final ByteBuffer serialized) { - serialized.asDoubleBuffer().get(data); + public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { + final ByteBuffer serialized = ByteBuffer.wrap(readData.allBytes()); + serialized.order(byteOrder).asDoubleBuffer().get(data); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index 7211177d7..9c3a4c8c9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -25,8 +25,10 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import org.janelia.saalfeldlab.n5.readdata.ReadData; public class FloatArrayDataBlock extends AbstractDataBlock { @@ -36,15 +38,16 @@ public FloatArrayDataBlock(final int[] size, final long[] gridPosition, final fl } @Override - public ByteBuffer serialize(final ByteOrder byteOrder) { - final ByteBuffer buffer = ByteBuffer.allocate(Float.BYTES * data.length); - buffer.order(byteOrder).asFloatBuffer().put(data); - return buffer; + public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { + final ByteBuffer serialized = ByteBuffer.wrap(readData.allBytes()); + serialized.order(byteOrder).asFloatBuffer().get(data); } @Override - public void deserialize(final ByteBuffer serialized) { - serialized.asFloatBuffer().get(data); + public ReadData writeData(final ByteOrder byteOrder) { + final ByteBuffer serialized = ByteBuffer.allocate(Float.BYTES * data.length); + serialized.order(byteOrder).asFloatBuffer().put(data); + return ReadData.from(serialized); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index a7175bb1b..590d2a17b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -25,8 +25,10 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import org.janelia.saalfeldlab.n5.readdata.ReadData; public class IntArrayDataBlock extends AbstractDataBlock { @@ -36,15 +38,16 @@ public IntArrayDataBlock(final int[] size, final long[] gridPosition, final int[ } @Override - public ByteBuffer serialize(final ByteOrder byteOrder) { - final ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES * data.length); - buffer.order(byteOrder).asIntBuffer().put(data); - return buffer; + public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { + final ByteBuffer serialized = ByteBuffer.wrap(readData.allBytes()); + serialized.order(byteOrder).asIntBuffer().get(data); } @Override - public void deserialize(final ByteBuffer serialized) { - serialized.asIntBuffer().get(data); + public ReadData writeData(final ByteOrder byteOrder) { + final ByteBuffer serialized = ByteBuffer.allocate(Integer.BYTES * data.length); + serialized.order(byteOrder).asIntBuffer().put(data); + return ReadData.from(serialized); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index e974c2bce..d6829ffd5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -25,8 +25,10 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import org.janelia.saalfeldlab.n5.readdata.ReadData; public class LongArrayDataBlock extends AbstractDataBlock { @@ -36,15 +38,16 @@ public LongArrayDataBlock(final int[] size, final long[] gridPosition, final lon } @Override - public ByteBuffer serialize(final ByteOrder byteOrder) { - final ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * data.length); - buffer.order(byteOrder).asLongBuffer().put(data); - return buffer; + public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { + final ByteBuffer serialized = ByteBuffer.wrap(readData.allBytes()); + serialized.order(byteOrder).asLongBuffer().get(data); } @Override - public void deserialize(final ByteBuffer serialized) { - serialized.asLongBuffer().get(data); + public ReadData writeData(final ByteOrder byteOrder) { + final ByteBuffer serialized = ByteBuffer.allocate(Long.BYTES * data.length); + serialized.order(byteOrder).asLongBuffer().put(data); + return ReadData.from(serialized); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index cbbd401ba..9d8b10a9f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -43,6 +43,7 @@ import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.janelia.saalfeldlab.n5.readdata.ReadData; /** * A simple structured container for hierarchies of chunked @@ -323,9 +324,9 @@ default T readSerializedBlock( if (block == null) return null; - final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.serialize(ByteOrder.BIG_ENDIAN).array()); - try (ObjectInputStream in = new ObjectInputStream(byteArrayInputStream)) { - return (T)in.readObject(); + final ReadData serialized = block.writeData(ByteOrder.BIG_ENDIAN); + try (ObjectInputStream in = new ObjectInputStream(serialized.inputStream())) { + return (T) in.readObject(); } catch (final IOException | UncheckedIOException e) { throw new N5Exception.N5IOException(e); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 08e3f3b8f..bdebfa04f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -25,8 +25,10 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import org.janelia.saalfeldlab.n5.readdata.ReadData; public class ShortArrayDataBlock extends AbstractDataBlock { @@ -36,15 +38,16 @@ public ShortArrayDataBlock(final int[] size, final long[] gridPosition, final sh } @Override - public ByteBuffer serialize(final ByteOrder byteOrder) { - final ByteBuffer buffer = ByteBuffer.allocate(Short.BYTES * data.length); - buffer.order(byteOrder).asShortBuffer().put(data); - return buffer; + public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { + final ByteBuffer serialized = ByteBuffer.wrap(readData.allBytes()); + serialized.order(byteOrder).asShortBuffer().get(data); } @Override - public void deserialize(final ByteBuffer serialized) { - serialized.asShortBuffer().get(data); + public ReadData writeData(final ByteOrder byteOrder) { + final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * data.length); + serialized.order(byteOrder).asShortBuffer().put(data); + return ReadData.from(serialized); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 979a9ffce..d7973012b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -25,10 +25,11 @@ */ package org.janelia.saalfeldlab.n5; -import java.nio.ByteBuffer; +import java.io.IOException; import java.nio.ByteOrder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import org.janelia.saalfeldlab.n5.readdata.ReadData; public class StringDataBlock extends AbstractDataBlock { @@ -43,24 +44,28 @@ public StringDataBlock(final int[] size, final long[] gridPosition, final String } @Override - public ByteBuffer serialize(final ByteOrder byteOrder) { - final String flattenedArray = String.join(NULLCHAR, actualData) + NULLCHAR; - return ByteBuffer.wrap(flattenedArray.getBytes(ENCODING)); + public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { + serializedData = readData.allBytes(); + final String rawChars = new String(serializedData, ENCODING); + actualData = rawChars.split(NULLCHAR); + } + + private byte[] serialize() { + if (serializedData == null) { + final String flattenedArray = String.join(NULLCHAR, actualData) + NULLCHAR; + serializedData = flattenedArray.getBytes(ENCODING); + } + return serializedData; } @Override - public void deserialize(final ByteBuffer serialized) { - serializedData = serialized.array(); - final String rawChars = new String(serializedData, ENCODING); - actualData = rawChars.split(NULLCHAR); + public ReadData writeData(final ByteOrder byteOrder) { + return ReadData.from(serialize()); } @Override public int getNumElements() { - if (serializedData == null) { - serializedData = serialize(ByteOrder.BIG_ENDIAN).array(); - } - return serializedData.length; + return serialize().length; } @Override From 3bf8fe830747aa4f96e53ae50de374b25649133d Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 4 Feb 2025 16:02:20 -0500 Subject: [PATCH 158/423] refactor: large codecs/shards implementation refactor --- README.md | 5 +- .../janelia/saalfeldlab/n5/Compression.java | 18 +- .../saalfeldlab/n5/DatasetAttributes.java | 238 ++++++++++-------- .../saalfeldlab/n5/DefaultBlockWriter.java | 6 +- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 103 +++----- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 41 +-- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 4 +- .../org/janelia/saalfeldlab/n5/N5Reader.java | 30 ++- .../org/janelia/saalfeldlab/n5/N5Writer.java | 30 +-- .../n5/ShardedDatasetAttributes.java | 6 +- .../janelia/saalfeldlab/n5/codec/Codec.java | 98 ++++++-- .../FixedLengthConvertedInputStream.java | 2 +- .../saalfeldlab/n5/codec/N5BlockCodec.java | 189 ++++++++------ .../codec/{BytesCodec.java => RawBytes.java} | 23 +- .../saalfeldlab/n5/shard/AbstractShard.java | 6 +- .../saalfeldlab/n5/shard/InMemoryShard.java | 67 ++--- .../janelia/saalfeldlab/n5/shard/Shard.java | 2 +- .../saalfeldlab/n5/shard/ShardIndex.java | 52 ++-- .../saalfeldlab/n5/shard/ShardParameters.java | 31 +-- .../saalfeldlab/n5/shard/ShardingCodec.java | 31 ++- .../saalfeldlab/n5/shard/VirtualShard.java | 87 ++++--- .../saalfeldlab/n5/AbstractN5Test.java | 10 +- .../n5/FileSystemKeyValueAccessTest.java | 10 - .../org/janelia/saalfeldlab/n5/N5FSTest.java | 1 - .../saalfeldlab/n5/codec/BytesTests.java | 7 +- .../codec/FixedConvertedOutputStreamTest.java | 69 +++-- .../saalfeldlab/n5/demo/BlockIterators.java | 6 +- .../saalfeldlab/n5/shard/ShardIndexTest.java | 17 +- .../n5/shard/ShardPropertiesTests.java | 74 +++--- .../saalfeldlab/n5/shard/ShardTest.java | 127 +++++----- 30 files changed, 748 insertions(+), 642 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/codec/{BytesCodec.java => RawBytes.java} (89%) diff --git a/README.md b/README.md index bc97e9b8b..59b25df2a 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ N5 group is not a single file but simply a directory on the file system. Meta-d 1. All directories of the file system are N5 groups. 2. A JSON file `attributes.json` in a directory contains arbitrary attributes. A group without attributes may not have an `attributes.json` file. -3. The version of this specification is 4.0.0 and is stored in the "n5" attribute of the root group "/". +3. The version of this specification is 1.0.0 and is stored in the "n5" attribute of the root group "/". 4. A dataset is a group with the mandatory attributes: * dimensions (e.g. [100, 200, 300]), * blockSize (e.g. [64, 64, 64]), @@ -38,7 +38,7 @@ N5 group is not a single file but simply a directory on the file system. Meta-d * xz with parameters * preset (integer, default 6). - Custom compression schemes with arbitrary parameters can be added using [compression annotations](#extensible-compression-schemes), e.g. [N5 Blosc](https://github.com/saalfeldlab/n5-blosc) and [N5 ZStandard](https://github.com/JaneliaSciComp/n5-zstandard/). + Custom compression schemes with arbitrary parameters can be added using [compression annotations](#extensible-compression-schemes), e.g. [N5 Blosc](https://github.com/saalfeldlab/n5-blosc). 5. Chunks are stored in a directory hierarchy that enumerates their positive integer position in the chunk grid (e.g. `0/4/1/7` for chunk grid position p=(0, 4, 1, 7)). 6. Datasets are sparse, i.e. there is no guarantee that all chunks of a dataset exist. 7. Chunks cannot be larger than 2GB (231Bytes). @@ -134,3 +134,4 @@ Custom compression schemes can be implemented using the annotation discovery mec HDF5 is a great format that provides a wealth of conveniences that I do not want to miss. It's inefficiency for parallel writing, however, limit its applicability for handling of very large n-dimensional data. N5 uses the native filesystem of the target platform and JSON files to specify basic and custom meta-data as attributes. It aims at preserving the convenience of HDF5 where possible but doesn't try too hard to be a full replacement. +Please do not take this project too seriously, we will see where it will get us and report back when more data is available. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index ac0c49b55..2e8b9cdfd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -39,6 +39,10 @@ import org.scijava.annotations.Indexable; /** + * Deprecated: {@link Compression}s are no longer a special case. + *
    + * Use {@link Codec.BytesCodec} for implementing compressors + *

    * Compression scheme interface. * * @author Stephan Saalfeld @@ -53,7 +57,7 @@ public interface Compression extends Serializable, Codec.BytesCodec { @Inherited @Target(ElementType.TYPE) @Indexable - public static @interface CompressionType { + @interface CompressionType { String value(); } @@ -65,10 +69,10 @@ public interface Compression extends Serializable, Codec.BytesCodec { @Retention(RetentionPolicy.RUNTIME) @Inherited @Target(ElementType.FIELD) - public static @interface CompressionParameter {} + @interface CompressionParameter {} @Override - public default String getType() { + default String getType() { final CompressionType compressionType = getClass().getAnnotation(CompressionType.class); if (compressionType == null) @@ -78,9 +82,9 @@ public default String getType() { } - public BlockReader getReader(); + BlockReader getReader(); - public BlockWriter getWriter(); + BlockWriter getWriter(); /** * Decode an {@link InputStream}. @@ -90,7 +94,7 @@ public default String getType() { * @return the decoded input stream */ @Override - public InputStream decode(InputStream in) throws IOException; + InputStream decode(InputStream in) throws IOException; /** * Encode an {@link OutputStream}. @@ -100,6 +104,6 @@ public default String getType() { * @return the encoded output stream */ @Override - public OutputStream encode(OutputStream out) throws IOException; + OutputStream encode(OutputStream out) throws IOException; } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index eca190d96..d2cfa4a51 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -4,12 +4,13 @@ import java.lang.reflect.Type; import java.util.Arrays; import java.util.HashMap; +import java.util.stream.Stream; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec; +import org.janelia.saalfeldlab.n5.shard.ShardParameters; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; @@ -19,6 +20,9 @@ import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; + /** * Mandatory dataset attributes: * @@ -26,18 +30,13 @@ *
  • long[] : dimensions
    • *
  • int[] : blockSize
    • *
  • {@link DataType} : dataType
    • - *
  • {@link Compression} : compression
    • - * - * - * Optional dataset attributes: - *
      - *
    1. {@link Codec}[] : codecs
      2. + *
    2. {@link Codec}... : encode/decode routines
      3. *
    * * @author Stephan Saalfeld - * */ -public class DatasetAttributes implements BlockParameters, Serializable { +//TODO Caleb: try to delete ShardParameters? +public class DatasetAttributes implements BlockParameters, ShardParameters, Serializable { private static final long serialVersionUID = -4521467080388947553L; @@ -58,60 +57,91 @@ public class DatasetAttributes implements BlockParameters, Serializable { private final long[] dimensions; private final int[] blockSize; private final DataType dataType; - private final Compression compression; private final ArrayCodec arrayCodec; private final BytesCodec[] byteCodecs; - + @Nullable private final int[] shardSize; + + /** + * Constructs a DatasetAttributes instance with specified dimensions, block size, data type, + * and array of codecs. + * + * @param dimensions the dimensions of the dataset + * @param blockSize the size of the blocks in the dataset + * @param dataType the data type of the dataset + * @param codecs the codecs used encode/decode the data + */ public DatasetAttributes( final long[] dimensions, + @Nullable final int[] shardSize, final int[] blockSize, final DataType dataType, - final Compression compression, - final Codec[] codecs) { + final Codec... codecs) { this.dimensions = dimensions; + this.shardSize = shardSize; this.blockSize = blockSize; this.dataType = dataType; - if (codecs == null && !(compression instanceof RawCompression)) { + if (codecs == null || codecs.length == 0) { byteCodecs = new BytesCodec[]{}; arrayCodec = new N5BlockCodec(); - } else if (codecs == null || codecs.length == 0) { - byteCodecs = new BytesCodec[]{}; + } else if (codecs.length == 1 && codecs[0] instanceof Compression) { + final BytesCodec compression = (BytesCodec)codecs[0]; + byteCodecs = compression instanceof RawCompression ? new BytesCodec[]{} : new BytesCodec[]{compression}; arrayCodec = new N5BlockCodec(); } else { if (!(codecs[0] instanceof ArrayCodec)) - throw new N5Exception("Expected first element of codecs to be ArrayCodec, but was: " + codecs[0]); + throw new N5Exception("Expected first element of codecs to be ArrayCodec, but was: " + codecs[0].getClass()); + + if (Arrays.stream(codecs).filter(c -> c instanceof ArrayCodec).count() > 1) + throw new N5Exception("Multiple ArrayCodecs found. Only one is allowed."); arrayCodec = (ArrayCodec)codecs[0]; - byteCodecs = new BytesCodec[codecs.length - 1]; - for (int i = 0; i < byteCodecs.length; i++) - byteCodecs[i] = (BytesCodec)codecs[i + 1]; + byteCodecs = Stream.of(codecs) + .skip(1) + .filter(c -> !(c instanceof RawCompression)) + .filter(c -> c instanceof BytesCodec) + .toArray(BytesCodec[]::new); } - //TODO Caleb: Do we want to do this? - this.compression = Arrays.stream(byteCodecs) - .filter(codec -> codec instanceof Compression) - .map(codec -> (Compression)codec) - .findFirst() - .orElse(compression == null ? new RawCompression() : compression); } + /** + * Constructs a DatasetAttributes instance with specified dimensions, block size, data type, + * and array of codecs. + * + * @param dimensions the dimensions of the dataset + * @param blockSize the size of the blocks in the dataset + * @param dataType the data type of the dataset + * @param codecs the codecs used encode/decode the data + */ public DatasetAttributes( final long[] dimensions, final int[] blockSize, final DataType dataType, - final Codec[] codecs) { - this(dimensions, blockSize, dataType, null, codecs); + final Codec... codecs) { + this( dimensions, null, blockSize, dataType, codecs ); } + /** + * Deprecated. {@link Compression} are {@link Codec}. Use {@code Code...} constructor instead + * Constructs a DatasetAttributes instance with specified dimensions, block size, data type, + * and compression scheme. This constructor is deprecated and redirects to another constructor + * with codec support. + * + * @param dimensions the dimensions of the dataset + * @param blockSize the size of the blocks in the dataset + * @param dataType the data type of the dataset + * @param compression the compression scheme used for storing the dataset + */ + @Deprecated public DatasetAttributes( final long[] dimensions, final int[] blockSize, final DataType dataType, final Compression compression) { - this(dimensions, blockSize, dataType, compression, null); + this(dimensions, blockSize, dataType, (Codec)compression); } @Override @@ -126,15 +156,33 @@ public int getNumDimensions() { return dimensions.length; } + @Override + @CheckForNull + public int[] getShardSize() { + + return shardSize; + } + @Override public int[] getBlockSize() { return blockSize; } + /** + * Deprecated. {@link Compression} is no longer a special case. prefer to reference {@link #getCodecs()} + * Will return {@link RawCompression} if no compression is otherwise provided, for legacy compatibility. + * + * @return compression Codec, if one was present + */ + @Deprecated public Compression getCompression() { - return compression; + return Arrays.stream(byteCodecs) + .filter(it -> it instanceof Compression) + .map(it -> (Compression)it) + .findFirst() + .orElse(new RawCompression()); } public DataType getDataType() { @@ -152,62 +200,22 @@ public BytesCodec[] getCodecs() { return byteCodecs; } + /** + * Deprecated in favor of {@link DatasetAttributesAdapter} for serialization + * + * @return serilizable properties of {@link DatasetAttributes} + */ + @Deprecated public HashMap asMap() { final HashMap map = new HashMap<>(); map.put(DIMENSIONS_KEY, dimensions); map.put(BLOCK_SIZE_KEY, blockSize); map.put(DATA_TYPE_KEY, dataType); - map.put(COMPRESSION_KEY, compression); - map.put(CODEC_KEY, concatenateCodecs()); // TODO : consider not adding to map when null + map.put(COMPRESSION_KEY, getCompression()); return map; } - static DatasetAttributes from( - final long[] dimensions, - final DataType dataType, - int[] blockSize, - Compression compression, - final String compressionVersion0Name) { - - return from(dimensions, dataType, blockSize, compression, compressionVersion0Name, null); - } - - static DatasetAttributes from( - final long[] dimensions, - final DataType dataType, - int[] blockSize, - Compression compression, - final String compressionVersion0Name, - Codec[] codecs) { - - if (blockSize == null) - blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - - /* version 0 */ - if (compression == null) { - compression = getCompressionVersion0(compressionVersion0Name); - } - - return new DatasetAttributes(dimensions, blockSize, dataType, compression, codecs); - } - - private static Compression getCompressionVersion0(final String compressionVersion0Name) { - - switch (compressionVersion0Name) { - case "raw": - return new RawCompression(); - case "gzip": - return new GzipCompression(); - case "bzip2": - return new Bzip2Compression(); - case "lz4": - return new Lz4Compression(); - case "xz": - return new XzCompression(); - } - return null; - } protected Codec[] concatenateCodecs() { @@ -227,50 +235,57 @@ public static DatasetAttributesAdapter getJsonAdapter() { return adapter; } + public static class InvalidN5DatasetException extends N5Exception { + + public InvalidN5DatasetException(String dataset, String reason, Throwable cause) { + + this(String.format("Invalid dataset %s: %s", dataset, reason), cause); + } + + public InvalidN5DatasetException(String message, Throwable cause) { + + super(message, cause); + } + } public static class DatasetAttributesAdapter implements JsonSerializer, JsonDeserializer { @Override public DatasetAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { if (json == null || !json.isJsonObject()) return null; final JsonObject obj = json.getAsJsonObject(); - if (!obj.has(DIMENSIONS_KEY) || !obj.has(BLOCK_SIZE_KEY) || !obj.has(DATA_TYPE_KEY) || !obj.has(COMPRESSION_KEY)) + final boolean validKeySet = obj.has(DIMENSIONS_KEY) + && obj.has(BLOCK_SIZE_KEY) + && obj.has(DATA_TYPE_KEY) + && (obj.has(CODEC_KEY) || obj.has(COMPRESSION_KEY) || obj.has(compressionTypeKey)); + + if (!validKeySet) return null; final long[] dimensions = context.deserialize(obj.get(DIMENSIONS_KEY), long[].class); final int[] blockSize = context.deserialize(obj.get(BLOCK_SIZE_KEY), int[].class); int[] shardSize = null; - if (obj.has(SHARD_SIZE_KEY)) { + if (obj.has(SHARD_SIZE_KEY)) shardSize = context.deserialize(obj.get(SHARD_SIZE_KEY), int[].class); - } final DataType dataType = context.deserialize(obj.get(DATA_TYPE_KEY), DataType.class); - Compression compression = null; - if (obj.has(COMPRESSION_KEY)) { - compression = CompressionAdapter.getJsonAdapter().deserialize(obj.get(COMPRESSION_KEY), Compression.class, context); - } else if (obj.has(compressionTypeKey)) { - compression = DatasetAttributes.getCompressionVersion0(obj.get(compressionTypeKey).getAsString()); - } - if (compression == null) - return null; final Codec[] codecs; if (obj.has(CODEC_KEY)) { codecs = context.deserialize(obj.get(CODEC_KEY), Codec[].class); - } else codecs = null; - - if (codecs != null && codecs.length == 1 && codecs[0] instanceof ShardingCodec) { - final ShardingCodec shardingCodec = (ShardingCodec)codecs[0]; - return new ShardedDatasetAttributes( - dimensions, - shardSize, - blockSize, - dataType, - shardingCodec - ); + } else if (obj.has(COMPRESSION_KEY)) { + final Compression compression = CompressionAdapter.getJsonAdapter().deserialize(obj.get(COMPRESSION_KEY), Compression.class, context); + final N5BlockCodec n5BlockCodec = dataType == DataType.UINT8 || dataType == DataType.INT8 ? new N5BlockCodec(null) : new N5BlockCodec(); + codecs = new Codec[]{compression, n5BlockCodec}; + } else if (obj.has(compressionTypeKey)) { + final Compression compression = getCompressionVersion0(obj.get(compressionTypeKey).getAsString()); + final N5BlockCodec n5BlockCodec = dataType == DataType.UINT8 || dataType == DataType.INT8 ? new N5BlockCodec(null) : new N5BlockCodec(); + codecs = new Codec[]{compression, n5BlockCodec}; + } else { + return null; } - return new DatasetAttributes(dimensions, blockSize, dataType, compression, codecs); + return new DatasetAttributes(dimensions, shardSize, blockSize, dataType, codecs); } @Override public JsonElement serialize(DatasetAttributes src, Type typeOfSrc, JsonSerializationContext context) { @@ -279,15 +294,32 @@ public static class DatasetAttributesAdapter implements JsonSerializer void writeBlock( final ArrayCodec arrayCodec = datasetAttributes.getArrayCodec(); final DataBlockOutputStream dataBlockOutput = arrayCodec.encode(datasetAttributes, dataBlock, out); - OutputStream stream = dataBlockOutput; - for (final BytesCodec codec : codecs) - stream = codec.encode(stream); + OutputStream stream = encode(dataBlockOutput, codecs); dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); stream.close(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index ba8eb7b4e..78e10811a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -25,23 +25,20 @@ */ package org.janelia.saalfeldlab.n5; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.VirtualShard; +import org.janelia.saalfeldlab.n5.util.Position; + import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.shard.Shard; -import org.janelia.saalfeldlab.n5.shard.ShardParameters; -import org.janelia.saalfeldlab.n5.shard.VirtualShard; -import org.janelia.saalfeldlab.n5.util.Position; - -import com.google.gson.Gson; -import com.google.gson.JsonElement; - /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON * attributes parsed with {@link Gson}. @@ -95,58 +92,46 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { } @SuppressWarnings({ "unchecked", "rawtypes" }) - @Override - default     Shard readShard(final String pathName, - final A datasetAttributes, long... shardGridPosition) { + default Shard readShard( + final String keyPath, + final DatasetAttributes datasetAttributes, + long... shardGridPosition) { - final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), shardGridPosition); - return new VirtualShard(datasetAttributes, shardGridPosition, getKeyValueAccess(), path); + final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(keyPath), shardGridPosition); + return new VirtualShard<>(datasetAttributes, shardGridPosition, getKeyValueAccess(), path); } @Override - default DataBlock readBlock( + default DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { - if (datasetAttributes instanceof ShardedDatasetAttributes) { - final ShardedDatasetAttributes shardedAttrs = (ShardedDatasetAttributes) datasetAttributes; - final long[] shardPosition = shardedAttrs.getShardPositionForBlock(gridPosition); - final Shard shard = readShard(pathName, shardedAttrs, shardPosition); - return shard.getBlock(gridPosition); - } - - final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), gridPosition); + final long[] keyPos = datasetAttributes.getArrayCodec().getPositionForBlock(datasetAttributes, gridPosition); + final String keyPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), keyPos); - try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(path)) { - return DefaultBlockReader.readBlock(lockedChannel.newInputStream(), datasetAttributes, gridPosition); - } catch (final N5Exception.N5NoSuchKeyException e) { - return null; - } catch (final IOException | UncheckedIOException e) { - throw new N5IOException( - "Failed to read block " + Arrays.toString(gridPosition) + " from dataset " + path, - e); - } + return datasetAttributes.getArrayCodec().readBlock( + getKeyValueAccess(), + keyPath, + datasetAttributes, + gridPosition + ); } @Override - default List> readBlocks( + default List> readBlocks( final String pathName, final DatasetAttributes datasetAttributes, final List blockPositions) throws N5Exception { // TODO which interface should have this implementation? - if (datasetAttributes instanceof ShardParameters) { - - final ShardParameters shardAttributes = (ShardParameters)datasetAttributes; - + if (datasetAttributes.getShardSize() != null) { /* Group by shard position */ - final Map> shardBlockMap = shardAttributes.groupBlockPositions(blockPositions); - final ArrayList> blocks = new ArrayList<>(); + final Map> shardBlockMap = datasetAttributes.groupBlockPositions(blockPositions); + final ArrayList> blocks = new ArrayList<>(); for( Entry> e : shardBlockMap.entrySet()) { - final Shard shard = readShard(pathName, (DatasetAttributes & ShardParameters) shardAttributes, - e.getKey().get()); + final Shard shard = readShard(pathName, datasetAttributes, e.getKey().get()); for (final long[] blkPosition : e.getValue()) { blocks.add(shard.getBlock(blkPosition)); @@ -154,8 +139,8 @@ default List> readBlocks( } return blocks; - } else - return GsonN5Reader.super.readBlocks(pathName, datasetAttributes, blockPositions); + } + return GsonN5Reader.super.readBlocks(pathName, datasetAttributes, blockPositions); } @Override @@ -171,6 +156,9 @@ default String[] list(final String pathName) throws N5Exception { /** * Constructs the path for a data block in a dataset at a given grid * position. + *
    + * If the gridPosition passed in refers to shard position + * in a sharded dataset, this will return the path to the shard key *

    * The returned path is * @@ -198,33 +186,6 @@ default String absoluteDataBlockPath( return getKeyValueAccess().compose(getURI(), components); } - /** - * Constructs the path for a shard in a dataset at a given grid position. - *

    - * The returned path is - * - * 

    -	 * $basePath/datasetPathName/$shardPosition[0]/$shardPosition[1]/.../$shardPosition[n]
-	 *
    - *

    - * This is the file into which the shard will be stored. - * - * @param normalPath normalized dataset path - * @param shardGridPosition to the target shard - * @return the absolute path to the shard at shardGridPosition - */ - default String absoluteShardPath( - final String normalPath, - final long... shardGridPosition) { - - final String[] components = new String[shardGridPosition.length + 1]; - components[0] = normalPath; - int i = 0; - for (final long p : shardGridPosition) - components[++i] = Long.toString(p); - - return getKeyValueAccess().compose(getURI(), components); - } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index ac6f733ab..36f5ef1db 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -82,7 +82,7 @@ default void createGroup(final String path) throws N5Exception { try { getKeyValueAccess().createDirectories(absoluteGroupPath(normalPath)); } catch (final IOException | UncheckedIOException e) { - throw new N5Exception.N5IOException("Failed to create group " + path, e); + throw new N5IOException("Failed to create group " + path, e); } } @@ -106,7 +106,7 @@ default void writeAttributes( try (final LockedChannel lock = getKeyValueAccess().lockForWriting(absoluteAttributesPath(normalGroupPath))) { GsonUtils.writeAttributes(lock.newWriter(), attributes, getGson()); } catch (final IOException | UncheckedIOException e) { - throw new N5Exception.N5IOException("Failed to write attributes into " + normalGroupPath, e); + throw new N5IOException("Failed to write attributes into " + normalGroupPath, e); } } @@ -223,26 +223,23 @@ default boolean removeAttributes(final String pathName, final List attri final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { - if (datasetAttributes instanceof ShardParameters) { - - final ShardParameters shardAttributes = (ShardParameters)datasetAttributes; + if (datasetAttributes.getShardSize() != null) { /* Group blocks by shard index */ - final Map>> shardBlockMap = shardAttributes.groupBlocks( + final Map>> shardBlockMap = datasetAttributes.groupBlocks( Arrays.stream(dataBlocks).collect(Collectors.toList())); for( final Entry>> e : shardBlockMap.entrySet()) { final long[] shardPosition = e.getKey().get(); - @SuppressWarnings("unchecked") - final Shard currentShard = (Shard) readShard(datasetPath, (DatasetAttributes & ShardParameters)shardAttributes, + final Shard currentShard = readShard(datasetPath, datasetAttributes, shardPosition); final InMemoryShard newShard = InMemoryShard.fromShard(currentShard); for( DataBlock blk : e.getValue()) newShard.addBlock(blk); - writeShard(datasetPath, (DatasetAttributes & ShardParameters)shardAttributes, newShard); + writeShard(datasetPath, datasetAttributes, newShard); } } else { @@ -256,26 +253,14 @@ default void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception { - /* Delegate to shard for writing block? How to know what type of shard? */ - if (datasetAttributes instanceof ShardParameters) { - ShardParameters shardDatasetAttrs = (ShardParameters)datasetAttributes; - final long[] shardPos = shardDatasetAttrs.getShardPositionForBlock(dataBlock.getGridPosition()); - final String shardPath = absoluteShardPath(N5URI.normalizeGroupPath(path), shardPos); - final VirtualShard shard = new VirtualShard<>((DatasetAttributes & ShardParameters)shardDatasetAttrs, shardPos, getKeyValueAccess(), shardPath); - shard.writeBlock(dataBlock); - return; - } + final long[] keyPos = datasetAttributes.getArrayCodec().getPositionForBlock(datasetAttributes, dataBlock); + final String keyPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), keyPos); - final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition()); - try (final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath)) { - try (final OutputStream out = lock.newOutputStream()) { - DefaultBlockWriter.writeBlock(out, datasetAttributes, dataBlock); - } - } catch (final IOException | UncheckedIOException e) { - throw new N5IOException( - "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, - e); - } + datasetAttributes.getArrayCodec().writeBlock( + getKeyValueAccess(), + keyPath, + datasetAttributes, + dataBlock); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 157c9bdd0..03741476e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -35,7 +35,7 @@ import java.util.Map; import java.util.regex.Matcher; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.codec.Codec; import com.google.gson.Gson; @@ -61,7 +61,7 @@ static Gson registerGson(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, DatasetAttributes.getJsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); - gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, BytesCodec.byteOrderAdapter); + gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBytes.byteOrderAdapter); gsonBuilder.registerTypeHierarchyAdapter(ShardingCodec.IndexLocation.class, ShardingCodec.indexLocationAdapter); gsonBuilder.disableHtmlEscaping(); return gsonBuilder.create(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 04c9d8892..20a04b330 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -25,6 +25,8 @@ */ package org.janelia.saalfeldlab.n5; +import org.janelia.saalfeldlab.n5.shard.Shard; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.ObjectInputStream; @@ -43,9 +45,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.janelia.saalfeldlab.n5.shard.Shard; -import org.janelia.saalfeldlab.n5.shard.ShardParameters; - /** * A simple structured container for hierarchies of chunked * n-dimensional datasets and attributes. @@ -55,7 +54,7 @@ */ public interface N5Reader extends AutoCloseable { - public static class Version { + class Version { private final int major; private final int minor; @@ -192,12 +191,12 @@ public boolean isCompatible(final Version version) { /** * SemVer version of this N5 spec. */ - public static final Version VERSION = new Version(4, 0, 0); + Version VERSION = new Version(4, 0, 0); /** * Version attribute key. */ - public static final String VERSION_KEY = "n5"; + String VERSION_KEY = "n5"; /** * Get the SemVer version of this container as specified in the 'version' @@ -223,7 +222,7 @@ default Version getVersion() throws N5Exception { * @return the base path URI */ // TODO: should this throw URISyntaxException or can we assume that this is - // never possible if we were able to instantiate this N5Reader? + // never possible if we were able to instantiate this N5Reader? URI getURI(); /** @@ -291,7 +290,7 @@ T getAttribute( * @throws N5Exception * the exception */ - DataBlock readBlock( + DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception; @@ -299,14 +298,13 @@ DataBlock readBlock( /** * Reads the {@link Shard} at the corresponding grid position. * - * @param - * @param datasetPath - * @param datasetAttributes - * @param shardGridPosition + * @param the data access type for the blocks in the shard + * @param datasetPath to read the shard from + * @param datasetAttributes for the shard + * @param shardGridPosition of the shard we are reading * @return the shard */ - public Shard readShard(final String datasetPath, - final A datasetAttributes, long... shardGridPosition); + Shard readShard(final String datasetPath, final DatasetAttributes datasetAttributes, long... shardGridPosition); /** * Reads multiple {@link DataBlock}s. @@ -324,12 +322,12 @@ public Shard readShard(final * @throws N5Exception * the exception */ - default List> readBlocks( + default List> readBlocks( final String pathName, final DatasetAttributes datasetAttributes, final List gridPositions) throws N5Exception { - final ArrayList> blocks = new ArrayList<>(); + final ArrayList> blocks = new ArrayList<>(); for( final long[] p : gridPositions ) blocks.add(readBlock(pathName, datasetAttributes, p)); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 74df1da32..2471135cb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -30,15 +30,15 @@ import java.io.ObjectOutputStream; import java.io.Serializable; import java.io.UncheckedIOException; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.shard.Shard; import org.janelia.saalfeldlab.n5.shard.ShardParameters; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; /** * A simple structured container API for hierarchies of chunked @@ -212,18 +212,6 @@ default void createDataset( setDatasetAttributes(normalPath, datasetAttributes); } - default void createDataset( - final String datasetPath, - final long[] dimensions, - final int[] shardSize, - final int[] blockSize, - final DataType dataType, - final Compression compression) throws N5Exception { - - final Codec[] codecs = new Codec[]{new ShardingCodec(blockSize, null, null, IndexLocation.END)}; - - createDataset(datasetPath, new DatasetAttributes(dimensions, shardSize, dataType, compression, codecs)); - } /** * Creates a dataset. This does not create any data but the path and @@ -233,8 +221,7 @@ default void createDataset( * @param dimensions the dataset dimensions * @param blockSize the block size * @param dataType the data type - * @param compression the compression - * @param codecs optional codecs (may be null) + * @param codecs codecs to encode/decode with * @throws N5Exception the exception */ default void createDataset( @@ -242,13 +229,15 @@ default void createDataset( final long[] dimensions, final int[] blockSize, final DataType dataType, - final Compression compression, - final Codec[] codecs) throws N5Exception { + final Codec... codecs) throws N5Exception { - createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression, codecs)); + createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, codecs)); } /** + * DEPRECATED. {@link Compression}s are {@link Codec}s. + * Use {@link #createDataset(String, long[], int[], DataType, Codec...)} + *

    * Creates a dataset. This does not create any data but the path and * mandatory attributes only. * @@ -259,6 +248,7 @@ default void createDataset( * @param compression the compression * @throws N5Exception the exception */ + @Deprecated default void createDataset( final String datasetPath, final long[] dimensions, @@ -266,7 +256,7 @@ default void createDataset( final DataType dataType, final Compression compression) throws N5Exception { - createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression, null)); + createDataset(datasetPath, dimensions, blockSize, dataType, new N5BlockCodec(), compression); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java index 04a8a0371..335c252ed 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java @@ -6,11 +6,11 @@ import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; -import org.janelia.saalfeldlab.n5.shard.ShardIndex; import org.janelia.saalfeldlab.n5.shard.ShardParameters; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; +@Deprecated public class ShardedDatasetAttributes extends DatasetAttributes implements ShardParameters { private static final long serialVersionUID = -4559068841006651814L; @@ -28,7 +28,8 @@ public ShardedDatasetAttributes ( final DeterministicSizeCodec[] indexCodecs, final IndexLocation indexLocation ) { - super(dimensions, blockSize, dataType, null, blocksCodecs); + //TODO Caleb: Can we just let the super codecs() return this ShardCodec? + super(dimensions, blockSize, dataType, blocksCodecs); if (!validateShardBlockSize(shardSize, blockSize)) { throw new N5Exception(String.format("Invalid shard %s / block size %s", @@ -96,7 +97,6 @@ protected Codec[] concatenateCodecs() { return new Codec[] { shardingCodec }; } - @Override public IndexLocation getIndexLocation() { return getShardingCodec().getIndexLocation(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index 209d169b6..ed5442571 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -6,24 +6,29 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.Serializable; +import java.io.UncheckedIOException; +import java.util.Arrays; import org.apache.commons.io.input.ProxyInputStream; import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.serialization.NameConfig; /** * Interface representing a filter can encode a {@link OutputStream}s when writing data, and decode * the {@link InputStream}s when reading data. - * + *

    * Modeled after Filters in * Zarr. */ @NameConfig.Prefix("codec") public interface Codec extends Serializable { - public static OutputStream encode(OutputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { + static OutputStream encode(OutputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { OutputStream stream = out; for (final BytesCodec codec : bytesCodecs) stream = codec.encode(stream); @@ -31,7 +36,7 @@ public static OutputStream encode(OutputStream out, Codec.BytesCodec... bytesCod return stream; } - public static InputStream decode(InputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { + static InputStream decode(InputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { InputStream stream = out; for (final BytesCodec codec : bytesCodecs) stream = codec.decode(stream); @@ -39,44 +44,55 @@ public static InputStream decode(InputStream out, Codec.BytesCodec... bytesCodec return stream; } - public interface BytesCodec extends Codec { + interface BytesCodec extends Codec { /** * Decode an {@link InputStream}. * - * @param in - * input stream + * @param in input stream * @return the decoded input stream */ - public InputStream decode(final InputStream in) throws IOException; + InputStream decode(final InputStream in) throws IOException; /** * Encode an {@link OutputStream}. * - * @param out - * the output stream + * @param out the output stream * @return the encoded output stream */ - public OutputStream encode(final OutputStream out) throws IOException; + OutputStream encode(final OutputStream out) throws IOException; } interface ArrayCodec extends DeterministicSizeCodec { + default long[] getPositionForBlock(final DatasetAttributes attributes, final DataBlock datablock) { + + return datablock.getGridPosition(); + } + + default long[] getPositionForBlock(final DatasetAttributes attributes, final long... blockPosition) { + + return blockPosition; + } /** * Decode an {@link InputStream}. * - * @param in - * input stream + * @param in input stream * @return the DataBlock corresponding to the input stream */ - public DataBlockInputStream decode(final DatasetAttributes attributes, final long[] gridPosition, final InputStream in) throws IOException; + DataBlockInputStream decode( + final DatasetAttributes attributes, + final long[] gridPosition, + final InputStream in) throws IOException; /** * Encode a {@link DataBlock}. * * @param datablock the datablock to encode */ - public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock datablock, + DataBlockOutputStream encode( + final DatasetAttributes attributes, + final DataBlock datablock, final OutputStream out) throws IOException; @Override default long encodedSize(long size) { @@ -88,9 +104,55 @@ public DataBlockOutputStream encode(final DatasetAttributes attributes, final Da return size; } + default void writeBlock( + final KeyValueAccess kva, + final String keyPath, + final DatasetAttributes datasetAttributes, + final DataBlock dataBlock) { + + try (final LockedChannel lock = kva.lockForWriting(keyPath)) { + try (final OutputStream out = lock.newOutputStream()) { + final DataBlockOutputStream dataBlockOutput = encode(datasetAttributes, dataBlock, out); + try (final OutputStream stream = Codec.encode(dataBlockOutput, datasetAttributes.getCodecs())) { + dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); + } + } + } catch (final IOException | UncheckedIOException e) { + final String msg = "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + keyPath; + throw new N5Exception.N5IOException( msg, e); + } + } + + default DataBlock readBlock( + final KeyValueAccess kva, + final String keyPath, + final DatasetAttributes datasetAttributes, + final long[] gridPosition) { + + try (final LockedChannel lockedChannel = kva.lockForReading(keyPath)) { + try(final InputStream in = lockedChannel.newInputStream()) { + + final BytesCodec[] codecs = datasetAttributes.getCodecs(); + final ArrayCodec arrayCodec = datasetAttributes.getArrayCodec(); + final DataBlockInputStream dataBlockStream = arrayCodec.decode(datasetAttributes, gridPosition, in); + InputStream stream = Codec.decode(dataBlockStream, codecs); + + final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); + dataBlock.readData(dataBlockStream.getDataInput(stream)); + stream.close(); + + return dataBlock; + } + } catch (final N5Exception.N5NoSuchKeyException e) { + return null; + } catch (final IOException | UncheckedIOException e) { + final String msg = "Failed to read block " + Arrays.toString(gridPosition) + " from dataset " + keyPath; + throw new N5Exception.N5IOException( msg, e); + } + } } - public abstract class DataBlockInputStream extends ProxyInputStream { + abstract class DataBlockInputStream extends ProxyInputStream { protected DataBlockInputStream(InputStream in) { @@ -98,12 +160,12 @@ protected DataBlockInputStream(InputStream in) { super(in); } - public abstract DataBlock allocateDataBlock() throws IOException; + public abstract DataBlock allocateDataBlock() throws IOException; public abstract DataInput getDataInput(final InputStream inputStream); } - public abstract class DataBlockOutputStream extends ProxyOutputStream { + abstract class DataBlockOutputStream extends ProxyOutputStream { protected DataBlockOutputStream(final OutputStream out) { @@ -113,6 +175,6 @@ protected DataBlockOutputStream(final OutputStream out) { public abstract DataOutput getDataOutput(final OutputStream outputStream); } - public String getType(); + String getType(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java index f9d65a87c..78d6313af 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java @@ -5,7 +5,7 @@ import java.nio.ByteBuffer; import java.util.function.BiConsumer; -/* +/** * An {@link InputStream} that converts between two fixed-length types. */ public class FixedLengthConvertedInputStream extends InputStream { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java index 7232d9aca..6b83f4666 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java @@ -17,13 +17,20 @@ import com.google.common.io.LittleEndianDataInputStream; import com.google.common.io.LittleEndianDataOutputStream; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; + @NameConfig.Name(value = N5BlockCodec.TYPE) public class N5BlockCodec implements Codec.ArrayCodec { private static final long serialVersionUID = 3523505403978222360L; public static final String TYPE = "n5bytes"; + public static final int MODE_DEFAULT = 0; + public static final int MODE_VARLENGTH = 1; + public static final int MODE_OBJECT = 2; + @Nullable @NameConfig.Parameter(value = "endian", optional = true) protected final ByteOrder byteOrder; @@ -32,18 +39,103 @@ public N5BlockCodec() { this(ByteOrder.BIG_ENDIAN); } - public N5BlockCodec(final ByteOrder byteOrder) { + public N5BlockCodec(@Nullable final ByteOrder byteOrder) { this.byteOrder = byteOrder; } + /** + * ByteOrder used to encode/decode this block of data.
    + * Will be `null` when {@link DatasetAttributes#getDataType()} refers to a single-byte type, + * + * @return the byte order for this codec + */ + @CheckForNull public ByteOrder getByteOrder() { return byteOrder; } @Override public DataBlockInputStream decode(final DatasetAttributes attributes, final long[] gridPosition, InputStream in) throws IOException { - return new DataBlockInputStream(in) { + return new N5DataBlockInputStream(in, attributes, gridPosition, byteOrder); + } + + + @Override + public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, + final OutputStream out) + throws IOException { + + return new N5DataBlockOutputStream(out, attributes, dataBlock, byteOrder); + } + + @Override + public String getType() { + + return TYPE; + } + + private static class N5DataBlockOutputStream extends DataBlockOutputStream { + + private final DatasetAttributes attributes; + private final DataBlock dataBlock; + private final ByteOrder byteOrder; + boolean start = true; + + + public N5DataBlockOutputStream(final OutputStream out, final DatasetAttributes attributes, final DataBlock dataBlock, ByteOrder byteOrder) { + super(out); + this.attributes = attributes; + this.dataBlock = dataBlock; + this.byteOrder = byteOrder; + } + + @Override + protected void beforeWrite(int n) throws IOException { + + if (start) { + writeHeader(); + start = false; + } + } + + private void writeHeader() throws IOException { + final DataOutput dos = getDataOutput(out); + + final int mode; + if (attributes.getDataType() == DataType.OBJECT || dataBlock.getSize() == null) + mode = MODE_OBJECT; + else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSize())) + mode = MODE_DEFAULT; + else + mode = MODE_VARLENGTH; + + dos.writeShort(mode); + + if (mode != MODE_OBJECT) { + dos.writeShort(attributes.getNumDimensions()); + for (final int size : dataBlock.getSize()) + dos.writeInt(size); + } + + if (mode != MODE_DEFAULT) + dos.writeInt(dataBlock.getNumElements()); + } + + @Override + public DataOutput getDataOutput(final OutputStream outputStream) { + + if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) + return new DataOutputStream(outputStream); + else + return new LittleEndianDataOutputStream(outputStream); + } + } + + private static class N5DataBlockInputStream extends DataBlockInputStream { + private final DatasetAttributes attributes; + private final long[] gridPosition; + private final ByteOrder byteOrder; private short mode = -1; private int[] blockSize = null; @@ -51,6 +143,12 @@ public ByteOrder getByteOrder() { private boolean start = true; + N5DataBlockInputStream(final InputStream in, final DatasetAttributes attributes, final long[] gridPosition, ByteOrder byteOrder) { + super(in); + this.attributes = attributes; + this.gridPosition = gridPosition; + this.byteOrder = byteOrder; + } @Override protected void beforeRead(int n) throws IOException { if (start) { @@ -60,32 +158,31 @@ public ByteOrder getByteOrder() { } @Override - public DataBlock allocateDataBlock() throws IOException { + public DataBlock allocateDataBlock() throws IOException { if (start) { readHeader(); start = false; } - if (mode != 2) { - return attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); - } else { - return attributes.getDataType().createDataBlock(null, gridPosition, numElements); + if (mode == MODE_OBJECT) { + return (DataBlock) attributes.getDataType().createDataBlock(null, gridPosition, numElements); } + return (DataBlock) attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); } private void readHeader() throws IOException { final DataInput dis = getDataInput(in); mode = dis.readShort(); - if (mode != 2) { - final int nDim = dis.readShort(); - blockSize = new int[nDim]; - for (int d = 0; d < nDim; ++d) - blockSize[d] = dis.readInt(); - if (mode == 0) { - numElements = DataBlock.getNumElements(blockSize); - } else { - numElements = dis.readInt(); - } + if (mode == MODE_OBJECT) { + numElements = dis.readInt(); + return; + } + final int nDim = dis.readShort(); + blockSize = new int[nDim]; + for (int d = 0; d < nDim; ++d) + blockSize[d] = dis.readInt(); + if (mode == MODE_DEFAULT) { + numElements = DataBlock.getNumElements(blockSize); } else { numElements = dis.readInt(); } @@ -99,65 +196,7 @@ public DataInput getDataInput(final InputStream inputStream) { else return new LittleEndianDataInputStream(inputStream); } - }; - } - - - @Override - public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, - final OutputStream out) - throws IOException { - - return new DataBlockOutputStream(out) { - - boolean start = true; - - @Override - protected void beforeWrite(int n) throws IOException { - - if (start) { - writeHeader(); - start = false; - } - } - - private void writeHeader() throws IOException { - final DataOutput dos = getDataOutput(out); - final int mode; - if (attributes.getDataType() == DataType.OBJECT || dataBlock.getSize() == null) - mode = 2; - else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSize())) - mode = 0; - else - mode = 1; - dos.writeShort(mode); - - if (mode != 2) { - dos.writeShort(attributes.getNumDimensions()); - for (final int size : dataBlock.getSize()) - dos.writeInt(size); - } - - if (mode != 0) - dos.writeInt(dataBlock.getNumElements()); - } - - @Override - public DataOutput getDataOutput(final OutputStream outputStream) { - - if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) - return new DataOutputStream(outputStream); - else - return new LittleEndianDataOutputStream(outputStream); - } - }; - } - - @Override - public String getType() { - - return TYPE; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java similarity index 89% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java index 66e1a8b21..bb3232e43 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java @@ -24,8 +24,10 @@ import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; -@NameConfig.Name(value = BytesCodec.TYPE) -public class BytesCodec implements Codec.ArrayCodec { +import javax.annotation.Nullable; + +@NameConfig.Name(value = RawBytes.TYPE) +public class RawBytes implements Codec.ArrayCodec { private static final long serialVersionUID = 3282569607795127005L; @@ -34,16 +36,17 @@ public class BytesCodec implements Codec.ArrayCodec { @NameConfig.Parameter(value = "endian", optional = true) protected final ByteOrder byteOrder; - public BytesCodec() { + public RawBytes() { this(ByteOrder.LITTLE_ENDIAN); } - public BytesCodec(final ByteOrder byteOrder) { + public RawBytes(final ByteOrder byteOrder) { this.byteOrder = byteOrder; } + @Nullable public ByteOrder getByteOrder() { return byteOrder; } @@ -55,15 +58,13 @@ public DataBlockInputStream decode(final DatasetAttributes attributes, final lon return new DataBlockInputStream(in) { private int[] blockSize = attributes.getBlockSize(); - private int numElements = Arrays.stream(blockSize).reduce(1, (x, y) -> { - return x * y; - }); + private int numElements = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y); @Override - protected void beforeRead(int n) throws IOException {} + protected void beforeRead(int n) {} @Override - public DataBlock allocateDataBlock() throws IOException { + public DataBlock allocateDataBlock() { return attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); } @@ -73,8 +74,8 @@ public DataInput getDataInput(final InputStream inputStream) { if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) return new DataInputStream(inputStream); - else - return new LittleEndianDataInputStream(inputStream); + + return new LittleEndianDataInputStream(inputStream); } }; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java index 2cdb392f2..4e2ac8319 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java @@ -11,7 +11,7 @@ public abstract class AbstractShard implements Shard { private final long[] gridPosition; - public AbstractShard(final A datasetAttributes, final long[] gridPosition, + public AbstractShard(final DatasetAttributes datasetAttributes, final long[] gridPosition, final ShardIndex index) { this.datasetAttributes = datasetAttributes; @@ -20,9 +20,9 @@ public AbstractShard(final A dat } @Override - public A getDatasetAttributes() { + public DatasetAttributes getDatasetAttributes() { - return (A)datasetAttributes; + return datasetAttributes; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index b36dc5aca..f7a01fc87 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -1,21 +1,9 @@ package org.janelia.saalfeldlab.n5.shard; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - import org.apache.commons.io.input.BoundedInputStream; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.io.output.CountingOutputStream; import org.apache.commons.io.output.ProxyOutputStream; -import org.checkerframework.checker.units.qual.A; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.DefaultBlockReader; @@ -26,6 +14,15 @@ import org.janelia.saalfeldlab.n5.util.GridIterator; import org.janelia.saalfeldlab.n5.util.Position; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + public class InMemoryShard extends AbstractShard { /* Map of a hash of the DataBlocks `gridPosition` to the block */ @@ -37,14 +34,15 @@ public class InMemoryShard extends AbstractShard { * Use morton- or c-ording instead of writing blocks out in the order they're added? * (later) */ - public InMemoryShard(final A datasetAttributes, final long[] shardPosition) { + public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] shardPosition) { this( datasetAttributes, shardPosition, null); indexBuilder = new ShardIndexBuilder(this); - indexBuilder.indexLocation(datasetAttributes.getIndexLocation()); + final IndexLocation indexLocation = ((ShardingCodec)datasetAttributes.getArrayCodec()).getIndexLocation(); + indexBuilder.indexLocation(indexLocation); } - public InMemoryShard(final A datasetAttributes, final long[] gridPosition, + public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] gridPosition, ShardIndex index) { super(datasetAttributes, gridPosition, index); @@ -137,8 +135,8 @@ public void write(final OutputStream out) throws IOException { writeShardStart(out, this); } - public static InMemoryShard readShard( - final KeyValueAccess kva, final String key, final long[] gridPosition, final A attributes) + public static InMemoryShard readShard( + final KeyValueAccess kva, final String key, final long[] gridPosition, final DatasetAttributes attributes) throws IOException { try (final LockedChannel lockedChannel = kva.lockForReading(key)) { @@ -152,8 +150,8 @@ public static InMemoryShard InMemoryShard readShard( - final InputStream inputStream, final long[] gridPosition, final A attributes) throws IOException { + public static InMemoryShard readShard( + final InputStream inputStream, final long[] gridPosition, final DatasetAttributes attributes) throws IOException { try (ByteArrayOutputStream result = new ByteArrayOutputStream()) { byte[] buffer = new byte[1024]; @@ -164,11 +162,11 @@ public static InMemoryShard InMemoryShard readShard( + public static InMemoryShard readShard( final byte[] data, - long[] shardPosition, final A attributes) throws IOException { + long[] shardPosition, final DatasetAttributes attributes) throws IOException { - final ShardIndex index = attributes.createIndex(); + final ShardIndex index = ((ShardingCodec)attributes.getArrayCodec()).createIndex(attributes); ShardIndex.read(data, index); final InMemoryShard shard = new InMemoryShard(attributes, shardPosition, index); @@ -224,17 +222,18 @@ public static InMemoryShard fromShard(Shard shard) { return inMemoryShard; } - protected static void writeShardEndStream( + protected static void writeShardEndStream( final OutputStream out, InMemoryShard shard ) throws IOException { - final A datasetAttributes = shard.getDatasetAttributes(); + final DatasetAttributes datasetAttributes = shard.getDatasetAttributes(); final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); indexBuilder.indexLocation(IndexLocation.END); - indexBuilder.setCodecs(datasetAttributes.getShardingCodec().getIndexCodecs()); + final ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); + indexBuilder.setCodecs(shardingCodec.getIndexCodecs()); - // Neccesary to stop `close()` when writing blocks from closing out base OutputStream + // Necessary to stop `close()` when writing blocks from closing out base OutputStream final ProxyOutputStream nop = new ProxyOutputStream(out) { @Override public void close() { //nop @@ -255,15 +254,15 @@ protected static void writeSh ShardIndex.write(indexBuilder.build(), out); } - protected static void writeShardEnd( + protected static void writeShardEnd( final OutputStream out, InMemoryShard shard ) throws IOException { - final A datasetAttributes = shard.getDatasetAttributes(); - final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); indexBuilder.indexLocation(IndexLocation.END); - indexBuilder.setCodecs(datasetAttributes.getShardingCodec().getIndexCodecs()); + final DatasetAttributes datasetAttributes = shard.getDatasetAttributes(); + final ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); + indexBuilder.setCodecs(shardingCodec.getIndexCodecs()); for (DataBlock block : shard.getBlocks()) { final ByteArrayOutputStream os = new ByteArrayOutputStream(); @@ -276,14 +275,16 @@ protected static void writeSh ShardIndex.write(indexBuilder.build(), out); } - protected static void writeShardStart( + protected static void writeShardStart( final OutputStream out, InMemoryShard shard ) throws IOException { - final A datasetAttributes = shard.getDatasetAttributes(); + final DatasetAttributes datasetAttributes = shard.getDatasetAttributes(); + final ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); + final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); indexBuilder.indexLocation(IndexLocation.START); - indexBuilder.setCodecs(datasetAttributes.getShardingCodec().getIndexCodecs()); + indexBuilder.setCodecs(shardingCodec.getIndexCodecs()); final List blockData = new ArrayList<>(shard.numBlocks()); for (DataBlock block : shard.getBlocks()) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 69ed415fd..2e1d7e7e2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -24,7 +24,7 @@ default int[] getBlockGridSize() { return getDatasetAttributes().getBlocksPerShard(); } - public A getDatasetAttributes(); + DatasetAttributes getDatasetAttributes(); /** * Returns the size of shards in pixel units. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 10433c101..f50a6d623 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -14,6 +14,7 @@ import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import java.io.ByteArrayInputStream; @@ -137,43 +138,50 @@ public long numBytes() { return totalNumBytes; } - public static ShardIndex read(byte[] data, final ShardIndex index) throws IOException { + public static boolean read(byte[] data, final ShardIndex index) { final IndexByteBounds byteBounds = byteBounds(index, data.length); final ByteArrayInputStream is = new ByteArrayInputStream(data); is.skip(byteBounds.start); + try { BoundedInputStream bIs = BoundedInputStream.builder() .setInputStream(is) .setMaxCount(byteBounds.size).get(); - return read(bIs, index); + read(bIs, index); + return true; + } catch (IOException e) { + return false; + } } - public static ShardIndex read(InputStream in, final ShardIndex index) throws IOException { + public static void read(InputStream in, final ShardIndex index) throws IOException { @SuppressWarnings("unchecked") final DataBlock indexBlock = (DataBlock) DefaultBlockReader.readBlock(in, index.getIndexAttributes(), index.gridPosition); final long[] indexData = indexBlock.getData(); System.arraycopy(indexData, 0, index.data, 0, index.data.length); - return index; } - public static ShardIndex read( + public static boolean read( final KeyValueAccess keyValueAccess, final String key, final ShardIndex index - ) throws IOException { + ) { - final IndexByteBounds byteBounds = byteBounds(index, keyValueAccess.size(key)); - try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(key, byteBounds.start, byteBounds.end)) { - try (final InputStream in = lockedChannel.newInputStream()) { - return read(in,index); + try { + final IndexByteBounds byteBounds = byteBounds(index, keyValueAccess.size(key)); + try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(key, byteBounds.start, byteBounds.end)) { + try (final InputStream in = lockedChannel.newInputStream()) { + read(in,index); + return true; + } + } catch (final IOException | UncheckedIOException e) { + throw new N5IOException("Failed to read shard index from " + key, e); } - } catch (final N5Exception.N5NoSuchKeyException e) { - return null; - } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to read shard index from " + key, e); + } catch (final IOException | N5Exception.N5NoSuchKeyException e) { + return false; } } @@ -213,16 +221,18 @@ private DatasetAttributes getIndexAttributes() { Arrays.stream(getSize()).mapToLong(it -> it).toArray(), getSize(), DataType.UINT64, - null, codecs ); return indexAttributes; } - public static IndexByteBounds byteBounds(ShardedDatasetAttributes datasetAttributes, final long objectSize) { + public static IndexByteBounds byteBounds(DatasetAttributes datasetAttributes, final long objectSize) { + + ShardingCodec shardCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); + final ShardIndex index = shardCodec.createIndex(datasetAttributes); - final long indexSize = datasetAttributes.createIndex().numBytes(); - return byteBounds(indexSize, datasetAttributes.getIndexLocation(), objectSize); + final long indexSize = index.numBytes(); + return byteBounds(indexSize, index.location, objectSize); } public static IndexByteBounds byteBounds(final ShardIndex index, long objectSize) { @@ -252,15 +262,17 @@ public IndexByteBounds(long start, long end) { } } - public static ShardIndex read(FileChannel channel, ShardedDatasetAttributes datasetAttributes) throws IOException { + //TODO Caleb: Probably don't need to keep this eventually + public static ShardIndex read(FileChannel channel, DatasetAttributes datasetAttributes) throws IOException { // TODO need codecs // TODO FileChannel is too specific - generalize + ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); final int[] indexShape = prepend(2, datasetAttributes.getBlocksPerShard()); final int indexSize = (int)Arrays.stream(indexShape).reduce(1, (x, y) -> x * y); final int indexBytes = BYTES_PER_LONG * indexSize; - if (datasetAttributes.getIndexLocation() == IndexLocation.END) { + if (shardingCodec.getIndexLocation() == IndexLocation.END) { channel.position(channel.size() - indexBytes); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java index ddc3ba28b..1791c9445 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java @@ -5,6 +5,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Spliterator; import java.util.Spliterators; import java.util.TreeMap; @@ -14,26 +15,23 @@ import org.janelia.saalfeldlab.n5.BlockParameters; import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.janelia.saalfeldlab.n5.util.Position; +import javax.annotation.CheckForNull; + +@Deprecated public interface ShardParameters extends BlockParameters { - public ShardingCodec getShardingCodec(); /** * The size of the blocks in pixel units. * * @return the number of pixels per dimension for this shard. */ - public int[] getShardSize(); - - public IndexLocation getIndexLocation(); + @CheckForNull + int[] getShardSize(); - default ShardIndex createIndex() { - return new ShardIndex(getBlocksPerShard(), getIndexLocation(), getShardingCodec().getIndexCodecs()); - } /** * Returns the number of blocks per dimension for a shard. @@ -42,11 +40,13 @@ default ShardIndex createIndex() { */ default int[] getBlocksPerShard() { + final int[] shardSize = getShardSize(); + Objects.requireNonNull(shardSize, "getShardSize() must not be null"); final int nd = getNumDimensions(); final int[] blocksPerShard = new int[nd]; final int[] blockSize = getBlockSize(); for (int i = 0; i < nd; i++) - blocksPerShard[i] = getShardSize()[i] / blockSize[i]; + blocksPerShard[i] = shardSize[i] / blockSize[i]; return blocksPerShard; } @@ -57,9 +57,9 @@ default int[] getBlocksPerShard() { * @return blocks per image */ default long[] blocksPerImage() { - return IntStream.range(0, getNumDimensions()).mapToLong(i -> { - return (long) Math.ceil(getDimensions()[i] / getBlockSize()[i]); - }).toArray(); + return IntStream.range(0, getNumDimensions()) + .mapToLong(i -> (long) Math.ceil(getDimensions()[i] / getBlockSize()[i])) + .toArray(); } /** @@ -68,9 +68,9 @@ default long[] blocksPerImage() { * @return shards per image */ default long[] shardsPerImage() { - return IntStream.range(0, getNumDimensions()).mapToLong(i -> { - return (long)Math.ceil(getDimensions()[i] / getShardSize()[i]); - }).toArray(); + return IntStream.range(0, getNumDimensions()) + .mapToLong(i -> (long)Math.ceil(getDimensions()[i] / getShardSize()[i])) + .toArray(); } /** @@ -141,6 +141,7 @@ default long[] getBlockMinFromShardPosition(final long[] shardPosition, final lo // is this useful? final int[] blockSize = getBlockSize(); final int[] shardSize = getShardSize(); + Objects.requireNonNull(shardSize, "getShardSize() must not be null"); final long[] blockImagePos = new long[shardSize.length]; for (int i = 0; i < shardSize.length; i++) { blockImagePos[i] = (shardPosition[i] * shardSize[i]) + (blockPosition[i] * blockSize[i]); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index cb65f1a4d..79c609b86 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -9,6 +9,7 @@ import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.serialization.N5Annotations; @@ -99,14 +100,40 @@ public DeterministicSizeCodec[] getIndexCodecs() { return indexCodecs; } + @Override public long[] getPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { + + final long[] blockPosition = datablock.getGridPosition(); + return attributes.getShardPositionForBlock(blockPosition); + } + + @Override public long[] getPositionForBlock(DatasetAttributes attributes, final long... blockPosition) { + + return attributes.getShardPositionForBlock(blockPosition); + } @Override public DataBlockInputStream decode(DatasetAttributes attributes, long[] gridPosition, InputStream in) throws IOException { return getArrayCodec().decode(attributes, gridPosition, in); } - @Override public DataBlockOutputStream encode(DatasetAttributes attributes, DataBlock datablock, OutputStream out) throws IOException { + @Override public DataBlockOutputStream encode(DatasetAttributes attributes, DataBlock dataBlock, OutputStream out) throws IOException { + + return getArrayCodec().encode(attributes, dataBlock, out); + } + + @Override public void writeBlock(KeyValueAccess kva, String keyPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) { + + final long[] shardPos = datasetAttributes.getShardPositionForBlock(dataBlock.getGridPosition()); + new VirtualShard(datasetAttributes, shardPos, kva, keyPath).writeBlock(dataBlock); + } + + @Override public DataBlock readBlock(final KeyValueAccess kva, final String keyPath, final DatasetAttributes datasetAttributes, final long... gridPosition) { + + final long[] shardPosition = datasetAttributes.getShardPositionForBlock(gridPosition); + return new VirtualShard(datasetAttributes, shardPosition, kva, keyPath).getBlock(gridPosition); + } - return getArrayCodec().encode(attributes, datablock, out); + ShardIndex createIndex(final DatasetAttributes attributes) { + return new ShardIndex(attributes.getBlocksPerShard(), getIndexLocation(), getIndexCodecs()); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 88aa6b086..ad70e4d59 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -15,12 +16,11 @@ import org.apache.commons.io.input.ProxyInputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.DefaultBlockReader; -import org.janelia.saalfeldlab.n5.DefaultBlockWriter; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.util.GridIterator; public class VirtualShard extends AbstractShard { @@ -28,7 +28,7 @@ public class VirtualShard extends AbstractShard { final private KeyValueAccess keyValueAccess; final private String path; - public VirtualShard(final A datasetAttributes, long[] gridPosition, + public VirtualShard(final DatasetAttributes datasetAttributes, long[] gridPosition, final KeyValueAccess keyValueAccess, final String path) { super(datasetAttributes, gridPosition, null); @@ -36,23 +36,34 @@ public VirtualShard(final A data this.path = path; } - public VirtualShard(final A datasetAttributes, long[] gridPosition) { + public VirtualShard(final DatasetAttributes datasetAttributes, long[] gridPosition) { this(datasetAttributes, gridPosition, null, null); } @SuppressWarnings("unchecked") - public DataBlock getBlock(InputStream inputStream, long... blockGridPosition) throws IOException { - - // TODO this method is just a wrapper around readBlock - // is it worth keeping/ - return (DataBlock) DefaultBlockReader.readBlock( - new ProxyInputStream( inputStream ) { - @Override - public void close( ) { - //nop - } - }, datasetAttributes, blockGridPosition); + public DataBlock getBlock(InputStream in, long... blockGridPosition) throws IOException { + + ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); + final Codec.BytesCodec[] codecs = shardingCodec.getCodecs(); + final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); + + final ProxyInputStream proxyIn = new ProxyInputStream(in) { + @Override + public void close() { + //nop + } + }; + final Codec.DataBlockInputStream dataBlockStream = arrayCodec.decode(datasetAttributes, blockGridPosition, proxyIn); + + final InputStream stream = Codec.decode(in, codecs); + final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); + dataBlock.readData(dataBlockStream.getDataInput(stream)); + stream.close(); + + return dataBlock; + + } @Override @@ -72,21 +83,19 @@ public List> getBlocks(final int[] blockIndexes) { // sort index offsets // and keep track of relevant positions final long[] indexData = index.getData(); - List sortedOffsets = Arrays.stream(blockIndexes).mapToObj(i -> { - return new long[] { indexData[i * 2], i }; - }).filter(x -> { - return x[0] != ShardIndex.EMPTY_INDEX_NBYTES; - }).collect(Collectors.toList()); - - Collections.sort(sortedOffsets, (a, b) -> Long.compare(((long[]) a)[0], ((long[]) b)[0])); + List sortedOffsets = Arrays.stream(blockIndexes) + .mapToObj(i -> new long[]{indexData[i * 2], i}) + .filter(x -> x[0] != ShardIndex.EMPTY_INDEX_NBYTES) + .sorted(Comparator.comparingLong(a -> ((long[])a)[0])) + .collect(Collectors.toList()); final int nd = getDatasetAttributes().getNumDimensions(); long[] position = new long[nd]; final int[] blocksPerShard = getDatasetAttributes().getBlocksPerShard(); - final long[] blockGridMin = IntStream.range(0, nd).mapToLong(i -> { - return blocksPerShard[i] * getGridPosition()[i]; - }).toArray(); + final long[] blockGridMin = IntStream.range(0, nd) + .mapToLong(i -> blocksPerShard[i] * getGridPosition()[i]) + .toArray(); long streamPosition = 0; try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path)) { @@ -166,8 +175,8 @@ public void writeBlock(final DataBlock block) { try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(path, startByte, size)) { try ( final OutputStream channelOut = lockedChannel.newOutputStream()) { - try (final CountingOutputStream out = new CountingOutputStream(channelOut)) { - DefaultBlockWriter.writeBlock(out, datasetAttributes, block); + try (final CountingOutputStream out = new CountingOutputStream(channelOut)) {; + writeBlock(out, datasetAttributes, block); /* Update and write the index to the shard*/ index.set(startByte, out.getNumBytes(), relativePosition); @@ -184,23 +193,31 @@ public void writeBlock(final DataBlock block) { } } + void writeBlock( + final OutputStream out, + final DatasetAttributes datasetAttributes, + final DataBlock dataBlock) throws IOException { + + ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); + final Codec.BytesCodec[] codecs = shardingCodec.getCodecs(); + final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); + final Codec.DataBlockOutputStream dataBlockOutput = arrayCodec.encode(datasetAttributes, dataBlock, out); + final OutputStream stream = Codec.encode(dataBlockOutput, codecs); + + dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); + stream.close(); + } public ShardIndex createIndex() { // Empty index of the correct size - return getDatasetAttributes().createIndex(); + return ((ShardingCodec)getDatasetAttributes().getArrayCodec()).createIndex(getDatasetAttributes()); } @Override public ShardIndex getIndex() { - try { - final ShardIndex readIndex = ShardIndex.read(keyValueAccess, path, getDatasetAttributes().createIndex()); - index = readIndex == null ? createIndex() : readIndex; - } catch (final N5Exception.N5NoSuchKeyException e) { index = createIndex(); - } catch (IOException e) { - throw new N5IOException("Failed to read index at " + path, e); - } + ShardIndex.read(keyValueAccess, path, index); return index; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index a19163273..9b1e08313 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -216,7 +216,7 @@ public void testCreateDataset() { final DatasetAttributes info; try (N5Writer writer = createTempN5Writer()) { - writer.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression()); + writer.createDataset(datasetName, dimensions, blockSize, DataType.UINT64); assertTrue("Dataset does not exist", writer.exists(datasetName)); @@ -264,12 +264,12 @@ public void testWriteReadByteBlockMultipleCodecs() { final long[] longBlock1 = new long[]{1,2,3,4,5,6,7,8}; final long[] dimensions1 = new long[]{2,2,2}; final int[] blockSize1 = new int[]{2,2,2}; - n5.createDataset(datasetName, dimensions1, blockSize1, DataType.INT8, new RawCompression(), codecs); + n5.createDataset(datasetName, dimensions1, blockSize1, DataType.INT8, codecs); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize1, new long[]{0, 0, 0}, longBlock1); n5.writeBlock(datasetName, attributes, dataBlock); - final DatasetAttributes fakeAttributes = new DatasetAttributes(dimensions1, blockSize1, DataType.INT64, new RawCompression(), codecs); + final DatasetAttributes fakeAttributes = new DatasetAttributes(dimensions1, blockSize1, DataType.INT64, codecs); final DataBlock loadedDataBlock = n5.readBlock(datasetName, fakeAttributes, 0, 0, 0); assertArrayEquals(longBlock1, (long[])loadedDataBlock.getData()); assertTrue(n5.remove(datasetName)); @@ -336,7 +336,7 @@ public void testWriteReadIntBlock() { DataType.INT32}) { try (final N5Writer n5 = createTempN5Writer()) { - n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); + n5.createDataset(datasetName, dimensions, blockSize, dataType, (Codec)compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final IntArrayDataBlock dataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); n5.writeBlock(datasetName, attributes, dataBlock); @@ -910,7 +910,7 @@ public void testDeepList() throws ExecutionException, InterruptedException { for (final String subGroup : subGroupNames) assertTrue("deepList contents", Arrays.asList(n5.deepList("")).contains(groupName.replaceFirst("/", "") + "/" + subGroup)); - final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64, new RawCompression()); + final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64); final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize, new long[]{0, 0, 0}, new long[blockNumElements]); n5.createDataset(datasetName, datasetAttributes); n5.writeBlock(datasetName, datasetAttributes, dataBlock); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccessTest.java index e902eb880..a42f8e078 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccessTest.java @@ -7,9 +7,6 @@ import java.nio.file.FileSystems; import java.nio.file.Paths; -import java.util.Arrays; - -import org.junit.BeforeClass; import org.junit.Test; @@ -47,12 +44,6 @@ public class FileSystemKeyValueAccessTest { {""} }; - /** - * @throws java.lang.Exception - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception {} - @Test public void testComponents() { @@ -61,7 +52,6 @@ public void testComponents() { for (int i = 0; i < testPaths.length; ++i) { final String[] components = access.components(testPaths[i]); - System.out.println(String.format("%d: %s -> %s", i, testPaths[i], Arrays.toString(components))); assertArrayEquals(testPathComponents[i], components); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index bd1e43aaf..da0a38ec0 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -199,7 +199,6 @@ public void testWriteLock() throws IOException { @Test public void testLockReleaseByReader() throws IOException, ExecutionException, InterruptedException, TimeoutException { - System.out.println("Testing lock release by Reader."); final Path path = Paths.get(tempN5PathName(), "lock"); final LockedChannel lock = access.lockForWriting(path); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java index 66ef86321..ea24a7d90 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java @@ -24,12 +24,12 @@ public void testSerialization() { factory.cacheAttributes(false); final GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); - gsonBuilder.registerTypeAdapter(ByteOrder.class, BytesCodec.byteOrderAdapter); + gsonBuilder.registerTypeAdapter(ByteOrder.class, RawBytes.byteOrderAdapter); factory.gsonBuilder(gsonBuilder); final N5Writer reader = factory.openWriter("n5:src/test/resources/shardExamples/test.n5"); final Codec bytes = reader.getAttribute("mid_sharded", "codecs[0]/configuration/codecs[0]", Codec.class); - assertTrue("as BytesCodec", bytes instanceof BytesCodec); + assertTrue("as RawBytes", bytes instanceof RawBytes); final N5Writer writer = factory.openWriter("n5:src/test/resources/shardExamples/test.n5"); @@ -37,11 +37,8 @@ public void testSerialization() { new long[]{8, 8}, new int[]{4, 4}, DataType.UINT8, - new RawCompression(), - new Codec[]{ new N5BlockCodec(ByteOrder.LITTLE_ENDIAN), new IdentityCodec() - } ); writer.createGroup("shard"); //Should already exist, but this will ensure. writer.setAttribute("shard", "/", datasetAttributes); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java index 1035e271a..f8cf5215c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java @@ -1,8 +1,11 @@ package org.janelia.saalfeldlab.n5.codec; import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; @@ -38,9 +41,7 @@ public void testLengthOne() throws IOException final ByteArrayOutputStream outPlusOne = new ByteArrayOutputStream(N); final FixedLengthConvertedOutputStream convertedPlusOne = new FixedLengthConvertedOutputStream(1, 1, - (x, y) -> { - y.put((byte)(x.get() + 1)); - }, + (x, y) -> y.put((byte)(x.get() + 1)), outPlusOne); convertedPlusOne.write(expectedData); @@ -54,14 +55,10 @@ public void testIntToByte() throws IOException final int N = 16; final ByteBuffer buf = ByteBuffer.allocate(Integer.BYTES * N); - IntStream.range(0, N).forEach( x -> { - buf.putInt(x); - }); + IntStream.range(0, N).forEach(buf::putInt); final ByteBuffer expected = ByteBuffer.allocate(N); - IntStream.range(0, N).forEach( x -> { - expected.put((byte)x); - }); + IntStream.range(0, N).forEach( x -> expected.put((byte)x)); final ByteArrayOutputStream outStream = new ByteArrayOutputStream(N); final FixedLengthConvertedOutputStream intToByte = new FixedLengthConvertedOutputStream( @@ -72,38 +69,28 @@ public void testIntToByte() throws IOException intToByte.write(buf.array()); intToByte.close(); - System.out.println(Arrays.toString(buf.array())); - System.out.println(Arrays.toString(expected.array())); - System.out.println(Arrays.toString(outStream.toByteArray())); -// -// assertArrayEquals(expected.array(), outStream.toByteArray()); + assertArrayEquals(expected.array(), outStream.toByteArray()); + } + @Test + public void testByteToInt() throws IOException + { + + final int N = 16; + final byte[] data = new byte[16]; + for( int i = 0; i < N; i++ ) + data[i] = (byte)i; + + FixedLengthConvertedInputStream byteToInt = new FixedLengthConvertedInputStream( + 1, 4, + (input, output) -> output.putInt(input.get()), + new ByteArrayInputStream(data)); + + final DataInputStream dataStream = new DataInputStream(byteToInt); + for( int i = 0; i < N; i++ ) + assertEquals(i, dataStream.readInt()); + + dataStream.close(); + byteToInt.close(); } -// -// @Test -// public void testByteToInt() throws IOException -// { -// -// final int N = 16; -// final byte[] data = new byte[16]; -// for( int i = 0; i < N; i++ ) -// data[i] = (byte)i; -// -// FixedLengthConvertedInputStream byteToInt = new FixedLengthConvertedInputStream( -// 1, 4, -// (x, y) -> { -// y[0] = 0; // the setting to zero is not strictly necessary in this case -// y[1] = 0; -// y[2] = 0; -// y[3] = x[0]; -// }, -// new ByteArrayInputStream(data)); -// -// final DataInputStream dataStream = new DataInputStream(byteToInt); -// for( int i = 0; i < N; i++ ) -// assertEquals(i, dataStream.readInt()); -// -// dataStream.close(); -// byteToInt.close(); -// } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java index 1631b9450..c96edc079 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java @@ -12,7 +12,7 @@ import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; @@ -33,8 +33,8 @@ public static void shardBlockIterator() { new int[] {6, 4}, // shard size new int[] {2, 2}, // block size DataType.UINT8, - new Codec[] { new BytesCodec() }, - new DeterministicSizeCodec[] { new BytesCodec() }, + new Codec[] { new RawBytes() }, + new DeterministicSizeCodec[] { new RawBytes() }, IndexLocation.END); shardPositions(attrs) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index f260b7085..0c8ee24ae 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -6,23 +6,16 @@ import java.io.InputStream; import java.nio.file.Paths; -import org.apache.commons.io.output.ByteArrayOutputStream; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.GzipCompression; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5FSTest; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; -import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.junit.After; -import org.junit.Ignore; import org.junit.Test; public class ShardIndexTest { @@ -40,7 +33,7 @@ public void testOffsetIndex() throws IOException { int[] shardBlockGridSize = new int[]{5,4,3}; ShardIndex index = new ShardIndex( shardBlockGridSize, - IndexLocation.END, new BytesCodec()); + IndexLocation.END, new RawBytes()); GridIterator it = new GridIterator(shardBlockGridSize); int i = 0; @@ -54,7 +47,7 @@ public void testOffsetIndex() throws IOException { shardBlockGridSize = new int[]{5,4,3,13}; index = new ShardIndex( shardBlockGridSize, - IndexLocation.END, new BytesCodec()); + IndexLocation.END, new RawBytes()); it = new GridIterator(shardBlockGridSize); i = 0; @@ -74,7 +67,7 @@ public void testReadVirtual() throws IOException { final int[] shardBlockGridSize = new int[] { 6, 5 }; final IndexLocation indexLocation = IndexLocation.END; - final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { new BytesCodec(), + final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { new RawBytes(), new Crc32cChecksumCodec() }; final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "0").toString(); @@ -101,7 +94,7 @@ public void testReadInMemory() throws IOException { final int[] shardBlockGridSize = new int[] { 6, 5 }; final IndexLocation indexLocation = IndexLocation.END; final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { - new BytesCodec(), + new RawBytes(), new Crc32cChecksumCodec() }; final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "indexTest").toString(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java index 77d46ac45..48652cd10 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java @@ -1,21 +1,21 @@ package org.janelia.saalfeldlab.n5.shard; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.Position; import org.junit.Test; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + public class ShardPropertiesTests { @Test @@ -26,17 +26,20 @@ public void testShardProperties() throws Exception { final long[] shardPosition = new long[]{1, 1}; final int[] blkSize = new int[]{4, 4}; - final ShardedDatasetAttributes dsetAttrs = new ShardedDatasetAttributes( + final DatasetAttributes dsetAttrs = new DatasetAttributes( arraySize, shardSize, blkSize, DataType.UINT8, - new Codec[]{}, - new DeterministicSizeCodec[]{}, - IndexLocation.END); + new ShardingCodec( + blkSize, + new Codec[]{}, + new DeterministicSizeCodec[]{}, + IndexLocation.END + ) + ); - @SuppressWarnings({"rawtypes", "unchecked"}) - final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); + @SuppressWarnings({"rawtypes", "unchecked"}) final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); assertArrayEquals(new int[]{4, 4}, shard.getBlockGridSize()); @@ -45,8 +48,8 @@ public void testShardProperties() throws Exception { assertArrayEquals(new long[]{1, 0}, shard.getShard(5, 0)); assertArrayEquals(new long[]{0, 1}, shard.getShard(0, 5)); -// assertNull(shard.getBlockPosition(0, 0)); -// assertNull(shard.getBlockPosition(3, 3)); + // assertNull(shard.getBlockPosition(0, 0)); + // assertNull(shard.getBlockPosition(3, 3)); assertArrayEquals(new int[]{0, 0}, shard.getBlockPosition(4, 4)); assertArrayEquals(new int[]{1, 1}, shard.getBlockPosition(5, 5)); @@ -62,17 +65,20 @@ public void testShardBlockPositionIterator() throws Exception { final long[] shardPosition = new long[]{1, 1}; final int[] blkSize = new int[]{4, 4}; - final ShardedDatasetAttributes dsetAttrs = new ShardedDatasetAttributes( + final DatasetAttributes dsetAttrs = new DatasetAttributes( arraySize, shardSize, blkSize, DataType.UINT8, - new Codec[]{}, - new DeterministicSizeCodec[]{}, - IndexLocation.END); + new ShardingCodec( + blkSize, + new Codec[]{}, + new DeterministicSizeCodec[]{}, + IndexLocation.END + ) + ); - @SuppressWarnings({"rawtypes", "unchecked"}) - final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); + @SuppressWarnings({"rawtypes", "unchecked"}) final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); int i = 0; Iterator it = shard.blockPositionIterator(); @@ -80,13 +86,13 @@ public void testShardBlockPositionIterator() throws Exception { while (it.hasNext()) { p = it.next(); - if( i == 0 ) - assertArrayEquals(new long[]{4,4}, p); + if (i == 0) + assertArrayEquals(new long[]{4, 4}, p); i++; } - assertEquals(16,i); - assertArrayEquals(new long[]{7,7}, p); + assertEquals(16, i); + assertArrayEquals(new long[]{7, 7}, p); } @Test @@ -96,14 +102,18 @@ public void testShardGrouping() { final int[] shardSize = new int[]{4, 6}; final int[] blkSize = new int[]{2, 3}; - final ShardedDatasetAttributes attrs = new ShardedDatasetAttributes( + final DatasetAttributes attrs = new DatasetAttributes( arraySize, shardSize, blkSize, DataType.UINT8, - new Codec[]{}, - new DeterministicSizeCodec[]{}, - IndexLocation.END); + new ShardingCodec( + blkSize, + new Codec[]{}, + new DeterministicSizeCodec[]{}, + IndexLocation.END + ) + ); List blockPositions = attrs.blockPositions().collect(Collectors.toList()); final Map> result = attrs.groupBlockPositions(blockPositions); @@ -112,7 +122,7 @@ public void testShardGrouping() { assertEquals(4, result.keySet().size()); // there are four blocks per shard in this image - result.values().stream().forEach( x -> assertEquals(4, x.size())); + result.values().stream().forEach(x -> assertEquals(4, x.size())); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 6848b9df1..b19e07e5f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -3,16 +3,16 @@ import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5FSTest; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; +import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; @@ -23,17 +23,15 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import static org.junit.Assert.assertArrayEquals; - import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; -import java.util.Iterator; -import java.util.List; import java.util.Map; +import static org.junit.Assert.assertArrayEquals; + @RunWith(Parameterized.class) public class ShardTest { @@ -66,33 +64,39 @@ public static Collection data() { @After public void removeTempWriters() { + tempN5Factory.removeTempWriters(); } - private ShardedDatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] blockSize) { - return new ShardedDatasetAttributes( + private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] blockSize) { + + return new DatasetAttributes( dimensions, shardSize, blockSize, DataType.UINT8, - new Codec[]{new N5BlockCodec(dataByteOrder), new GzipCompression(4)}, - new DeterministicSizeCodec[]{new BytesCodec(indexByteOrder), new Crc32cChecksumCodec()}, - indexLocation + new ShardingCodec( + blockSize, + new Codec[]{new N5BlockCodec(dataByteOrder)}, //, new GzipCompression(4)}, + new DeterministicSizeCodec[]{new RawBytes(indexByteOrder), new Crc32cChecksumCodec()}, + indexLocation + ) ); } - private ShardedDatasetAttributes getTestAttributes() { - return getTestAttributes(new long[]{8, 8}, new int[]{4, 4}, new int[]{2, 2}); + private DatasetAttributes getTestAttributes() { + + return getTestAttributes(new long[]{8, 8}, new int[]{4, 4}, new int[]{2, 2}); } @Test public void writeReadBlocksTest() { final N5Writer writer = tempN5Factory.createTempN5Writer(); - final ShardedDatasetAttributes datasetAttributes = getTestAttributes( - new long[]{24,24}, - new int[]{8,8}, - new int[]{2,2} + final DatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + new int[]{2, 2} ); writer.createDataset("shard", datasetAttributes); @@ -105,22 +109,21 @@ public void writeReadBlocksTest() { data[i] = (byte)((100) + (10) + i); } - writer.writeBlocks( "shard", datasetAttributes, /* shard (0, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{0,0}, data), - new ByteArrayDataBlock(blockSize, new long[]{0,1}, data), - new ByteArrayDataBlock(blockSize, new long[]{1,0}, data), - new ByteArrayDataBlock(blockSize, new long[]{1,1}, data), + new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), /* shard (1, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{4,0}, data), - new ByteArrayDataBlock(blockSize, new long[]{5,0}, data), + new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), /* shard (2, 2) */ - new ByteArrayDataBlock(blockSize, new long[]{11,11}, data) + new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) ); final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); @@ -135,7 +138,7 @@ public void writeReadBlocksTest() { Assert.assertTrue("Shard at" + Arrays.toString(key) + "Does not exist", kva.exists(shard)); } - final long[][] blockIndices = new long[][]{ {0,0}, {0,1}, {1,0}, {1,1}, {4,0}, {5,0}, {11,11}}; + final long[][] blockIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; for (long[] blockIndex : blockIndices) { final DataBlock block = writer.readBlock("shard", datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); @@ -149,15 +152,15 @@ public void writeReadBlocksTest() { "shard", datasetAttributes, /* shard (0, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{0,0}, data2), - new ByteArrayDataBlock(blockSize, new long[]{1,1}, data2), + new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data2), + new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data2), /* shard (0, 1) */ - new ByteArrayDataBlock(blockSize, new long[]{0,4}, data2), - new ByteArrayDataBlock(blockSize, new long[]{0,5}, data2), + new ByteArrayDataBlock(blockSize, new long[]{0, 4}, data2), + new ByteArrayDataBlock(blockSize, new long[]{0, 5}, data2), /* shard (2, 2) */ - new ByteArrayDataBlock(blockSize, new long[]{10,10}, data2) + new ByteArrayDataBlock(blockSize, new long[]{10, 10}, data2) ); final String[][] keys2 = new String[][]{ @@ -171,13 +174,13 @@ public void writeReadBlocksTest() { Assert.assertTrue("Shard at" + Arrays.toString(key) + "Does not exist", kva.exists(shard)); } - final long[][] oldBlockIndices = new long[][]{{0,1}, {1,0}, {4,0}, {5,0}, {11,11}}; + final long[][] oldBlockIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; for (long[] blockIndex : oldBlockIndices) { final DataBlock block = writer.readBlock("shard", datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); } - final long[][] newBlockIndices = new long[][]{{0,0}, {1,1}, {0,4}, {0,5}, {10,10}}; + final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; for (long[] blockIndex : newBlockIndices) { final DataBlock block = writer.readBlock("shard", datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])block.getData()); @@ -188,7 +191,7 @@ public void writeReadBlocksTest() { public void writeReadBlockTest() { final N5Writer writer = tempN5Factory.createTempN5Writer(); - final ShardedDatasetAttributes datasetAttributes = getTestAttributes(); + final DatasetAttributes datasetAttributes = getTestAttributes(); writer.createDataset("shard", datasetAttributes); writer.deleteBlock("shard", 0, 0); @@ -229,15 +232,7 @@ public void writeReadShardTest() { final N5Writer writer = tempN5Factory.createTempN5Writer(); - final ShardedDatasetAttributes datasetAttributes = new ShardedDatasetAttributes( - new long[]{4, 4}, - new int[]{4, 4}, - new int[]{2, 2}, - DataType.UINT8, - new Codec[]{new N5BlockCodec(dataByteOrder)}, - new DeterministicSizeCodec[]{new BytesCodec(indexByteOrder), new Crc32cChecksumCodec()}, - indexLocation - ); + final DatasetAttributes datasetAttributes = getTestAttributes(); writer.createDataset("wholeShard", datasetAttributes); writer.deleteBlock("wholeShard", 0, 0); @@ -277,44 +272,48 @@ public void writeReadShardTest() { public void writeReadNestedShards() { int[] blockSize = new int[]{4, 4}; - int N = Arrays.stream(blockSize).reduce(1, (x,y) -> x*y); + int N = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y); final N5Writer writer = tempN5Factory.createTempN5Writer(); - final ShardedDatasetAttributes datasetAttributes = getNestedShardCodecsAttributes(blockSize); + final DatasetAttributes datasetAttributes = getNestedShardCodecsAttributes(blockSize); writer.createDataset("nestedShards", datasetAttributes); final byte[] data = new byte[N]; Arrays.fill(data, (byte)4); writer.writeBlocks("nestedShards", datasetAttributes, - new ByteArrayDataBlock(blockSize, new long[] { 1, 1 }, data), - new ByteArrayDataBlock(blockSize, new long[] { 0, 2 }, data), - new ByteArrayDataBlock(blockSize, new long[] { 2, 1 }, data)); + new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + new ByteArrayDataBlock(blockSize, new long[]{0, 2}, data), + new ByteArrayDataBlock(blockSize, new long[]{2, 1}, data)); - assertArrayEquals(data, (byte[]) writer.readBlock("nestedShards", datasetAttributes, 1, 1).getData()); - assertArrayEquals(data, (byte[]) writer.readBlock("nestedShards", datasetAttributes, 0, 2).getData()); - assertArrayEquals(data, (byte[]) writer.readBlock("nestedShards", datasetAttributes, 2, 1).getData()); + assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 1, 1).getData()); + assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 0, 2).getData()); + assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 2, 1).getData()); } - private ShardedDatasetAttributes getNestedShardCodecsAttributes(int[] blockSize) { + private DatasetAttributes getNestedShardCodecsAttributes(int[] blockSize) { - final int[] innerShardSize = new int[] { 2 * blockSize[0], 2 * blockSize[1] }; - final int[] shardSize = new int[] { 4 * blockSize[0], 4 * blockSize[1] }; + final int[] innerShardSize = new int[]{2 * blockSize[0], 2 * blockSize[1]}; + final int[] shardSize = new int[]{4 * blockSize[0], 4 * blockSize[1]}; final long[] dimensions = GridIterator.int2long(shardSize); // TODO: its not even clear how we build this given - // this constructor. Is the block size of the sharded dataset attributes - // the innermost (block) size or the intermediate shard size? - // probably better to forget about this class - only use DatasetAttributes - // and detect shading in another way + // this constructor. Is the block size of the sharded dataset attributes + // the innermost (block) size or the intermediate shard size? + // probably better to forget about this class - only use DatasetAttributes + // and detect shading in another way final ShardingCodec innerShard = new ShardingCodec(innerShardSize, - new Codec[] { new BytesCodec() }, - new DeterministicSizeCodec[] { new BytesCodec(indexByteOrder), new Crc32cChecksumCodec() }, + new Codec[]{new RawBytes()}, + new DeterministicSizeCodec[]{new RawBytes(indexByteOrder), new Crc32cChecksumCodec()}, IndexLocation.START); - return new ShardedDatasetAttributes(dimensions, shardSize, blockSize, DataType.UINT8, - new Codec[] { innerShard }, - new DeterministicSizeCodec[] { new BytesCodec(indexByteOrder), new Crc32cChecksumCodec() }, - IndexLocation.END); + return new DatasetAttributes( + dimensions, shardSize, blockSize, DataType.UINT8, + new ShardingCodec( + blockSize, + new Codec[]{innerShard}, + new DeterministicSizeCodec[]{new RawBytes(indexByteOrder), new Crc32cChecksumCodec()}, + IndexLocation.END) + ); } } From 0d1eb525c8065f77fa73280f25629e2ff73a0d25 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 4 Feb 2025 17:10:40 -0500 Subject: [PATCH 159/423] refactor: more from refactorShard branch --- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 8 ++++---- .../org/janelia/saalfeldlab/n5/N5Writer.java | 6 +++--- .../saalfeldlab/n5/shard/InMemoryShard.java | 2 +- .../org/janelia/saalfeldlab/n5/shard/Shard.java | 11 +++++++---- .../saalfeldlab/n5/shard/ShardIndex.java | 7 ++----- .../saalfeldlab/n5/shard/ShardingCodec.java | 2 +- .../saalfeldlab/n5/shard/VirtualShard.java | 17 ++++++++--------- .../n5/shard/ShardPropertiesTests.java | 8 ++++---- 8 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 36f5ef1db..b5e4cdf2a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -264,15 +264,15 @@ default void writeBlock( } @Override - default void writeShard( + default void writeShard( final String path, - final A datasetAttributes, + final DatasetAttributes datasetAttributes, final Shard shard) throws N5Exception { final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shard.getGridPosition()); try (final LockedChannel lock = getKeyValueAccess().lockForWriting(shardPath)) { - try (final OutputStream out = lock.newOutputStream()) { - InMemoryShard.fromShard(shard).write(out); + try (final OutputStream shardOut = lock.newOutputStream()) { + InMemoryShard.fromShard(shard).write(shardOut); } } catch (final IOException | UncheckedIOException e) { throw new N5IOException( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 2471135cb..016062043 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Map; +import org.checkerframework.checker.units.qual.A; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.shard.Shard; @@ -299,12 +300,11 @@ default void writeBlocks( * @param datasetAttributes the dataset attributes * @param shard the shard * @param the data block data type - * @param the attribute type * @throws N5Exception the exception */ - void writeShard( + void writeShard( final String datasetPath, - final A datasetAttributes, + final DatasetAttributes datasetAttributes, final Shard shard) throws N5Exception; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index f7a01fc87..c7274d850 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -95,7 +95,7 @@ public List> getBlocks( int[] blockIndexes ) { for( int idx : blockIndexes ) { GridIterator.indexToPosition(idx, blocksPerShard, position); DataBlock blk = getBlock(position); - if( blk != null ); + if( blk != null ) out.add(blk); } return out; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 2e1d7e7e2..3f55dfbc2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -84,7 +84,7 @@ default long[] getShardMinPosition(long... shardPosition) { * * @return the shard position */ - default long[] getShard(long... blockPosition) { + default long[] getShardPosition(long... blockPosition) { final int[] shardBlockDimensions = getBlockGridSize(); final long[] shardGridPosition = new long[shardBlockDimensions.length]; @@ -99,6 +99,8 @@ default long[] getShard(long... blockPosition) { public void writeBlock(DataBlock block); + //TODO Caleb: add writeBlocks that does NOT always expect to overwrite the entire existing Shard + default Iterator> iterator() { return new DataBlockIterator<>(this); @@ -130,9 +132,9 @@ default Iterator blockPositionIterator() { return new GridIterator(GridIterator.int2long(getBlockGridSize()), min); } - public ShardIndex getIndex(); + ShardIndex getIndex(); - public static Shard createEmpty(final A attributes, long... shardPosition) { + static Shard createEmpty(final A attributes, long... shardPosition) { final long[] emptyIndex = new long[(int)(2 * attributes.getNumBlocks())]; Arrays.fill(emptyIndex, ShardIndex.EMPTY_INDEX_NBYTES); @@ -140,11 +142,12 @@ public static Shard createE return new InMemoryShard(attributes, shardPosition, shardIndex); } - public static class DataBlockIterator implements Iterator> { + class DataBlockIterator implements Iterator> { private final GridIterator it; private final Shard shard; private final ShardIndex index; + // TODO ShardParameters is deprecated? private final ShardParameters attributes; private int blockIndex = 0; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index f50a6d623..655bf62c5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -11,10 +11,8 @@ import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import java.io.ByteArrayInputStream; @@ -75,7 +73,7 @@ public int getNumBlocks() { public boolean isEmpty() { - return !IntStream.range(0, getNumBlocks()).anyMatch(i -> exists(i)); + return !IntStream.range(0, getNumBlocks()).anyMatch(this::exists); } public IndexLocation getLocation() { @@ -146,7 +144,7 @@ public static boolean read(byte[] data, final ShardIndex index) { try { BoundedInputStream bIs = BoundedInputStream.builder() .setInputStream(is) - .setMaxCount(byteBounds.size).get(); + .setMaxCount(index.numBytes()).get(); read(bIs, index); return true; @@ -321,6 +319,5 @@ public boolean equals(Object other) { } return true; } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 79c609b86..4da31effb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -132,7 +132,7 @@ public DeterministicSizeCodec[] getIndexCodecs() { return new VirtualShard(datasetAttributes, shardPosition, kva, keyPath).getBlock(gridPosition); } - ShardIndex createIndex(final DatasetAttributes attributes) { + public ShardIndex createIndex(final DatasetAttributes attributes) { return new ShardIndex(attributes.getBlocksPerShard(), getIndexLocation(), getIndexCodecs()); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index ad70e4d59..3b115a900 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -136,17 +136,16 @@ public DataBlock getBlock(long... blockGridPosition) { throw new N5IOException("Attempted to read a block from the wrong shard."); final ShardIndex idx = getIndex(); - - final long startByte = idx.getOffset(relativePosition); - - if (startByte == ShardIndex.EMPTY_INDEX_NBYTES ) + if (!idx.exists(relativePosition)) return null; - final long size = idx.getNumBytes(relativePosition); - try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path, startByte, size)) { - try ( final InputStream channelIn = lockedChannel.newInputStream()) { + final long blockOffset = idx.getOffset(relativePosition); + final long blockSize = idx.getNumBytes(relativePosition); + + try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path, blockOffset, blockSize)) { + try ( final InputStream in = lockedChannel.newInputStream()) { final long[] blockPosInImg = getDatasetAttributes().getBlockPositionFromShardPosition(getGridPosition(), blockGridPosition); - return getBlock( channelIn, blockPosInImg ); + return getBlock( in, blockPosInImg ); } } catch (final N5Exception.N5NoSuchKeyException e) { return null; @@ -216,7 +215,7 @@ public ShardIndex createIndex() { @Override public ShardIndex getIndex() { - index = createIndex(); + index = createIndex(); ShardIndex.read(keyValueAccess, path, index); return index; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java index 48652cd10..eb0d6de47 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java @@ -43,10 +43,10 @@ public void testShardProperties() throws Exception { assertArrayEquals(new int[]{4, 4}, shard.getBlockGridSize()); - assertArrayEquals(new long[]{0, 0}, shard.getShard(0, 0)); - assertArrayEquals(new long[]{1, 1}, shard.getShard(5, 5)); - assertArrayEquals(new long[]{1, 0}, shard.getShard(5, 0)); - assertArrayEquals(new long[]{0, 1}, shard.getShard(0, 5)); + assertArrayEquals(new long[]{0, 0}, shard.getShardPosition(0, 0)); + assertArrayEquals(new long[]{1, 1}, shard.getShardPosition(5, 5)); + assertArrayEquals(new long[]{1, 0}, shard.getShardPosition(5, 0)); + assertArrayEquals(new long[]{0, 1}, shard.getShardPosition(0, 5)); // assertNull(shard.getBlockPosition(0, 0)); // assertNull(shard.getBlockPosition(3, 3)); From 765a7b428502a992c32a98e38ad28d94f5390be4 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 24 Jan 2025 15:27:27 -0500 Subject: [PATCH 160/423] feat(wip): initial SplitableData implementation --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 94 +++++++--- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 20 ++- .../saalfeldlab/n5/SplitByteBufferedData.java | 158 ++++++++++++++++ .../n5/SplitKeyValueAccessData.java | 72 ++++++++ .../janelia/saalfeldlab/n5/SplitableData.java | 21 +++ .../janelia/saalfeldlab/n5/codec/Codec.java | 96 +++++----- .../saalfeldlab/n5/shard/InMemoryShard.java | 132 +++----------- .../janelia/saalfeldlab/n5/shard/Shard.java | 46 ++++- .../saalfeldlab/n5/shard/ShardIndex.java | 68 +++---- .../saalfeldlab/n5/shard/ShardingCodec.java | 19 +- .../saalfeldlab/n5/shard/VirtualShard.java | 170 ++++++++---------- .../saalfeldlab/n5/AbstractN5Test.java | 2 + .../saalfeldlab/n5/shard/ShardIndexTest.java | 94 +++++----- .../saalfeldlab/n5/shard/ShardTest.java | 151 ++++++++-------- .../writeReadBlockTest.n5/attributes.json | 1 + .../writeReadBlockTest.n5/shard/0/0 | Bin 0 -> 336 bytes .../shard/attributes.json | 1 + 17 files changed, 696 insertions(+), 449 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/SplitByteBufferedData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/SplitableData.java create mode 100644 src/test/resources/shardExamples/writeReadBlockTest.n5/attributes.json create mode 100644 src/test/resources/shardExamples/writeReadBlockTest.n5/shard/0/0 create mode 100644 src/test/resources/shardExamples/writeReadBlockTest.n5/shard/attributes.json diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 78e10811a..52f2aa7af 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -25,13 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.Gson; -import com.google.gson.JsonElement; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.shard.Shard; -import org.janelia.saalfeldlab.n5.shard.VirtualShard; -import org.janelia.saalfeldlab.n5.util.Position; - import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; @@ -39,6 +32,15 @@ import java.util.Map; import java.util.Map.Entry; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.ShardParameters; +import org.janelia.saalfeldlab.n5.shard.VirtualShard; +import org.janelia.saalfeldlab.n5.util.Position; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON * attributes parsed with {@link Gson}. @@ -98,40 +100,62 @@ default Shard readShard( long... shardGridPosition) { final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(keyPath), shardGridPosition); - return new VirtualShard<>(datasetAttributes, shardGridPosition, getKeyValueAccess(), path); + final SplitKeyValueAccessData splitableData; + try { + splitableData = new SplitKeyValueAccessData(getKeyValueAccess(), path); + } catch (IOException e) { + throw new N5IOException(e); + } + return new VirtualShard( + datasetAttributes, + shardGridPosition, + splitableData); } @Override - default DataBlock readBlock( + default DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { - final long[] keyPos = datasetAttributes.getArrayCodec().getPositionForBlock(datasetAttributes, gridPosition); - final String keyPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), keyPos); + if (datasetAttributes instanceof ShardedDatasetAttributes) { + final ShardedDatasetAttributes shardedAttrs = (ShardedDatasetAttributes) datasetAttributes; + final long[] shardPosition = shardedAttrs.getShardPositionForBlock(gridPosition); + final Shard shard = readShard(pathName, shardedAttrs, shardPosition); + return shard.getBlock(gridPosition); + } + final SplitKeyValueAccessData splitData; + try { + splitData = new SplitKeyValueAccessData(getKeyValueAccess(), pathName); + } catch (IOException e) { + throw new N5IOException(e); + } return datasetAttributes.getArrayCodec().readBlock( - getKeyValueAccess(), - keyPath, + splitData, datasetAttributes, gridPosition ); } @Override - default List> readBlocks( + default List> readBlocks( final String pathName, final DatasetAttributes datasetAttributes, final List blockPositions) throws N5Exception { // TODO which interface should have this implementation? - if (datasetAttributes.getShardSize() != null) { + if (datasetAttributes instanceof ShardParameters) { + + final ShardParameters shardAttributes = (ShardParameters)datasetAttributes; + /* Group by shard position */ - final Map> shardBlockMap = datasetAttributes.groupBlockPositions(blockPositions); - final ArrayList> blocks = new ArrayList<>(); + final Map> shardBlockMap = shardAttributes.groupBlockPositions(blockPositions); + final ArrayList> blocks = new ArrayList<>(); for( Entry> e : shardBlockMap.entrySet()) { - final Shard shard = readShard(pathName, datasetAttributes, e.getKey().get()); + final Shard shard = readShard(pathName, (DatasetAttributes & ShardParameters) shardAttributes, + e.getKey().get()); for (final long[] blkPosition : e.getValue()) { blocks.add(shard.getBlock(blkPosition)); @@ -139,8 +163,8 @@ default List> readBlocks( } return blocks; - } - return GsonN5Reader.super.readBlocks(pathName, datasetAttributes, blockPositions); + } else + return GsonN5Reader.super.readBlocks(pathName, datasetAttributes, blockPositions); } @Override @@ -156,9 +180,6 @@ default String[] list(final String pathName) throws N5Exception { /** * Constructs the path for a data block in a dataset at a given grid * position. - *
    - * If the gridPosition passed in refers to shard position - * in a sharded dataset, this will return the path to the shard key *

    * The returned path is * @@ -186,6 +207,33 @@ default String absoluteDataBlockPath( return getKeyValueAccess().compose(getURI(), components); } + /** + * Constructs the path for a shard in a dataset at a given grid position. + *

    + * The returned path is + * + * 

    +	 * $basePath/datasetPathName/$shardPosition[0]/$shardPosition[1]/.../$shardPosition[n]
+	 *
    + *

    + * This is the file into which the shard will be stored. + * + * @param normalPath normalized dataset path + * @param shardGridPosition to the target shard + * @return the absolute path to the shard at shardGridPosition + */ + default String absoluteShardPath( + final String normalPath, + final long... shardGridPosition) { + + final String[] components = new String[shardGridPosition.length + 1]; + components[0] = normalPath; + int i = 0; + for (final long p : shardGridPosition) + components[++i] = Long.toString(p); + + return getKeyValueAccess().compose(getURI(), components); + } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index b5e4cdf2a..938c20b09 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.Arrays; @@ -44,7 +45,6 @@ import com.google.gson.JsonNull; import com.google.gson.JsonObject; import com.google.gson.JsonSyntaxException; -import org.janelia.saalfeldlab.n5.shard.VirtualShard; import org.janelia.saalfeldlab.n5.util.Position; /** @@ -255,10 +255,14 @@ default void writeBlock( final long[] keyPos = datasetAttributes.getArrayCodec().getPositionForBlock(datasetAttributes, dataBlock); final String keyPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), keyPos); - + final SplitKeyValueAccessData splitData; + try { + splitData = new SplitKeyValueAccessData(getKeyValueAccess(), keyPath); + } catch (IOException e) { + throw new N5IOException(e); + } datasetAttributes.getArrayCodec().writeBlock( - getKeyValueAccess(), - keyPath, + splitData, datasetAttributes, dataBlock); } @@ -270,9 +274,11 @@ default void writeShard( final Shard shard) throws N5Exception { final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shard.getGridPosition()); - try (final LockedChannel lock = getKeyValueAccess().lockForWriting(shardPath)) { - try (final OutputStream shardOut = lock.newOutputStream()) { - InMemoryShard.fromShard(shard).write(shardOut); + try (final OutputStream shardOut = new SplitKeyValueAccessData(getKeyValueAccess(), shardPath, 0, Long.MAX_VALUE).newOutputStream()) { + try (final InputStream shardIn = shard.getAsStream()) { + while (shardIn.available() > 0) { + shardOut.write(shardIn.read()); + } } } catch (final IOException | UncheckedIOException e) { throw new N5IOException( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/SplitByteBufferedData.java b/src/main/java/org/janelia/saalfeldlab/n5/SplitByteBufferedData.java new file mode 100644 index 000000000..3fe9b2d83 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/SplitByteBufferedData.java @@ -0,0 +1,158 @@ +package org.janelia.saalfeldlab.n5; + +import org.checkerframework.checker.units.qual.A; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.rmi.AccessException; +import java.util.function.Consumer; +import java.util.function.Supplier; + +public class SplitByteBufferedData implements SplitableData { + + private byte[] sharedHierarchyData; + private int maxCount = 0; + + private final AccessibleByteArrayOutputStream data; + private long offset; + + public SplitByteBufferedData() { + this(32); + } + + public SplitByteBufferedData(final int initialSize) { + + this(new byte[initialSize]); + } + + public SplitByteBufferedData(final int initialSize, final long offset) { + + this(new byte[initialSize], offset); + } + + public SplitByteBufferedData(final byte[] initialBuffer) { + + this(initialBuffer, 0); + } + + public SplitByteBufferedData(final byte[] initialBuffer, final long offset) { + + this.offset = offset; + this.sharedHierarchyData = initialBuffer; + this.data = new AccessibleByteArrayOutputStream(this::getSharedBuffer, this::setSharedBuffer, this::getMaxCount, this::setMaxCount); + data.setCount((int)offset); + maxCount = initialBuffer.length; + } + + private SplitByteBufferedData(final AccessibleByteArrayOutputStream splitData, final long offset, final long size) { + + this.offset = offset; + this.data = splitData; + data.setCount((int)offset); + } + + private byte[] getSharedBuffer() { + return sharedHierarchyData; + } + + private void setSharedBuffer(byte[] buffer) { + sharedHierarchyData = buffer; + } + + private void setMaxCount(int count) { + if (count > maxCount) + maxCount = count; + } + + private int getMaxCount() { + return maxCount; + } + + @Override + public long getOffset() { + + return offset; + } + + @Override + public long getSize() { + + return getMaxCount() - getOffset(); + } + + @Override + public InputStream newInputStream() { + + final byte[] sharedBuffer = data.getBuf(); + return new ByteArrayInputStream(sharedBuffer, (int)offset, (int)(data.getMaxCount.get() - offset)); + } + + @Override + public OutputStream newOutputStream() { + + return data; + } + + @Override + public SplitableData split(long offset, long size) { + + final AccessibleByteArrayOutputStream newData = new AccessibleByteArrayOutputStream( + data.getSharedBuffer, data.setSharedBuffer, + data.getMaxCount, data.setMaxCount + ); + return new SplitByteBufferedData(newData, offset, size); + } + + private static class AccessibleByteArrayOutputStream extends ByteArrayOutputStream { + + private final Supplier getSharedBuffer; + private final Consumer setSharedBuffer; + private final Supplier getMaxCount; + private final Consumer setMaxCount; + + AccessibleByteArrayOutputStream( + final Supplier getSharedBuffer, + final Consumer setSharedBuffer, + final Supplier getMaxCount, + final Consumer setMaxCount) + { + + this.getSharedBuffer = getSharedBuffer; + this.setSharedBuffer = setSharedBuffer; + this.getMaxCount = getMaxCount; + this.setMaxCount = setMaxCount; + } + + byte[] getBuf() { + return getSharedBuffer.get(); + } + + void setCount(int count) { + this.count = count; + } + + @Override public synchronized void write(int b) { + + final byte[] sharedBufBeforeWrite = getSharedBuffer.get(); + + if (buf != sharedBufBeforeWrite) buf = sharedBufBeforeWrite; + super.write(b); + if (buf != sharedBufBeforeWrite) setSharedBuffer.accept(buf); + setMaxCount.accept(count); + } + + @Override public synchronized void write(byte[] b, int off, int len) { + + final byte[] sharedBufBeforeWrite = getSharedBuffer.get(); + if (buf != sharedBufBeforeWrite) buf = sharedBufBeforeWrite; + super.write(b, off, len); + if (buf != sharedBufBeforeWrite) setSharedBuffer.accept(buf); + setMaxCount.accept(count); + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java b/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java new file mode 100644 index 000000000..e6f5d40d7 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java @@ -0,0 +1,72 @@ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public class SplitKeyValueAccessData implements SplitableData { + + private final KeyValueAccess access; + private final String key; + private final long offset; + private final long size; + + public SplitKeyValueAccessData( + final KeyValueAccess access, + final String key) throws IOException { + this.access = access; + this.key = key; + this.offset = 0; + long keySize; + try { + /*If the file exists we need to know the real size, in case we need to read the + * Index N bytes from the end, for example. Potentially we could do this lazily */ + keySize = access.size(key); + } catch (N5Exception.N5NoSuchKeyException e) { + keySize = 0; //TODO Caleb: 0? -1? ??? + } + this.size = keySize; + } + + public SplitKeyValueAccessData( + final KeyValueAccess access, + final String key, + final long offset, + final long size){ + + this.access = access; + this.key = key; + this.offset = offset; + this.size = size; + } + + @Override + public long getOffset() { + + return offset; + } + + @Override + public long getSize() { + + return size; + } + + @Override + public InputStream newInputStream() throws IOException { + //TODO Caleb: Should wrap with BoundedInputStream? + return access.lockForReading(key, offset, size).newInputStream(); + } + + @Override + public OutputStream newOutputStream() throws IOException { + //TODO Caleb: If 0 -> -1? to handle non-existent files + return access.lockForWriting(key, offset, size).newOutputStream(); + } + + @Override + public SplitableData split(long offset, long size) { + + return new SplitKeyValueAccessData(access, key, getOffset() + offset, size); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/SplitableData.java b/src/main/java/org/janelia/saalfeldlab/n5/SplitableData.java new file mode 100644 index 000000000..86c152c2a --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/SplitableData.java @@ -0,0 +1,21 @@ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public interface SplitableData { + + // TODO Caleb: + // Do we need a getter for this? If it's intended to be relative + // after a split, shouldn't offset always be (locally) 0? + // maybe only need (offset) during #split + long getOffset(); + + long getSize(); + + InputStream newInputStream() throws IOException; + OutputStream newOutputStream() throws IOException; + + SplitableData split(long offset, long size); +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index ed5442571..936bade1a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -1,5 +1,13 @@ package org.janelia.saalfeldlab.n5.codec; +import org.apache.commons.io.input.ProxyInputStream; +import org.apache.commons.io.output.ProxyOutputStream; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.SplitableData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -8,16 +16,6 @@ import java.io.Serializable; import java.io.UncheckedIOException; import java.util.Arrays; - -import org.apache.commons.io.input.ProxyInputStream; -import org.apache.commons.io.output.ProxyOutputStream; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.serialization.NameConfig; - /** * Interface representing a filter can encode a {@link OutputStream}s when writing data, and decode * the {@link InputStream}s when reading data. @@ -28,7 +26,8 @@ @NameConfig.Prefix("codec") public interface Codec extends Serializable { - static OutputStream encode(OutputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { + public static OutputStream encode(OutputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { + OutputStream stream = out; for (final BytesCodec codec : bytesCodecs) stream = codec.encode(stream); @@ -36,7 +35,8 @@ static OutputStream encode(OutputStream out, Codec.BytesCodec... bytesCodecs) th return stream; } - static InputStream decode(InputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { + public static InputStream decode(InputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { + InputStream stream = out; for (final BytesCodec codec : bytesCodecs) stream = codec.decode(stream); @@ -44,7 +44,7 @@ static InputStream decode(InputStream out, Codec.BytesCodec... bytesCodecs) thro return stream; } - interface BytesCodec extends Codec { + public interface BytesCodec extends Codec { /** * Decode an {@link InputStream}. @@ -52,7 +52,7 @@ interface BytesCodec extends Codec { * @param in input stream * @return the decoded input stream */ - InputStream decode(final InputStream in) throws IOException; + public InputStream decode(final InputStream in) throws IOException; /** * Encode an {@link OutputStream}. @@ -60,7 +60,7 @@ interface BytesCodec extends Codec { * @param out the output stream * @return the encoded output stream */ - OutputStream encode(final OutputStream out) throws IOException; + public OutputStream encode(final OutputStream out) throws IOException; } interface ArrayCodec extends DeterministicSizeCodec { @@ -74,25 +74,21 @@ default long[] getPositionForBlock(final DatasetAttributes attributes, final lon return blockPosition; } + /** * Decode an {@link InputStream}. * * @param in input stream * @return the DataBlock corresponding to the input stream */ - DataBlockInputStream decode( - final DatasetAttributes attributes, - final long[] gridPosition, - final InputStream in) throws IOException; + public DataBlockInputStream decode(final DatasetAttributes attributes, final long[] gridPosition, final InputStream in) throws IOException; /** * Encode a {@link DataBlock}. * * @param datablock the datablock to encode */ - DataBlockOutputStream encode( - final DatasetAttributes attributes, - final DataBlock datablock, + public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock datablock, final OutputStream out) throws IOException; @Override default long encodedSize(long size) { @@ -104,68 +100,64 @@ DataBlockOutputStream encode( return size; } + default void writeBlock( - final KeyValueAccess kva, - final String keyPath, + final SplitableData splitData, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) { - try (final LockedChannel lock = kva.lockForWriting(keyPath)) { - try (final OutputStream out = lock.newOutputStream()) { - final DataBlockOutputStream dataBlockOutput = encode(datasetAttributes, dataBlock, out); - try (final OutputStream stream = Codec.encode(dataBlockOutput, datasetAttributes.getCodecs())) { - dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); - } + final SplitableData truncateExisting = splitData.split(0, Long.MAX_VALUE); + try (final OutputStream out = truncateExisting.newOutputStream()) { + final DataBlockOutputStream dataBlockOutput = encode(datasetAttributes, dataBlock, out); + try (final OutputStream stream = Codec.encode(dataBlockOutput, datasetAttributes.getCodecs())) { + dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); } } catch (final IOException | UncheckedIOException e) { - final String msg = "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + keyPath; - throw new N5Exception.N5IOException( msg, e); + final String msg = "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()); + throw new N5Exception.N5IOException(msg, e); } } default DataBlock readBlock( - final KeyValueAccess kva, - final String keyPath, + final SplitableData splitData, final DatasetAttributes datasetAttributes, final long[] gridPosition) { - try (final LockedChannel lockedChannel = kva.lockForReading(keyPath)) { - try(final InputStream in = lockedChannel.newInputStream()) { - - final BytesCodec[] codecs = datasetAttributes.getCodecs(); - final ArrayCodec arrayCodec = datasetAttributes.getArrayCodec(); - final DataBlockInputStream dataBlockStream = arrayCodec.decode(datasetAttributes, gridPosition, in); - InputStream stream = Codec.decode(dataBlockStream, codecs); + try (final InputStream in = splitData.newInputStream()) { + final BytesCodec[] codecs = datasetAttributes.getCodecs(); + final ArrayCodec arrayCodec = datasetAttributes.getArrayCodec(); + final DataBlockInputStream dataBlockStream = arrayCodec.decode(datasetAttributes, gridPosition, in); + InputStream stream = Codec.decode(dataBlockStream, codecs); - final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); - dataBlock.readData(dataBlockStream.getDataInput(stream)); - stream.close(); + final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); + dataBlock.readData(dataBlockStream.getDataInput(stream)); + stream.close(); - return dataBlock; - } + return dataBlock; } catch (final N5Exception.N5NoSuchKeyException e) { + //TODO Caleb: Not sure these catch(s) belong here return null; } catch (final IOException | UncheckedIOException e) { - final String msg = "Failed to read block " + Arrays.toString(gridPosition) + " from dataset " + keyPath; - throw new N5Exception.N5IOException( msg, e); + //TODO Caleb: Not sure these catch(s) belong here + final String msg = "Failed to read block " + Arrays.toString(gridPosition); + throw new N5Exception.N5IOException(msg, e); } } } abstract class DataBlockInputStream extends ProxyInputStream { - protected DataBlockInputStream(InputStream in) { super(in); } - public abstract DataBlock allocateDataBlock() throws IOException; + public abstract DataBlock allocateDataBlock() throws IOException; public abstract DataInput getDataInput(final InputStream inputStream); } - abstract class DataBlockOutputStream extends ProxyOutputStream { + public abstract class DataBlockOutputStream extends ProxyOutputStream { protected DataBlockOutputStream(final OutputStream out) { @@ -175,6 +167,6 @@ protected DataBlockOutputStream(final OutputStream out) { public abstract DataOutput getDataOutput(final OutputStream outputStream); } - String getType(); + public String getType(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index c7274d850..6a1c03976 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -1,22 +1,17 @@ package org.janelia.saalfeldlab.n5.shard; -import org.apache.commons.io.input.BoundedInputStream; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.io.output.CountingOutputStream; import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.DefaultBlockReader; import org.janelia.saalfeldlab.n5.DefaultBlockWriter; -import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.SplitByteBufferedData; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.janelia.saalfeldlab.n5.util.Position; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.List; @@ -34,15 +29,15 @@ public class InMemoryShard extends AbstractShard { * Use morton- or c-ording instead of writing blocks out in the order they're added? * (later) */ - public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] shardPosition) { + public InMemoryShard(final A datasetAttributes, final long[] shardPosition) { - this( datasetAttributes, shardPosition, null); + this(datasetAttributes, shardPosition, null); indexBuilder = new ShardIndexBuilder(this); final IndexLocation indexLocation = ((ShardingCodec)datasetAttributes.getArrayCodec()).getIndexLocation(); indexBuilder.indexLocation(indexLocation); } - public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] gridPosition, + public InMemoryShard(final A datasetAttributes, final long[] gridPosition, ShardIndex index) { super(datasetAttributes, gridPosition, index); @@ -66,7 +61,7 @@ private void storeBlock(DataBlock block) { @Override public void writeBlock(DataBlock block) { - + addBlock(block); } @@ -79,23 +74,23 @@ public int numBlocks() { return blocks.size(); } - + @Override public List> getBlocks() { return new ArrayList<>(blocks.values()); } - public List> getBlocks( int[] blockIndexes ) { + public List> getBlocks(int[] blockIndexes) { final ArrayList> out = new ArrayList<>(); final int[] blocksPerShard = getDatasetAttributes().getBlocksPerShard(); - long[] position = new long[ getSize().length ]; - for( int idx : blockIndexes ) { + long[] position = new long[getSize().length]; + for (int idx : blockIndexes) { GridIterator.indexToPosition(idx, blocksPerShard, position); DataBlock blk = getBlock(position); - if( blk != null ) + if (blk != null) out.add(blk); } return out; @@ -112,21 +107,12 @@ protected IndexLocation indexLocation() { @Override public ShardIndex getIndex() { - if( index != null ) + if (index != null) return index; else return indexBuilder.build(); } - public void write(final KeyValueAccess keyValueAccess, final String path) throws IOException { - - try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(path)) { - try (final OutputStream os = lockedChannel.newOutputStream()) { - write(os); - } - } - } - public void write(final OutputStream out) throws IOException { if (indexLocation() == IndexLocation.END) @@ -135,84 +121,10 @@ public void write(final OutputStream out) throws IOException { writeShardStart(out, this); } - public static InMemoryShard readShard( - final KeyValueAccess kva, final String key, final long[] gridPosition, final DatasetAttributes attributes) - throws IOException { - - try (final LockedChannel lockedChannel = kva.lockForReading(key)) { - try (final InputStream is = lockedChannel.newInputStream()) { - return readShard(is, gridPosition, attributes); - } - } - - // Another possible implementation -// return fromShard(new VirtualShard<>(attributes, gridPosition, kva, key)); - } - - @SuppressWarnings("hiding") - public static InMemoryShard readShard( - final InputStream inputStream, final long[] gridPosition, final DatasetAttributes attributes) throws IOException { - - try (ByteArrayOutputStream result = new ByteArrayOutputStream()) { - byte[] buffer = new byte[1024]; - for (int length; (length = inputStream.read(buffer)) != -1;) { - result.write(buffer, 0, length); - } - return readShard(result.toByteArray(), gridPosition, attributes); - } - } - - public static InMemoryShard readShard( - final byte[] data, - long[] shardPosition, final DatasetAttributes attributes) throws IOException { - - final ShardIndex index = ((ShardingCodec)attributes.getArrayCodec()).createIndex(attributes); - ShardIndex.read(data, index); - - final InMemoryShard shard = new InMemoryShard(attributes, shardPosition, index); - final GridIterator it = new GridIterator(attributes.getBlocksPerShard()); - while (it.hasNext()) { - - final long[] p = it.next(); - final int[] pInt = GridIterator.long2int(p); - - if (index.exists(pInt)) { - - final ByteArrayInputStream is = new ByteArrayInputStream(data); - is.skip(index.getOffset(pInt)); - BoundedInputStream bIs = BoundedInputStream.builder().setInputStream(is) - .setMaxCount(index.getNumBytes(pInt)).get(); - - final long[] blockGridPosition = attributes.getBlockPositionFromShardPosition(shardPosition, p); - @SuppressWarnings("unchecked") - final DataBlock blk = (DataBlock) DefaultBlockReader.readBlock(bIs, attributes, - blockGridPosition); - shard.addBlock(blk); - bIs.close(); - } - } - - return shard; - } - - public static void writeShard(final KeyValueAccess keyValueAccess, final String path, final InMemoryShard shard) throws IOException { - - try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(path)) { - try (final OutputStream os = lockedChannel.newOutputStream()) { - writeShard(os, shard); - } - } - } - - public static void writeShard(final OutputStream out, final Shard shard) throws IOException { - - fromShard(shard).write(out); - } - public static InMemoryShard fromShard(Shard shard) { if (shard instanceof InMemoryShard) - return (InMemoryShard) shard; + return (InMemoryShard)shard; final InMemoryShard inMemoryShard = new InMemoryShard( shard.getDatasetAttributes(), @@ -233,8 +145,9 @@ protected static void writeShardEndStream( final ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); indexBuilder.setCodecs(shardingCodec.getIndexCodecs()); - // Necessary to stop `close()` when writing blocks from closing out base OutputStream + // Neccesary to stop `close()` when writing blocks from closing out base OutputStream final ProxyOutputStream nop = new ProxyOutputStream(out) { + @Override public void close() { //nop } @@ -248,7 +161,7 @@ protected static void writeShardEndStream( final long size = cout.getByteCount() - bytesWritten; bytesWritten = cout.getByteCount(); - indexBuilder.addBlock( block.getGridPosition(), size); + indexBuilder.addBlock(block.getGridPosition(), size); } ShardIndex.write(indexBuilder.build(), out); @@ -284,7 +197,14 @@ protected static void writeShardStart( final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); indexBuilder.indexLocation(IndexLocation.START); - indexBuilder.setCodecs(shardingCodec.getIndexCodecs()); + indexBuilder.setCodecs(((ShardingCodec)datasetAttributes.getArrayCodec()).getIndexCodecs()); + + final SplitByteBufferedData splitData = new SplitByteBufferedData(); + try (final OutputStream blocksOut = splitData.newOutputStream()) { + for (DataBlock block : shard.getBlocks()) { + //TODO + } + } final List blockData = new ArrayList<>(shard.numBlocks()); for (DataBlock block : shard.getBlocks()) { @@ -293,14 +213,14 @@ protected static void writeShardStart( blockData.add(os.toByteArray()); indexBuilder.addBlock(block.getGridPosition(), os.size()); - } + } try { final ByteArrayOutputStream os = new ByteArrayOutputStream(); ShardIndex.write(indexBuilder.build(), os); out.write(os.toByteArray()); - for( byte[] data : blockData ) + for (byte[] data : blockData) out.write(data); } catch (Exception e) { @@ -308,5 +228,5 @@ protected static void writeShardStart( } } - + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 3f55dfbc2..11797a2ce 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -1,12 +1,21 @@ package org.janelia.saalfeldlab.n5.shard; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; +import org.apache.commons.io.output.CountingOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.SplitByteBufferedData; +import org.janelia.saalfeldlab.n5.SplitableData; +import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.util.GridIterator; public interface Shard extends Iterable> { @@ -134,7 +143,7 @@ default Iterator blockPositionIterator() { ShardIndex getIndex(); - static Shard createEmpty(final A attributes, long... shardPosition) { + static Shard createEmpty(final DatasetAttributes attributes, long... shardPosition) { final long[] emptyIndex = new long[(int)(2 * attributes.getNumBlocks())]; Arrays.fill(emptyIndex, ShardIndex.EMPTY_INDEX_NBYTES); @@ -142,6 +151,41 @@ static Shard createEmpty(fi return new InMemoryShard(attributes, shardPosition, shardIndex); } + default InputStream getAsStream() throws IOException { + + final DatasetAttributes datasetAttributes = getDatasetAttributes(); + final SplitableData splitData = new SplitByteBufferedData(); + ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); + final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); + final Codec.BytesCodec[] codecs = shardingCodec.getCodecs(); + final ShardIndex index = ShardIndex.createIndex(datasetAttributes); + long blocksOffset = index.getLocation() == ShardingCodec.IndexLocation.START ? index.numBytes() : 0; + final SplitableData blocksSplitData = splitData.split(blocksOffset, -1); + try (final OutputStream blocksOut = blocksSplitData.newOutputStream()) { + for (DataBlock block : getBlocks()) { + try (final CountingOutputStream blockOut = new CountingOutputStream(blocksOut)) { + final Codec.DataBlockOutputStream dataBlockOutput = arrayCodec.encode(datasetAttributes, block, blockOut); + try (final OutputStream stream = Codec.encode(dataBlockOutput, codecs)) { + block.writeData(dataBlockOutput.getDataOutput(stream)); + } + index.set(blocksOffset, blockOut.getByteCount(), getBlockPosition(block.getGridPosition())); + blocksOffset += blockOut.getByteCount(); + } + } + } catch (final IOException | UncheckedIOException e) { + throw new N5Exception.N5IOException( + "Failed to write block to shard " + Arrays.toString(getGridPosition()), e); + } + final long indexOffset = index.getLocation() == ShardingCodec.IndexLocation.START ? 0 : blocksOffset; + try { + ShardIndex.write(splitData.split(indexOffset, index.numBytes()), index); + } catch (final IOException | UncheckedIOException e) { + throw new N5Exception.N5IOException( + "Failed to write index to shard " + Arrays.toString(getGridPosition()), e); + } + return splitData.newInputStream(); + } + class DataBlockIterator implements Iterator> { private final GridIterator it; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 655bf62c5..d1122ab77 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -6,11 +6,10 @@ import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.DefaultBlockReader; import org.janelia.saalfeldlab.n5.DefaultBlockWriter; -import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.SplitableData; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; @@ -24,7 +23,6 @@ import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.util.Arrays; -import java.util.stream.IntStream; public class ShardIndex extends LongArrayDataBlock { @@ -87,6 +85,7 @@ public long getOffset(int... gridPosition) { } public long getOffsetByBlockIndex(int index) { + return data[index * 2]; } @@ -141,11 +140,11 @@ public static boolean read(byte[] data, final ShardIndex index) { final IndexByteBounds byteBounds = byteBounds(index, data.length); final ByteArrayInputStream is = new ByteArrayInputStream(data); is.skip(byteBounds.start); - try { - BoundedInputStream bIs = BoundedInputStream.builder() - .setInputStream(is) - .setMaxCount(index.numBytes()).get(); + try { + BoundedInputStream bIs = BoundedInputStream.builder() + .setInputStream(is) + .setMaxCount(index.numBytes()).get(); read(bIs, index); return true; } catch (IOException e) { @@ -155,55 +154,35 @@ public static boolean read(byte[] data, final ShardIndex index) { public static void read(InputStream in, final ShardIndex index) throws IOException { - @SuppressWarnings("unchecked") - final DataBlock indexBlock = (DataBlock) DefaultBlockReader.readBlock(in, - index.getIndexAttributes(), index.gridPosition); + @SuppressWarnings("unchecked") final DataBlock indexBlock = (DataBlock)DefaultBlockReader.readBlock(in, index.getIndexAttributes(), index.gridPosition); final long[] indexData = indexBlock.getData(); System.arraycopy(indexData, 0, index.data, 0, index.data.length); } public static boolean read( - final KeyValueAccess keyValueAccess, - final String key, + final SplitableData indexData, final ShardIndex index ) { - try { - final IndexByteBounds byteBounds = byteBounds(index, keyValueAccess.size(key)); - try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(key, byteBounds.start, byteBounds.end)) { - try (final InputStream in = lockedChannel.newInputStream()) { - read(in,index); - return true; - } - } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to read shard index from " + key, e); - } - } catch (final IOException | N5Exception.N5NoSuchKeyException e) { + try (final InputStream in = indexData.newInputStream()) { + read(in, index); + return true; + } catch (final N5Exception.N5NoSuchKeyException e) { return false; + } catch (final IOException | UncheckedIOException e) { + throw new N5IOException("Failed to read shard index", e); } } public static void write( - final ShardIndex index, - final KeyValueAccess keyValueAccess, - final String key + final SplitableData indexData, + final ShardIndex index ) throws IOException { - final long start = index.location == IndexLocation.START ? 0 : sizeOrZero( keyValueAccess, key) ; - try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(key, start, index.numBytes())) { - try (final OutputStream os = lockedChannel.newOutputStream()) { - write(index, os); - } + try (final OutputStream os = indexData.newOutputStream()) { + write(index, os); } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to write shard index to " + key, e); - } - } - - private static long sizeOrZero(final KeyValueAccess keyValueAccess, final String key) { - try { - return keyValueAccess.size(key); - } catch (Exception e) { - return 0; + throw new N5IOException("Failed to write shard index", e); } } @@ -234,6 +213,7 @@ public static IndexByteBounds byteBounds(DatasetAttributes datasetAttributes, fi } public static IndexByteBounds byteBounds(final ShardIndex index, long objectSize) { + return byteBounds(index.numBytes(), index.location, objectSize); } @@ -306,7 +286,7 @@ public boolean equals(Object other) { if (other instanceof ShardIndex) { - final ShardIndex index = (ShardIndex) other; + final ShardIndex index = (ShardIndex)other; if (this.location != index.location) return false; @@ -319,5 +299,11 @@ public boolean equals(Object other) { } return true; } + + public static ShardIndex createIndex(final DatasetAttributes attributes) { + + ShardingCodec shardingCodec = (ShardingCodec)attributes.getArrayCodec(); + return shardingCodec.createIndex(attributes); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 4da31effb..53f18cdeb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -9,7 +9,7 @@ import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.SplitableData; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.serialization.N5Annotations; @@ -110,6 +110,7 @@ public DeterministicSizeCodec[] getIndexCodecs() { return attributes.getShardPositionForBlock(blockPosition); } + @Override public DataBlockInputStream decode(DatasetAttributes attributes, long[] gridPosition, InputStream in) throws IOException { return getArrayCodec().decode(attributes, gridPosition, in); @@ -120,19 +121,27 @@ public DeterministicSizeCodec[] getIndexCodecs() { return getArrayCodec().encode(attributes, dataBlock, out); } - @Override public void writeBlock(KeyValueAccess kva, String keyPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) { + @Override public void writeBlock( + final SplitableData splitData, + final DatasetAttributes datasetAttributes, + final DataBlock dataBlock) { final long[] shardPos = datasetAttributes.getShardPositionForBlock(dataBlock.getGridPosition()); - new VirtualShard(datasetAttributes, shardPos, kva, keyPath).writeBlock(dataBlock); + new VirtualShard(datasetAttributes, shardPos, splitData).writeBlock(dataBlock); } - @Override public DataBlock readBlock(final KeyValueAccess kva, final String keyPath, final DatasetAttributes datasetAttributes, final long... gridPosition) { + @Override + public DataBlock readBlock( + final SplitableData splitData, + final DatasetAttributes datasetAttributes, + final long... gridPosition) { final long[] shardPosition = datasetAttributes.getShardPositionForBlock(gridPosition); - return new VirtualShard(datasetAttributes, shardPosition, kva, keyPath).getBlock(gridPosition); + return new VirtualShard<>(datasetAttributes, shardPosition, splitData).getBlock(gridPosition); } public ShardIndex createIndex(final DatasetAttributes attributes) { + return new ShardIndex(attributes.getBlocksPerShard(), getIndexLocation(), getIndexCodecs()); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 3b115a900..d09e0ad31 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -1,73 +1,57 @@ package org.janelia.saalfeldlab.n5.shard; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.SplitableData; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.util.GridIterator; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; -import org.apache.commons.io.input.BoundedInputStream; -import org.apache.commons.io.input.ProxyInputStream; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.util.GridIterator; - public class VirtualShard extends AbstractShard { - final private KeyValueAccess keyValueAccess; - final private String path; + private final SplitableData splitableData; - public VirtualShard(final DatasetAttributes datasetAttributes, long[] gridPosition, - final KeyValueAccess keyValueAccess, final String path) { + public VirtualShard( + final DatasetAttributes datasetAttributes, + long[] gridPosition, + final SplitableData splitableData) { super(datasetAttributes, gridPosition, null); - this.keyValueAccess = keyValueAccess; - this.path = path; - } - - public VirtualShard(final DatasetAttributes datasetAttributes, long[] gridPosition) { - - this(datasetAttributes, gridPosition, null, null); + this.splitableData = splitableData; } @SuppressWarnings("unchecked") public DataBlock getBlock(InputStream in, long... blockGridPosition) throws IOException { ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); - final Codec.BytesCodec[] codecs = shardingCodec.getCodecs(); - final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); + final Codec.BytesCodec[] blockCodecs = shardingCodec.getCodecs(); + final Codec.ArrayCodec blockArrayCodec = shardingCodec.getArrayCodec(); - final ProxyInputStream proxyIn = new ProxyInputStream(in) { - @Override - public void close() { - //nop - } - }; - final Codec.DataBlockInputStream dataBlockStream = arrayCodec.decode(datasetAttributes, blockGridPosition, proxyIn); + final Codec.DataBlockInputStream dataBlockStream = blockArrayCodec.decode(datasetAttributes, blockGridPosition, in); - final InputStream stream = Codec.decode(in, codecs); final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); + final InputStream stream = Codec.decode(in, blockCodecs); dataBlock.readData(dataBlockStream.getDataInput(stream)); stream.close(); return dataBlock; - - } @Override - public List> getBlocks() { + public List> getBlocks() { + return getBlocks(IntStream.range(0, getNumBlocks()).toArray()); } @@ -97,34 +81,24 @@ public List> getBlocks(final int[] blockIndexes) { .mapToLong(i -> blocksPerShard[i] * getGridPosition()[i]) .toArray(); - long streamPosition = 0; - try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path)) { - try (final InputStream channelIn = lockedChannel.newInputStream()) { - - for (long[] offsetIndex : sortedOffsets) { - - final long offset = offsetIndex[0]; - if (offset < 0) - continue; - - final long idx = offsetIndex[1]; - GridIterator.indexToPosition(idx, blocksPerShard, blockGridMin, position); - - channelIn.skip(offset - streamPosition); - final long numBytes = index.getNumBytesByBlockIndex((int) idx); - final BoundedInputStream bIs = BoundedInputStream.builder().setInputStream(channelIn) - .setMaxCount(numBytes).get(); - - blocks.add(getBlock(bIs, position.clone())); - streamPosition = offset + numBytes; - } + for (long[] offsetIndex : sortedOffsets) { + final long offset = offsetIndex[0]; + if (offset < 0) + continue; + + final long idx = offsetIndex[1]; + GridIterator.indexToPosition(idx, blocksPerShard, blockGridMin, position); + + final long numBytes = index.getNumBytesByBlockIndex((int)idx); + try (final InputStream in = splitableData.split(offset, numBytes).newInputStream()) { + final DataBlock block = getBlock(in, position.clone()); + blocks.add(block); + } catch (final N5Exception.N5NoSuchKeyException e) { + return blocks; + } catch (final IOException | UncheckedIOException e) { + throw new N5IOException("Failed to read block from " + Arrays.toString(position), e); } - } catch (final N5Exception.N5NoSuchKeyException e) { - return blocks; - } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to read block from " + path, e); } - return blocks; } @@ -142,15 +116,13 @@ public DataBlock getBlock(long... blockGridPosition) { final long blockOffset = idx.getOffset(relativePosition); final long blockSize = idx.getNumBytes(relativePosition); - try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path, blockOffset, blockSize)) { - try ( final InputStream in = lockedChannel.newInputStream()) { - final long[] blockPosInImg = getDatasetAttributes().getBlockPositionFromShardPosition(getGridPosition(), blockGridPosition); - return getBlock( in, blockPosInImg ); - } + final long[] blockPosInImg = getDatasetAttributes().getBlockPositionFromShardPosition(getGridPosition(), blockGridPosition); + try (final InputStream in = splitableData.split(blockOffset, blockSize).newInputStream()) { + return getBlock(in, blockPosInImg); } catch (final N5Exception.N5NoSuchKeyException e) { return null; } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to read block from " + path, e); + throw new N5IOException("Failed to read block from " + Arrays.toString(blockGridPosition), e); } } @@ -162,34 +134,43 @@ public void writeBlock(final DataBlock block) { throw new N5IOException("Attempted to write block in the wrong shard."); final ShardIndex index = getIndex(); - long startByte = 0; - try { - startByte = keyValueAccess.size(path); - } catch (N5Exception.N5NoSuchKeyException e) { - startByte = index.getLocation() == ShardingCodec.IndexLocation.START ? index.numBytes() : 0; + final long blockOffset; + + //TODO Caleb: is it safe to assume we can ALWAYS write at an offset into a value that doesn't exist yet? + // Files seem to fill the beginning with 0, not sure how backends handle this. + // Otherwise, may need to explicitly write an empty index first. + if (index.getLocation() == ShardingCodec.IndexLocation.START) + blockOffset = splitableData.getSize() == 0 ? index.numBytes() : splitableData.getSize(); + else + blockOffset = splitableData.getSize(); + + final long size = Long.MAX_VALUE - blockOffset; + final SplitableData data = splitableData.split(blockOffset, size); //TODO Caleb: Should ideally remove offset also, but would need it to be absolute + + final long sizeWritten; + try (final OutputStream channelOut = data.newOutputStream()) { + try (final CountingOutputStream out = new CountingOutputStream(channelOut)) { + writeBlock(out, datasetAttributes, block); + + /* Update and write the index to the shard*/ + sizeWritten = out.getNumBytes(); + index.set(blockOffset, sizeWritten, relativePosition); + } } catch (IOException e) { - throw new N5IOException(e); + throw new N5IOException("Failed to write block to shard ", e); } - final long size = Long.MAX_VALUE - startByte; - - try (final LockedChannel lockedChannel = keyValueAccess.lockForWriting(path, startByte, size)) { - try ( final OutputStream channelOut = lockedChannel.newOutputStream()) { - try (final CountingOutputStream out = new CountingOutputStream(channelOut)) {; - writeBlock(out, datasetAttributes, block); - /* Update and write the index to the shard*/ - index.set(startByte, out.getNumBytes(), relativePosition); - } - } - } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to write block to shard " + path, e); - } + final long indexOffset = index.getLocation() == ShardingCodec.IndexLocation.START ? 0 : blockOffset + sizeWritten; try { - ShardIndex.write(index, keyValueAccess, path); + final SplitableData newSplitData = splitableData.split( + indexOffset, + index.numBytes()); + ShardIndex.write(newSplitData, index); } catch (IOException e) { - throw new N5IOException("Failed to write index to shard " + path, e); + throw new N5IOException("Failed to write index to shard ", e); } + } void writeBlock( @@ -206,6 +187,7 @@ void writeBlock( dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); stream.close(); } + public ShardIndex createIndex() { // Empty index of the correct size @@ -216,8 +198,8 @@ public ShardIndex createIndex() { public ShardIndex getIndex() { index = createIndex(); - ShardIndex.read(keyValueAccess, path, index); - + final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, splitableData.getSize()); + ShardIndex.read(splitableData.split(bounds.start, index.numBytes()), index); return index; } @@ -226,30 +208,35 @@ static class CountingOutputStream extends OutputStream { private long numBytes; public CountingOutputStream(OutputStream out) { + this.out = out; this.numBytes = 0; } @Override public void write(int b) throws IOException { + out.write(b); numBytes++; } @Override public void write(byte[] b) throws IOException { + out.write(b); numBytes += b.length; } @Override public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); numBytes += len; } @Override public void flush() throws IOException { + out.flush(); } @@ -259,6 +246,7 @@ public void close() throws IOException { } public long getNumBytes() { + return numBytes; } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 9b1e08313..21a547cdf 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -53,6 +53,7 @@ import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; +import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -94,6 +95,7 @@ public abstract class AbstractN5Test { public final N5Writer createTempN5Writer() { +// return N5Factory.createWriter("n5:src/test/resources/test.n5"); try { return createTempN5Writer(tempN5Location()); } catch (URISyntaxException | IOException e) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index 0c8ee24ae..72ad068ba 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -1,16 +1,10 @@ package org.janelia.saalfeldlab.n5.shard; -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Paths; - import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5FSTest; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; -import org.janelia.saalfeldlab.n5.codec.RawBytes; +import org.janelia.saalfeldlab.n5.SplitKeyValueAccessData; +import org.janelia.saalfeldlab.n5.SplitableData; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; @@ -18,43 +12,48 @@ import org.junit.After; import org.junit.Test; +import java.io.IOException; +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; + public class ShardIndexTest { private static final N5FSTest tempN5Factory = new N5FSTest(); @After public void removeTempWriters() { + tempN5Factory.removeTempWriters(); } @Test public void testOffsetIndex() throws IOException { - int[] shardBlockGridSize = new int[]{5,4,3}; + int[] shardBlockGridSize = new int[]{5, 4, 3}; ShardIndex index = new ShardIndex( shardBlockGridSize, - IndexLocation.END, new RawBytes()); + IndexLocation.END, new BytesCodec()); GridIterator it = new GridIterator(shardBlockGridSize); int i = 0; - while( it.hasNext()) { + while (it.hasNext()) { int j = index.getOffsetIndex(GridIterator.long2int(it.next())); assertEquals(i, j); - i+=2; + i += 2; } - - - shardBlockGridSize = new int[]{5,4,3,13}; + + shardBlockGridSize = new int[]{5, 4, 3, 13}; index = new ShardIndex( shardBlockGridSize, - IndexLocation.END, new RawBytes()); + IndexLocation.END, new BytesCodec()); it = new GridIterator(shardBlockGridSize); i = 0; - while( it.hasNext()) { + while (it.hasNext()) { int j = index.getOffsetIndex(GridIterator.long2int(it.next())); assertEquals(i, j); - i+=2; + i += 2; } } @@ -62,25 +61,28 @@ public void testOffsetIndex() throws IOException { @Test public void testReadVirtual() throws IOException { - final N5KeyValueWriter writer = (N5KeyValueWriter) tempN5Factory.createTempN5Writer(); + final N5KeyValueWriter writer = (N5KeyValueWriter)tempN5Factory.createTempN5Writer(); final KeyValueAccess kva = writer.getKeyValueAccess(); - final int[] shardBlockGridSize = new int[] { 6, 5 }; + final int[] shardBlockGridSize = new int[]{6, 5}; final IndexLocation indexLocation = IndexLocation.END; - final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { new RawBytes(), - new Crc32cChecksumCodec() }; + final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[]{new BytesCodec(), + new Crc32cChecksumCodec()}; final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "0").toString(); final ShardIndex index = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); - index.set(0, 6, new int[] { 0, 0 }); - index.set(19, 32, new int[] { 1, 0 }); - index.set(93, 111, new int[] { 3, 0 }); - index.set(143, 1, new int[] { 1, 2 }); - ShardIndex.write(index, kva, path); + index.set(0, 6, new int[]{0, 0}); + index.set(19, 32, new int[]{1, 0}); + index.set(93, 111, new int[]{3, 0}); + index.set(143, 1, new int[]{1, 2}); + final SplitKeyValueAccessData splitableData = new SplitKeyValueAccessData(kva, path); + final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, splitableData.getSize()); + final SplitableData indexData = splitableData.split(bounds.start, index.numBytes()); + ShardIndex.write(indexData, index); final ShardIndex other = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); - ShardIndex.read(kva, path, other); + ShardIndex.read(indexData, other); assertEquals(index, other); } @@ -88,37 +90,29 @@ public void testReadVirtual() throws IOException { @Test public void testReadInMemory() throws IOException { - final N5KeyValueWriter writer = (N5KeyValueWriter) tempN5Factory.createTempN5Writer(); + final N5KeyValueWriter writer = (N5KeyValueWriter)tempN5Factory.createTempN5Writer(); final KeyValueAccess kva = writer.getKeyValueAccess(); - final int[] shardBlockGridSize = new int[] { 6, 5 }; + final int[] shardBlockGridSize = new int[]{6, 5}; final IndexLocation indexLocation = IndexLocation.END; - final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { - new RawBytes(), - new Crc32cChecksumCodec() }; + final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[]{ + new BytesCodec(), + new Crc32cChecksumCodec()}; final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "indexTest").toString(); final ShardIndex index = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); - index.set(0, 6, new int[] { 0, 0 }); - index.set(19, 32, new int[] { 1, 0 }); - index.set(93, 111, new int[] { 3, 0 }); - index.set(143, 1, new int[] { 1, 2 }); - ShardIndex.write(index, kva, path); + index.set(0, 6, new int[]{0, 0}); + index.set(19, 32, new int[]{1, 0}); + index.set(93, 111, new int[]{3, 0}); + index.set(143, 1, new int[]{1, 2}); + SplitKeyValueAccessData splitableData = new SplitKeyValueAccessData(kva, path); + final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, splitableData.getSize()); + final SplitableData indexData = splitableData.split(bounds.start, index.numBytes()); + ShardIndex.write(indexData, index); final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); - ShardIndex.read(rawBytes(kva, path), indexRead); + ShardIndex.read(indexData, indexRead); assertEquals(index, indexRead); } - - private static byte[] rawBytes(KeyValueAccess kva, String path) throws IOException { - - final byte[] rawBytes = new byte[(int) kva.size(path)]; - try (final LockedChannel lockedChannel = kva.lockForReading(path)) { - try (final InputStream is = lockedChannel.newInputStream()) { - is.read(rawBytes); - } - } - return rawBytes; - } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index b19e07e5f..3a763392c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -4,34 +4,34 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.GzipCompression; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5FSTest; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.SplitKeyValueAccessData; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; -import org.janelia.saalfeldlab.n5.codec.RawBytes; +import org.janelia.saalfeldlab.n5.codec.ZarrBlockCodec; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; -import org.janelia.saalfeldlab.n5.util.GridIterator; +import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.junit.After; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.io.IOException; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; -import static org.junit.Assert.assertArrayEquals; - @RunWith(Parameterized.class) public class ShardTest { @@ -41,9 +41,15 @@ public class ShardTest { public static Collection data() { final ArrayList params = new ArrayList<>(); - for (IndexLocation indexLoc : IndexLocation.values()) { - for (ByteOrder blockByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { - for (ByteOrder indexByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { + // final IndexLocation[] locs = new IndexLocation[]{IndexLocation.END}; + // final ByteOrder[] byteCodecs = {ByteOrder.BIG_ENDIAN}; + // final ByteOrder[] indexCodecs = {ByteOrder.BIG_ENDIAN}; + final IndexLocation[] locs = IndexLocation.values(); + final ByteOrder[] byteCodecs = {ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}; + final ByteOrder[] indexCodecs = {ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}; + for (IndexLocation indexLoc : locs) { + for (ByteOrder blockByteOrder : byteCodecs) { + for (ByteOrder indexByteOrder : indexCodecs) { params.add(new Object[]{indexLoc, blockByteOrder, indexByteOrder}); } } @@ -77,8 +83,8 @@ private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, DataType.UINT8, new ShardingCodec( blockSize, - new Codec[]{new N5BlockCodec(dataByteOrder)}, //, new GzipCompression(4)}, - new DeterministicSizeCodec[]{new RawBytes(indexByteOrder), new Crc32cChecksumCodec()}, + new Codec[]{new N5BlockCodec(dataByteOrder), new GzipCompression(4)}, + new DeterministicSizeCodec[]{new ZarrBlockCodec(indexByteOrder), new Crc32cChecksumCodec()}, indexLocation ) ); @@ -90,7 +96,7 @@ private DatasetAttributes getTestAttributes() { } @Test - public void writeReadBlocksTest() { + public void writeReadBlocksTest() throws IOException { final N5Writer writer = tempN5Factory.createTempN5Writer(); final DatasetAttributes datasetAttributes = getTestAttributes( @@ -99,7 +105,9 @@ public void writeReadBlocksTest() { new int[]{2, 2} ); - writer.createDataset("shard", datasetAttributes); + final String dataset = "writeReadBlocks"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); final int[] blockSize = datasetAttributes.getBlockSize(); final int numElements = blockSize[0] * blockSize[1]; @@ -110,7 +118,7 @@ public void writeReadBlocksTest() { } writer.writeBlocks( - "shard", + dataset, datasetAttributes, /* shard (0, 0) */ new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), @@ -128,10 +136,15 @@ public void writeReadBlocksTest() { final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); + final String shardKey = ((N5KeyValueWriter)writer).absoluteDataBlockPath(dataset, 2, 2); + final SplitKeyValueAccessData splitData = new SplitKeyValueAccessData(kva, shardKey); + final VirtualShard vs = new VirtualShard<>(datasetAttributes, new long[]{2, 2}, splitData); + final List> blocks = vs.getBlocks(); + final String[][] keys = new String[][]{ - {"shard", "0", "0"}, - {"shard", "1", "0"}, - {"shard", "2", "2"} + {dataset, "0", "0"}, + {dataset, "1", "0"}, + {dataset, "2", "2"} }; for (String[] key : keys) { final String shard = kva.compose(writer.getURI(), key); @@ -140,7 +153,7 @@ public void writeReadBlocksTest() { final long[][] blockIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; for (long[] blockIndex : blockIndices) { - final DataBlock block = writer.readBlock("shard", datasetAttributes, blockIndex); + final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); } @@ -149,7 +162,7 @@ public void writeReadBlocksTest() { data2[i] = (byte)(10 + i); } writer.writeBlocks( - "shard", + dataset, datasetAttributes, /* shard (0, 0) */ new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data2), @@ -164,10 +177,10 @@ public void writeReadBlocksTest() { ); final String[][] keys2 = new String[][]{ - {"shard", "0", "0"}, - {"shard", "1", "0"}, - {"shard", "0", "1"}, - {"shard", "2", "2"} + {dataset, "0", "0"}, + {dataset, "1", "0"}, + {dataset, "0", "1"}, + {dataset, "2", "2"} }; for (String[] key : keys2) { final String shard = kva.compose(writer.getURI(), key); @@ -176,13 +189,13 @@ public void writeReadBlocksTest() { final long[][] oldBlockIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; for (long[] blockIndex : oldBlockIndices) { - final DataBlock block = writer.readBlock("shard", datasetAttributes, blockIndex); + final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); } final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; for (long[] blockIndex : newBlockIndices) { - final DataBlock block = writer.readBlock("shard", datasetAttributes, blockIndex); + final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])block.getData()); } } @@ -191,10 +204,12 @@ public void writeReadBlocksTest() { public void writeReadBlockTest() { final N5Writer writer = tempN5Factory.createTempN5Writer(); + // final N5Writer writer = N5Factory.createWriter("n5:src/test/resources/shardExamples/writeReadBlockTest.n5"); final DatasetAttributes datasetAttributes = getTestAttributes(); - writer.createDataset("shard", datasetAttributes); - writer.deleteBlock("shard", 0, 0); + final String dataset = "writeReadBlock"; + writer.createDataset(dataset, datasetAttributes); + writer.deleteBlock(dataset, 0, 0); //TODO Caleb: We are abusing this here. It shouldn't delete the entire shard.. final int[] blockSize = datasetAttributes.getBlockSize(); final DataType dataType = datasetAttributes.getDataType(); @@ -210,15 +225,15 @@ public void writeReadBlockTest() { for (int i = 0; i < data.length; i++) { data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); } - writer.writeBlock("shard", datasetAttributes, dataBlock); + writer.writeBlock(dataset, datasetAttributes, dataBlock); - final DataBlock block = writer.readBlock("shard", datasetAttributes, gridPosition.clone()); + final DataBlock block = writer.readBlock(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); for (Map.Entry entry : writtenBlocks.entrySet()) { final long[] otherGridPosition = entry.getKey(); final byte[] otherData = entry.getValue(); - final DataBlock otherBlock = writer.readBlock("shard", datasetAttributes, otherGridPosition); + final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); } @@ -233,8 +248,10 @@ public void writeReadShardTest() { final N5Writer writer = tempN5Factory.createTempN5Writer(); final DatasetAttributes datasetAttributes = getTestAttributes(); - writer.createDataset("wholeShard", datasetAttributes); - writer.deleteBlock("wholeShard", 0, 0); + + final String dataset = "writeReadShard"; + writer.createDataset(dataset, datasetAttributes); + writer.deleteBlock(dataset, 0, 0); final int[] blockSize = datasetAttributes.getBlockSize(); final DataType dataType = datasetAttributes.getDataType(); @@ -257,63 +274,51 @@ public void writeReadShardTest() { } } - writer.writeShard("wholeShard", datasetAttributes, shard); + writer.writeShard(dataset, datasetAttributes, shard); for (Map.Entry entry : writtenBlocks.entrySet()) { final long[] otherGridPosition = entry.getKey(); final byte[] otherData = entry.getValue(); - final DataBlock otherBlock = writer.readBlock("wholeShard", datasetAttributes, otherGridPosition); + final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); } } - @Test - @Ignore - public void writeReadNestedShards() { - - int[] blockSize = new int[]{4, 4}; - int N = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y); + public static void main(String[] args) { - final N5Writer writer = tempN5Factory.createTempN5Writer(); - final DatasetAttributes datasetAttributes = getNestedShardCodecsAttributes(blockSize); - writer.createDataset("nestedShards", datasetAttributes); + final long[] imageSize = new long[]{32, 27}; + final int[] shardSize = new int[]{16, 9}; + final int[] blockSize = new int[]{4, 3}; + final int numBlockElements = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y); - final byte[] data = new byte[N]; - Arrays.fill(data, (byte)4); + try (final N5Writer n5 = new N5Factory().openWriter("n5:/tmp/tests/codeReview/sharded.n5")) { - writer.writeBlocks("nestedShards", datasetAttributes, - new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), - new ByteArrayDataBlock(blockSize, new long[]{0, 2}, data), - new ByteArrayDataBlock(blockSize, new long[]{2, 1}, data)); + final DatasetAttributes attributes = getDatasetAttributes(imageSize, shardSize, blockSize); + } - assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 1, 1).getData()); - assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 0, 2).getData()); - assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 2, 1).getData()); } - private DatasetAttributes getNestedShardCodecsAttributes(int[] blockSize) { + private static DatasetAttributes getDatasetAttributes(long[] imageSize, int[] shardSize, int[] blockSize) { - final int[] innerShardSize = new int[]{2 * blockSize[0], 2 * blockSize[1]}; - final int[] shardSize = new int[]{4 * blockSize[0], 4 * blockSize[1]}; - final long[] dimensions = GridIterator.int2long(shardSize); - - // TODO: its not even clear how we build this given - // this constructor. Is the block size of the sharded dataset attributes - // the innermost (block) size or the intermediate shard size? - // probably better to forget about this class - only use DatasetAttributes - // and detect shading in another way - final ShardingCodec innerShard = new ShardingCodec(innerShardSize, - new Codec[]{new RawBytes()}, - new DeterministicSizeCodec[]{new RawBytes(indexByteOrder), new Crc32cChecksumCodec()}, - IndexLocation.START); - - return new DatasetAttributes( - dimensions, shardSize, blockSize, DataType.UINT8, + final DatasetAttributes attributes = new DatasetAttributes( + imageSize, + shardSize, + blockSize, + DataType.INT32, new ShardingCodec( blockSize, - new Codec[]{innerShard}, - new DeterministicSizeCodec[]{new RawBytes(indexByteOrder), new Crc32cChecksumCodec()}, - IndexLocation.END) - ); + new Codec[]{ + // codecs applied to image data + new ZarrBlockCodec(ByteOrder.BIG_ENDIAN), + }, + new DeterministicSizeCodec[]{ + // codecs applied to the shard index, must not be compressors + new ZarrBlockCodec(ByteOrder.LITTLE_ENDIAN), + new Crc32cChecksumCodec() + }, + IndexLocation.START + ) + ); + return attributes; } } diff --git a/src/test/resources/shardExamples/writeReadBlockTest.n5/attributes.json b/src/test/resources/shardExamples/writeReadBlockTest.n5/attributes.json new file mode 100644 index 000000000..6c7c6d59d --- /dev/null +++ b/src/test/resources/shardExamples/writeReadBlockTest.n5/attributes.json @@ -0,0 +1 @@ +{"n5":"4.0.0"} \ No newline at end of file diff --git a/src/test/resources/shardExamples/writeReadBlockTest.n5/shard/0/0 b/src/test/resources/shardExamples/writeReadBlockTest.n5/shard/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..ff1fcdac32429df87d739fc21c4242a3f1bf0a8a GIT binary patch literal 336 zcmZQzU|?c^;=KHV!v7=!pk8DkaPXegM4$lN%#_r$baW%38bYA701kPWxdK<(9v?wA wlZ%^&7l$6WnNZCuV0OR^gYo6EJSPL$aCb5=GBHEt{=n2gX`;>KXpCM00M$cpU;qFB literal 0 HcmV?d00001 diff --git a/src/test/resources/shardExamples/writeReadBlockTest.n5/shard/attributes.json b/src/test/resources/shardExamples/writeReadBlockTest.n5/shard/attributes.json new file mode 100644 index 000000000..7b444a415 --- /dev/null +++ b/src/test/resources/shardExamples/writeReadBlockTest.n5/shard/attributes.json @@ -0,0 +1 @@ +{"dimensions":[8,8],"blockSize":[2,2],"shardSize":[4,4],"dataType":"uint8","codecs":[{"name":"sharding_indexed","configuration":{"index_codecs":[{"name":"bytes","configuration":{"endian":"big"}},{"name":"crc32c"}],"codecs":[{"name":"n5bytes","configuration":{"endian":"big"}}],"index_location":"end","chunk_shape":[2,2]}}]} \ No newline at end of file From 7970281a6de4237f88cfca05d15ac193dca76b0e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 24 Jan 2025 16:28:53 -0500 Subject: [PATCH 161/423] refactor: more SplitData implementation --- .../saalfeldlab/n5/SplitByteBufferedData.java | 3 +- .../n5/SplitKeyValueAccessData.java | 6 +- .../saalfeldlab/n5/shard/InMemoryShard.java | 104 ------------------ .../janelia/saalfeldlab/n5/shard/Shard.java | 6 +- .../saalfeldlab/n5/shard/VirtualShard.java | 17 +-- 5 files changed, 19 insertions(+), 117 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/SplitByteBufferedData.java b/src/main/java/org/janelia/saalfeldlab/n5/SplitByteBufferedData.java index 3fe9b2d83..2afaec1e5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/SplitByteBufferedData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/SplitByteBufferedData.java @@ -119,8 +119,7 @@ private static class AccessibleByteArrayOutputStream extends ByteArrayOutputStre final Supplier getSharedBuffer, final Consumer setSharedBuffer, final Supplier getMaxCount, - final Consumer setMaxCount) - { + final Consumer setMaxCount) { this.getSharedBuffer = getSharedBuffer; this.setSharedBuffer = setSharedBuffer; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java b/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java index e6f5d40d7..403b415fc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5; +import javax.annotation.Nonnegative; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -65,8 +66,9 @@ public OutputStream newOutputStream() throws IOException { } @Override - public SplitableData split(long offset, long size) { + public SplitableData split(@Nonnegative long offset, long size) { - return new SplitKeyValueAccessData(access, key, getOffset() + offset, size); + final long newOffset = this.offset + offset; + return new SplitKeyValueAccessData(access, key, newOffset, size - newOffset); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 6a1c03976..790f5a8b0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -113,14 +113,6 @@ public ShardIndex getIndex() { return indexBuilder.build(); } - public void write(final OutputStream out) throws IOException { - - if (indexLocation() == IndexLocation.END) - writeShardEndStream(out, this); - else - writeShardStart(out, this); - } - public static InMemoryShard fromShard(Shard shard) { if (shard instanceof InMemoryShard) @@ -133,100 +125,4 @@ public static InMemoryShard fromShard(Shard shard) { shard.forEach(blk -> inMemoryShard.addBlock(blk)); return inMemoryShard; } - - protected static void writeShardEndStream( - final OutputStream out, - InMemoryShard shard ) throws IOException { - - final DatasetAttributes datasetAttributes = shard.getDatasetAttributes(); - - final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); - indexBuilder.indexLocation(IndexLocation.END); - final ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); - indexBuilder.setCodecs(shardingCodec.getIndexCodecs()); - - // Neccesary to stop `close()` when writing blocks from closing out base OutputStream - final ProxyOutputStream nop = new ProxyOutputStream(out) { - - @Override public void close() { - //nop - } - }; - - final CountingOutputStream cout = new CountingOutputStream(nop); - - long bytesWritten = 0; - for (DataBlock block : shard.getBlocks()) { - DefaultBlockWriter.writeBlock(cout, datasetAttributes, block); - final long size = cout.getByteCount() - bytesWritten; - bytesWritten = cout.getByteCount(); - - indexBuilder.addBlock(block.getGridPosition(), size); - } - - ShardIndex.write(indexBuilder.build(), out); - } - - protected static void writeShardEnd( - final OutputStream out, - InMemoryShard shard ) throws IOException { - - final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); - indexBuilder.indexLocation(IndexLocation.END); - final DatasetAttributes datasetAttributes = shard.getDatasetAttributes(); - final ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); - indexBuilder.setCodecs(shardingCodec.getIndexCodecs()); - - for (DataBlock block : shard.getBlocks()) { - final ByteArrayOutputStream os = new ByteArrayOutputStream(); - DefaultBlockWriter.writeBlock(os, datasetAttributes, block); - - indexBuilder.addBlock(block.getGridPosition(), os.size()); - out.write(os.toByteArray()); - } - - ShardIndex.write(indexBuilder.build(), out); - } - - protected static void writeShardStart( - final OutputStream out, - InMemoryShard shard ) throws IOException { - - final DatasetAttributes datasetAttributes = shard.getDatasetAttributes(); - final ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); - - final ShardIndexBuilder indexBuilder = new ShardIndexBuilder(shard); - indexBuilder.indexLocation(IndexLocation.START); - indexBuilder.setCodecs(((ShardingCodec)datasetAttributes.getArrayCodec()).getIndexCodecs()); - - final SplitByteBufferedData splitData = new SplitByteBufferedData(); - try (final OutputStream blocksOut = splitData.newOutputStream()) { - for (DataBlock block : shard.getBlocks()) { - //TODO - } - } - - final List blockData = new ArrayList<>(shard.numBlocks()); - for (DataBlock block : shard.getBlocks()) { - final ByteArrayOutputStream os = new ByteArrayOutputStream(); - DefaultBlockWriter.writeBlock(os, datasetAttributes, block); - - blockData.add(os.toByteArray()); - indexBuilder.addBlock(block.getGridPosition(), os.size()); - } - - try { - final ByteArrayOutputStream os = new ByteArrayOutputStream(); - ShardIndex.write(indexBuilder.build(), os); - out.write(os.toByteArray()); - - for (byte[] data : blockData) - out.write(data); - - } catch (Exception e) { - e.printStackTrace(); - } - - } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 11797a2ce..0ee9fa98f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -154,13 +154,17 @@ static Shard createEmpty(final DatasetAttributes attributes, long... shar default InputStream getAsStream() throws IOException { final DatasetAttributes datasetAttributes = getDatasetAttributes(); + final SplitableData splitData = new SplitByteBufferedData(); + ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); final Codec.BytesCodec[] codecs = shardingCodec.getCodecs(); + final ShardIndex index = ShardIndex.createIndex(datasetAttributes); long blocksOffset = index.getLocation() == ShardingCodec.IndexLocation.START ? index.numBytes() : 0; - final SplitableData blocksSplitData = splitData.split(blocksOffset, -1); + + final SplitableData blocksSplitData = splitData.split(blocksOffset, Long.MAX_VALUE); try (final OutputStream blocksOut = blocksSplitData.newOutputStream()) { for (DataBlock block : getBlocks()) { try (final CountingOutputStream blockOut = new CountingOutputStream(blocksOut)) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index d09e0ad31..6296d6fc7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -90,6 +90,7 @@ public List> getBlocks(final int[] blockIndexes) { GridIterator.indexToPosition(idx, blocksPerShard, blockGridMin, position); final long numBytes = index.getNumBytesByBlockIndex((int)idx); + //TODO Caleb: Do this with a single access (start at first offset, read through the last) try (final InputStream in = splitableData.split(offset, numBytes).newInputStream()) { final DataBlock block = getBlock(in, position.clone()); blocks.add(block); @@ -144,12 +145,11 @@ public void writeBlock(final DataBlock block) { else blockOffset = splitableData.getSize(); - final long size = Long.MAX_VALUE - blockOffset; - final SplitableData data = splitableData.split(blockOffset, size); //TODO Caleb: Should ideally remove offset also, but would need it to be absolute + final SplitableData blockData = splitableData.split(blockOffset, Long.MAX_VALUE); final long sizeWritten; - try (final OutputStream channelOut = data.newOutputStream()) { - try (final CountingOutputStream out = new CountingOutputStream(channelOut)) { + try (final OutputStream blockOut = blockData.newOutputStream()) { + try (final CountingOutputStream out = new CountingOutputStream(blockOut)) { writeBlock(out, datasetAttributes, block); /* Update and write the index to the shard*/ @@ -160,13 +160,13 @@ public void writeBlock(final DataBlock block) { throw new N5IOException("Failed to write block to shard ", e); } + //TODO Caleb: Could do END in the blockOut block, to avoid an additional access final long indexOffset = index.getLocation() == ShardingCodec.IndexLocation.START ? 0 : blockOffset + sizeWritten; + + final SplitableData indexData = splitableData.split( indexOffset, index.numBytes()); try { - final SplitableData newSplitData = splitableData.split( - indexOffset, - index.numBytes()); - ShardIndex.write(newSplitData, index); + ShardIndex.write(indexData, index); } catch (IOException e) { throw new N5IOException("Failed to write index to shard ", e); } @@ -197,6 +197,7 @@ public ShardIndex createIndex() { @Override public ShardIndex getIndex() { + //TODO Caleb: How to handle whne this shard doesn't exist (splitableData.getSize() <= 0) index = createIndex(); final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, splitableData.getSize()); ShardIndex.read(splitableData.split(bounds.start, index.numBytes()), index); From 4c0e0fe8171c9a8285202a6dff528f5302f69667 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 3 Feb 2025 14:20:32 -0500 Subject: [PATCH 162/423] fix: revert some changes that broke split data for ShardTest --- .../janelia/saalfeldlab/n5/SplitKeyValueAccessData.java | 3 +-- .../org/janelia/saalfeldlab/n5/shard/VirtualShard.java | 2 +- .../java/org/janelia/saalfeldlab/n5/shard/ShardTest.java | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java b/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java index 403b415fc..5e34552d0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java @@ -68,7 +68,6 @@ public OutputStream newOutputStream() throws IOException { @Override public SplitableData split(@Nonnegative long offset, long size) { - final long newOffset = this.offset + offset; - return new SplitKeyValueAccessData(access, key, newOffset, size - newOffset); + return new SplitKeyValueAccessData(access, key, getOffset() + offset, size); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 6296d6fc7..7fccf846b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -145,7 +145,7 @@ public void writeBlock(final DataBlock block) { else blockOffset = splitableData.getSize(); - final SplitableData blockData = splitableData.split(blockOffset, Long.MAX_VALUE); + final SplitableData blockData = splitableData.split(blockOffset, Long.MAX_VALUE - blockOffset); //TODO Caleb: Should ideally remove offset also, but would need it to be absolute final long sizeWritten; try (final OutputStream blockOut = blockData.newOutputStream()) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 3a763392c..b92550931 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -41,9 +41,9 @@ public class ShardTest { public static Collection data() { final ArrayList params = new ArrayList<>(); - // final IndexLocation[] locs = new IndexLocation[]{IndexLocation.END}; - // final ByteOrder[] byteCodecs = {ByteOrder.BIG_ENDIAN}; - // final ByteOrder[] indexCodecs = {ByteOrder.BIG_ENDIAN}; +// final IndexLocation[] locs = new IndexLocation[]{IndexLocation.END}; +// final ByteOrder[] byteCodecs = {ByteOrder.BIG_ENDIAN}; +// final ByteOrder[] indexCodecs = {ByteOrder.BIG_ENDIAN}; final IndexLocation[] locs = IndexLocation.values(); final ByteOrder[] byteCodecs = {ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}; final ByteOrder[] indexCodecs = {ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}; @@ -204,10 +204,10 @@ public void writeReadBlocksTest() throws IOException { public void writeReadBlockTest() { final N5Writer writer = tempN5Factory.createTempN5Writer(); - // final N5Writer writer = N5Factory.createWriter("n5:src/test/resources/shardExamples/writeReadBlockTest.n5"); final DatasetAttributes datasetAttributes = getTestAttributes(); final String dataset = "writeReadBlock"; + writer.remove(dataset); writer.createDataset(dataset, datasetAttributes); writer.deleteBlock(dataset, 0, 0); //TODO Caleb: We are abusing this here. It shouldn't delete the entire shard.. From f854ac64f3c53e1a767e3b87dc306d1fa397cf1f Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 4 Feb 2025 16:53:01 -0500 Subject: [PATCH 163/423] chore: rebase cleanup --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 54 +++--------- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 6 +- .../janelia/saalfeldlab/n5/codec/Codec.java | 25 +++--- .../saalfeldlab/n5/shard/InMemoryShard.java | 13 ++- .../saalfeldlab/n5/shard/ShardIndex.java | 2 + .../saalfeldlab/n5/shard/ShardingCodec.java | 2 +- .../saalfeldlab/n5/shard/VirtualShard.java | 8 +- .../saalfeldlab/n5/AbstractN5Test.java | 1 - .../saalfeldlab/n5/shard/ShardIndexTest.java | 11 +-- .../saalfeldlab/n5/shard/ShardTest.java | 85 ++++++++++++++----- 10 files changed, 111 insertions(+), 96 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 52f2aa7af..87c05cc8b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -118,13 +118,6 @@ default DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { - if (datasetAttributes instanceof ShardedDatasetAttributes) { - final ShardedDatasetAttributes shardedAttrs = (ShardedDatasetAttributes) datasetAttributes; - final long[] shardPosition = shardedAttrs.getShardPositionForBlock(gridPosition); - final Shard shard = readShard(pathName, shardedAttrs, shardPosition); - return shard.getBlock(gridPosition); - } - final SplitKeyValueAccessData splitData; try { splitData = new SplitKeyValueAccessData(getKeyValueAccess(), pathName); @@ -139,23 +132,20 @@ default DataBlock readBlock( } @Override - default List> readBlocks( + default List> readBlocks( final String pathName, final DatasetAttributes datasetAttributes, final List blockPositions) throws N5Exception { // TODO which interface should have this implementation? - if (datasetAttributes instanceof ShardParameters) { - - final ShardParameters shardAttributes = (ShardParameters)datasetAttributes; + if (datasetAttributes.getShardSize() != null) { /* Group by shard position */ - final Map> shardBlockMap = shardAttributes.groupBlockPositions(blockPositions); - final ArrayList> blocks = new ArrayList<>(); + final Map> shardBlockMap = datasetAttributes.groupBlockPositions(blockPositions); + final ArrayList> blocks = new ArrayList<>(); for( Entry> e : shardBlockMap.entrySet()) { - final Shard shard = readShard(pathName, (DatasetAttributes & ShardParameters) shardAttributes, - e.getKey().get()); + final Shard shard = readShard(pathName, datasetAttributes, e.getKey().get()); for (final long[] blkPosition : e.getValue()) { blocks.add(shard.getBlock(blkPosition)); @@ -163,8 +153,8 @@ default List> readBlocks( } return blocks; - } else - return GsonN5Reader.super.readBlocks(pathName, datasetAttributes, blockPositions); + } + return GsonN5Reader.super.readBlocks(pathName, datasetAttributes, blockPositions); } @Override @@ -180,6 +170,9 @@ default String[] list(final String pathName) throws N5Exception { /** * Constructs the path for a data block in a dataset at a given grid * position. + *
    + * If the gridPosition passed in refers to shard position + * in a sharded dataset, this will return the path to the shard key *

    * The returned path is * @@ -204,33 +197,6 @@ default String absoluteDataBlockPath( for (final long p : gridPosition) components[++i] = Long.toString(p); - return getKeyValueAccess().compose(getURI(), components); - } - - /** - * Constructs the path for a shard in a dataset at a given grid position. - *

    - * The returned path is - * - * 

    -	 * $basePath/datasetPathName/$shardPosition[0]/$shardPosition[1]/.../$shardPosition[n]
-	 *
    - *

    - * This is the file into which the shard will be stored. - * - * @param normalPath normalized dataset path - * @param shardGridPosition to the target shard - * @return the absolute path to the shard at shardGridPosition - */ - default String absoluteShardPath( - final String normalPath, - final long... shardGridPosition) { - - final String[] components = new String[shardGridPosition.length + 1]; - components[0] = normalPath; - int i = 0; - for (final long p : shardGridPosition) - components[++i] = Long.toString(p); return getKeyValueAccess().compose(getURI(), components); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 938c20b09..927881e35 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -232,8 +232,7 @@ default boolean removeAttributes(final String pathName, final List attri for( final Entry>> e : shardBlockMap.entrySet()) { final long[] shardPosition = e.getKey().get(); - final Shard currentShard = readShard(datasetPath, datasetAttributes, - shardPosition); + final Shard currentShard = readShard(datasetPath, datasetAttributes, shardPosition); final InMemoryShard newShard = InMemoryShard.fromShard(currentShard); for( DataBlock blk : e.getValue()) @@ -274,7 +273,8 @@ default void writeShard( final Shard shard) throws N5Exception { final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shard.getGridPosition()); - try (final OutputStream shardOut = new SplitKeyValueAccessData(getKeyValueAccess(), shardPath, 0, Long.MAX_VALUE).newOutputStream()) { + final SplitKeyValueAccessData splitData = new SplitKeyValueAccessData(getKeyValueAccess(), shardPath, 0, Long.MAX_VALUE); + try (final OutputStream shardOut = splitData.newOutputStream()) { try (final InputStream shardIn = shard.getAsStream()) { while (shardIn.available() > 0) { shardOut.write(shardIn.read()); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index 936bade1a..6b64420ed 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -26,7 +26,7 @@ @NameConfig.Prefix("codec") public interface Codec extends Serializable { - public static OutputStream encode(OutputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { + static OutputStream encode(OutputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { OutputStream stream = out; for (final BytesCodec codec : bytesCodecs) @@ -35,7 +35,7 @@ public static OutputStream encode(OutputStream out, Codec.BytesCodec... bytesCod return stream; } - public static InputStream decode(InputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { + static InputStream decode(InputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { InputStream stream = out; for (final BytesCodec codec : bytesCodecs) @@ -44,7 +44,7 @@ public static InputStream decode(InputStream out, Codec.BytesCodec... bytesCodec return stream; } - public interface BytesCodec extends Codec { + interface BytesCodec extends Codec { /** * Decode an {@link InputStream}. @@ -52,7 +52,7 @@ public interface BytesCodec extends Codec { * @param in input stream * @return the decoded input stream */ - public InputStream decode(final InputStream in) throws IOException; + InputStream decode(final InputStream in) throws IOException; /** * Encode an {@link OutputStream}. @@ -60,7 +60,7 @@ public interface BytesCodec extends Codec { * @param out the output stream * @return the encoded output stream */ - public OutputStream encode(final OutputStream out) throws IOException; + OutputStream encode(final OutputStream out) throws IOException; } interface ArrayCodec extends DeterministicSizeCodec { @@ -81,14 +81,19 @@ default long[] getPositionForBlock(final DatasetAttributes attributes, final lon * @param in input stream * @return the DataBlock corresponding to the input stream */ - public DataBlockInputStream decode(final DatasetAttributes attributes, final long[] gridPosition, final InputStream in) throws IOException; + DataBlockInputStream decode( + final DatasetAttributes attributes, + final long[] gridPosition, + final InputStream in) throws IOException; /** * Encode a {@link DataBlock}. * * @param datablock the datablock to encode */ - public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock datablock, + DataBlockOutputStream encode( + final DatasetAttributes attributes, + final DataBlock datablock, final OutputStream out) throws IOException; @Override default long encodedSize(long size) { @@ -152,12 +157,12 @@ protected DataBlockInputStream(InputStream in) { super(in); } - public abstract DataBlock allocateDataBlock() throws IOException; + public abstract DataBlock allocateDataBlock() throws IOException; public abstract DataInput getDataInput(final InputStream inputStream); } - public abstract class DataBlockOutputStream extends ProxyOutputStream { + abstract class DataBlockOutputStream extends ProxyOutputStream { protected DataBlockOutputStream(final OutputStream out) { @@ -167,6 +172,6 @@ protected DataBlockOutputStream(final OutputStream out) { public abstract DataOutput getDataOutput(final OutputStream outputStream); } - public String getType(); + String getType(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 790f5a8b0..6bfa889e2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -29,7 +29,7 @@ public class InMemoryShard extends AbstractShard { * Use morton- or c-ording instead of writing blocks out in the order they're added? * (later) */ - public InMemoryShard(final A datasetAttributes, final long[] shardPosition) { + public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] shardPosition) { this(datasetAttributes, shardPosition, null); indexBuilder = new ShardIndexBuilder(this); @@ -37,7 +37,7 @@ public InMemoryShard(final A dat indexBuilder.indexLocation(indexLocation); } - public InMemoryShard(final A datasetAttributes, final long[] gridPosition, + public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] gridPosition, ShardIndex index) { super(datasetAttributes, gridPosition, index); @@ -81,21 +81,20 @@ public List> getBlocks() { return new ArrayList<>(blocks.values()); } - public List> getBlocks(int[] blockIndexes) { + public List> getBlocks( int[] blockIndexes ) { final ArrayList> out = new ArrayList<>(); final int[] blocksPerShard = getDatasetAttributes().getBlocksPerShard(); - long[] position = new long[getSize().length]; - for (int idx : blockIndexes) { + long[] position = new long[ getSize().length ]; + for( int idx : blockIndexes ) { GridIterator.indexToPosition(idx, blocksPerShard, position); DataBlock blk = getBlock(position); - if (blk != null) + if( blk != null ) out.add(blk); } return out; } - protected IndexLocation indexLocation() { if (index != null) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index d1122ab77..73f40afad 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -22,7 +22,9 @@ import java.io.UncheckedIOException; import java.nio.channels.Channels; import java.nio.channels.FileChannel; +import org.janelia.saalfeldlab.n5.SplitableData; import java.util.Arrays; +import java.util.stream.IntStream; public class ShardIndex extends LongArrayDataBlock { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 53f18cdeb..69aa5e1b8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -137,7 +137,7 @@ public DataBlock readBlock( final long... gridPosition) { final long[] shardPosition = datasetAttributes.getShardPositionForBlock(gridPosition); - return new VirtualShard<>(datasetAttributes, shardPosition, splitData).getBlock(gridPosition); + return new VirtualShard(datasetAttributes, shardPosition, splitData).getBlock(gridPosition); } public ShardIndex createIndex(final DatasetAttributes attributes) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 7fccf846b..6ff449669 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -36,13 +36,13 @@ public VirtualShard( public DataBlock getBlock(InputStream in, long... blockGridPosition) throws IOException { ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); - final Codec.BytesCodec[] blockCodecs = shardingCodec.getCodecs(); - final Codec.ArrayCodec blockArrayCodec = shardingCodec.getArrayCodec(); + final Codec.BytesCodec[] codecs = shardingCodec.getCodecs(); + final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); - final Codec.DataBlockInputStream dataBlockStream = blockArrayCodec.decode(datasetAttributes, blockGridPosition, in); + final Codec.DataBlockInputStream dataBlockStream = arrayCodec.decode(datasetAttributes, blockGridPosition, in); final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); - final InputStream stream = Codec.decode(in, blockCodecs); + final InputStream stream = Codec.decode(in, codecs); dataBlock.readData(dataBlockStream.getDataInput(stream)); stream.close(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 21a547cdf..df3909612 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -95,7 +95,6 @@ public abstract class AbstractN5Test { public final N5Writer createTempN5Writer() { -// return N5Factory.createWriter("n5:src/test/resources/test.n5"); try { return createTempN5Writer(tempN5Location()); } catch (URISyntaxException | IOException e) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index 72ad068ba..0fbca0a6e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -6,6 +6,7 @@ import org.janelia.saalfeldlab.n5.SplitKeyValueAccessData; import org.janelia.saalfeldlab.n5.SplitableData; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; @@ -28,12 +29,12 @@ public void removeTempWriters() { } @Test - public void testOffsetIndex() throws IOException { + public void testOffsetIndex() { int[] shardBlockGridSize = new int[]{5, 4, 3}; ShardIndex index = new ShardIndex( shardBlockGridSize, - IndexLocation.END, new BytesCodec()); + IndexLocation.END, new RawBytes()); GridIterator it = new GridIterator(shardBlockGridSize); int i = 0; @@ -46,7 +47,7 @@ public void testOffsetIndex() throws IOException { shardBlockGridSize = new int[]{5, 4, 3, 13}; index = new ShardIndex( shardBlockGridSize, - IndexLocation.END, new BytesCodec()); + IndexLocation.END, new RawBytes()); it = new GridIterator(shardBlockGridSize); i = 0; @@ -66,7 +67,7 @@ public void testReadVirtual() throws IOException { final int[] shardBlockGridSize = new int[]{6, 5}; final IndexLocation indexLocation = IndexLocation.END; - final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[]{new BytesCodec(), + final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { new RawBytes(), new Crc32cChecksumCodec()}; final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "0").toString(); @@ -96,7 +97,7 @@ public void testReadInMemory() throws IOException { final int[] shardBlockGridSize = new int[]{6, 5}; final IndexLocation indexLocation = IndexLocation.END; final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[]{ - new BytesCodec(), + new RawBytes(), new Crc32cChecksumCodec()}; final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "indexTest").toString(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index b92550931..0bbd040fd 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -4,21 +4,21 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5FSTest; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.SplitKeyValueAccessData; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; -import org.janelia.saalfeldlab.n5.codec.ZarrBlockCodec; +import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.janelia.saalfeldlab.n5.util.GridIterator; import org.junit.After; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -29,9 +29,10 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; -import java.util.List; import java.util.Map; +import static org.junit.Assert.assertArrayEquals; + @RunWith(Parameterized.class) public class ShardTest { @@ -41,15 +42,9 @@ public class ShardTest { public static Collection data() { final ArrayList params = new ArrayList<>(); -// final IndexLocation[] locs = new IndexLocation[]{IndexLocation.END}; -// final ByteOrder[] byteCodecs = {ByteOrder.BIG_ENDIAN}; -// final ByteOrder[] indexCodecs = {ByteOrder.BIG_ENDIAN}; - final IndexLocation[] locs = IndexLocation.values(); - final ByteOrder[] byteCodecs = {ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}; - final ByteOrder[] indexCodecs = {ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}; - for (IndexLocation indexLoc : locs) { - for (ByteOrder blockByteOrder : byteCodecs) { - for (ByteOrder indexByteOrder : indexCodecs) { + for (IndexLocation indexLoc : IndexLocation.values()) { + for (ByteOrder blockByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { + for (ByteOrder indexByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { params.add(new Object[]{indexLoc, blockByteOrder, indexByteOrder}); } } @@ -83,8 +78,8 @@ private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, DataType.UINT8, new ShardingCodec( blockSize, - new Codec[]{new N5BlockCodec(dataByteOrder), new GzipCompression(4)}, - new DeterministicSizeCodec[]{new ZarrBlockCodec(indexByteOrder), new Crc32cChecksumCodec()}, + new Codec[]{new N5BlockCodec(dataByteOrder)}, //, new GzipCompression(4)}, + new DeterministicSizeCodec[]{new RawBytes(indexByteOrder), new Crc32cChecksumCodec()}, indexLocation ) ); @@ -136,10 +131,6 @@ public void writeReadBlocksTest() throws IOException { final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); - final String shardKey = ((N5KeyValueWriter)writer).absoluteDataBlockPath(dataset, 2, 2); - final SplitKeyValueAccessData splitData = new SplitKeyValueAccessData(kva, shardKey); - final VirtualShard vs = new VirtualShard<>(datasetAttributes, new long[]{2, 2}, splitData); - final List> blocks = vs.getBlocks(); final String[][] keys = new String[][]{ {dataset, "0", "0"}, @@ -284,6 +275,58 @@ public void writeReadShardTest() { } } + + + @Test + @Ignore + public void writeReadNestedShards() { + + int[] blockSize = new int[]{4, 4}; + int N = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y); + + final N5Writer writer = tempN5Factory.createTempN5Writer(); + final DatasetAttributes datasetAttributes = getNestedShardCodecsAttributes(blockSize); + writer.createDataset("nestedShards", datasetAttributes); + + final byte[] data = new byte[N]; + Arrays.fill(data, (byte)4); + + writer.writeBlocks("nestedShards", datasetAttributes, + new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + new ByteArrayDataBlock(blockSize, new long[]{0, 2}, data), + new ByteArrayDataBlock(blockSize, new long[]{2, 1}, data)); + + assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 1, 1).getData()); + assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 0, 2).getData()); + assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 2, 1).getData()); + } + + private DatasetAttributes getNestedShardCodecsAttributes(int[] blockSize) { + + final int[] innerShardSize = new int[]{2 * blockSize[0], 2 * blockSize[1]}; + final int[] shardSize = new int[]{4 * blockSize[0], 4 * blockSize[1]}; + final long[] dimensions = GridIterator.int2long(shardSize); + + // TODO: its not even clear how we build this given + // this constructor. Is the block size of the sharded dataset attributes + // the innermost (block) size or the intermediate shard size? + // probably better to forget about this class - only use DatasetAttributes + // and detect shading in another way + final ShardingCodec innerShard = new ShardingCodec(innerShardSize, + new Codec[]{new RawBytes()}, + new DeterministicSizeCodec[]{new RawBytes(indexByteOrder), new Crc32cChecksumCodec()}, + IndexLocation.START); + + return new DatasetAttributes( + dimensions, shardSize, blockSize, DataType.UINT8, + new ShardingCodec( + blockSize, + new Codec[]{innerShard}, + new DeterministicSizeCodec[]{new RawBytes(indexByteOrder), new Crc32cChecksumCodec()}, + IndexLocation.END) + ); + } + public static void main(String[] args) { final long[] imageSize = new long[]{32, 27}; @@ -309,11 +352,11 @@ private static DatasetAttributes getDatasetAttributes(long[] imageSize, int[] sh blockSize, new Codec[]{ // codecs applied to image data - new ZarrBlockCodec(ByteOrder.BIG_ENDIAN), + new RawBytes(ByteOrder.BIG_ENDIAN), }, new DeterministicSizeCodec[]{ // codecs applied to the shard index, must not be compressors - new ZarrBlockCodec(ByteOrder.LITTLE_ENDIAN), + new RawBytes(ByteOrder.LITTLE_ENDIAN), new Crc32cChecksumCodec() }, IndexLocation.START From 5b9c4a371ff83921f7f3ef48977cac00cf4dbd49 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 5 Feb 2025 10:10:22 -0500 Subject: [PATCH 164/423] fix: readblock with split data --- .../org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 87c05cc8b..2f9658f75 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -113,14 +113,18 @@ default Shard readShard( } @Override - default DataBlock readBlock( + default DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { + final long[] keyPos = datasetAttributes.getArrayCodec().getPositionForBlock(datasetAttributes, gridPosition); + final String keyPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), keyPos); + + final SplitKeyValueAccessData splitData; try { - splitData = new SplitKeyValueAccessData(getKeyValueAccess(), pathName); + splitData = new SplitKeyValueAccessData(getKeyValueAccess(), keyPath); } catch (IOException e) { throw new N5IOException(e); } From 231aacdffd6e9335734cca7fe8cbfed688297b0b Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 5 Feb 2025 10:10:42 -0500 Subject: [PATCH 165/423] test: easier to local debug --- .../saalfeldlab/n5/AbstractN5Test.java | 6 ++++-- .../saalfeldlab/n5/N5CachedFSTest.java | 4 ++-- .../saalfeldlab/n5/shard/ShardTest.java | 21 ++++++++++++++----- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index df3909612..fa10ab2eb 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -93,7 +93,7 @@ public abstract class AbstractN5Test { protected final HashSet tempWriters = new HashSet<>(); - public final N5Writer createTempN5Writer() { + public N5Writer createTempN5Writer() { try { return createTempN5Writer(tempN5Location()); @@ -142,7 +142,9 @@ protected N5Writer createN5Writer() throws IOException, URISyntaxException { protected N5Writer createN5Writer(final String location) throws IOException, URISyntaxException { - return createN5Writer(location, new GsonBuilder()); + final N5Factory factory = new N5Factory(); + return factory.openWriter(N5Factory.StorageFormat.ZARR, location); +// return createN5Writer(location, new GsonBuilder()); } /* Tests that overide this should enusre that the `N5Writer` created will remove its container on close() */ diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 330bc1c03..e85058505 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -102,8 +102,8 @@ public void cacheGroupDatasetTest() throws IOException, URISyntaxException { final String groupName = "gg"; final String tmpLocation = tempN5Location(); - try (N5KeyValueWriter w1 = (N5KeyValueWriter) createN5Writer(tmpLocation); - N5KeyValueWriter w2 = (N5KeyValueWriter) createN5Writer(tmpLocation);) { + try (GsonKeyValueN5Writer w1 = (GsonKeyValueN5Writer) createN5Writer(tmpLocation); + GsonKeyValueN5Writer w2 = (GsonKeyValueN5Writer) createN5Writer(tmpLocation);) { // create a group, both writers know it exists w1.createGroup(groupName); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 0bbd040fd..cfb9d0199 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -36,7 +36,20 @@ @RunWith(Parameterized.class) public class ShardTest { - private static final N5FSTest tempN5Factory = new N5FSTest(); + private static final boolean LOCAL_DEBUG = false; + + private static final N5FSTest tempN5Factory = new N5FSTest() { + + @Override public N5Writer createTempN5Writer() { + + if (LOCAL_DEBUG) { + final N5Writer writer = N5Factory.createWriter("src/test/resources/test.n5"); + writer.remove(""); //Clear old when starting new test + return writer; + } + return super.createTempN5Writer(); + } + }; @Parameterized.Parameters(name = "IndexLocation({0}), Block ByteOrder({1}), Index ByteOrder({2})") public static Collection data() { @@ -49,7 +62,8 @@ public static Collection data() { } } } - final Object[][] paramArray = new Object[params.size()][]; + final int numParams = LOCAL_DEBUG ? 1 : params.size(); + final Object[][] paramArray = new Object[numParams][]; Arrays.setAll(paramArray, params::get); return Arrays.asList(paramArray); } @@ -131,7 +145,6 @@ public void writeReadBlocksTest() throws IOException { final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); - final String[][] keys = new String[][]{ {dataset, "0", "0"}, {dataset, "1", "0"}, @@ -275,8 +288,6 @@ public void writeReadShardTest() { } } - - @Test @Ignore public void writeReadNestedShards() { From 24dc39398fdcb218f3f68d25b9ca1cfc302092d9 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 5 Feb 2025 10:11:33 -0500 Subject: [PATCH 166/423] test: don't override with zarr (oops) --- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index fa10ab2eb..b8083e06c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -142,9 +142,7 @@ protected N5Writer createN5Writer() throws IOException, URISyntaxException { protected N5Writer createN5Writer(final String location) throws IOException, URISyntaxException { - final N5Factory factory = new N5Factory(); - return factory.openWriter(N5Factory.StorageFormat.ZARR, location); -// return createN5Writer(location, new GsonBuilder()); + return createN5Writer(location, new GsonBuilder()); } /* Tests that overide this should enusre that the `N5Writer` created will remove its container on close() */ From 4fd4723318f78195d10689ff35d9180aa1ce6fd9 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 3 Feb 2025 17:53:31 +0100 Subject: [PATCH 167/423] fix javadoc error --- .../java/org/janelia/saalfeldlab/n5/readdata/ReadData.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index f21f2f571..c6222a494 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -140,9 +140,6 @@ default ReadData encode(BytesCodec codec) throws IOException { * OutputStreamEncoder to use for encoding * * @return encoded ReadData - * - * @throws IOException - * if any I/O error occurs */ default ReadData encode(OutputStreamEncoder encoder) { return new EncodedReadData(this, encoder); From 8b097b5e7dda6db10992bef2daa32d8e87c84c77 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 3 Feb 2025 17:46:20 +0100 Subject: [PATCH 168/423] Add ReadData method toByteBuffer() --- .../saalfeldlab/n5/DoubleArrayDataBlock.java | 11 ++++----- .../saalfeldlab/n5/FloatArrayDataBlock.java | 3 +-- .../saalfeldlab/n5/IntArrayDataBlock.java | 3 +-- .../saalfeldlab/n5/LongArrayDataBlock.java | 3 +-- .../saalfeldlab/n5/ShortArrayDataBlock.java | 3 +-- .../saalfeldlab/n5/readdata/ReadData.java | 24 +++++++++++++++++-- 6 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 9424b4617..29b066f9d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -37,6 +37,11 @@ public DoubleArrayDataBlock(final int[] size, final long[] gridPosition, final d super(size, gridPosition, data); } + @Override + public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { + readData.toByteBuffer().order(byteOrder).asDoubleBuffer().get(data); + } + @Override public ReadData writeData(final ByteOrder byteOrder) { final ByteBuffer serialized = ByteBuffer.allocate(Double.BYTES * data.length); @@ -44,12 +49,6 @@ public ReadData writeData(final ByteOrder byteOrder) { return ReadData.from(serialized); } - @Override - public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - final ByteBuffer serialized = ByteBuffer.wrap(readData.allBytes()); - serialized.order(byteOrder).asDoubleBuffer().get(data); - } - @Override public int getNumElements() { return data.length; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index 9c3a4c8c9..3f37c4ed0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -39,8 +39,7 @@ public FloatArrayDataBlock(final int[] size, final long[] gridPosition, final fl @Override public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - final ByteBuffer serialized = ByteBuffer.wrap(readData.allBytes()); - serialized.order(byteOrder).asFloatBuffer().get(data); + readData.toByteBuffer().order(byteOrder).asFloatBuffer().get(data); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 590d2a17b..26614adf5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -39,8 +39,7 @@ public IntArrayDataBlock(final int[] size, final long[] gridPosition, final int[ @Override public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - final ByteBuffer serialized = ByteBuffer.wrap(readData.allBytes()); - serialized.order(byteOrder).asIntBuffer().get(data); + readData.toByteBuffer().order(byteOrder).asIntBuffer().get(data); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index d6829ffd5..346e83339 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -39,8 +39,7 @@ public LongArrayDataBlock(final int[] size, final long[] gridPosition, final lon @Override public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - final ByteBuffer serialized = ByteBuffer.wrap(readData.allBytes()); - serialized.order(byteOrder).asLongBuffer().get(data); + readData.toByteBuffer().order(byteOrder).asLongBuffer().get(data); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index bdebfa04f..5dcafc10e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -39,8 +39,7 @@ public ShortArrayDataBlock(final int[] size, final long[] gridPosition, final sh @Override public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - final ByteBuffer serialized = ByteBuffer.wrap(readData.allBytes()); - serialized.order(byteOrder).asShortBuffer().get(data); + readData.toByteBuffer().order(byteOrder).asShortBuffer().get(data); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index c6222a494..9f6a8eae6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -76,6 +76,28 @@ default long length() throws IOException { */ byte[] allBytes() throws IOException, IllegalStateException; + /** + * Return the contained data as a {@code ByteBuffer}. + *

    + * This may use {@link #inputStream()} to read the data. + * Because repeatedly calling {@link #inputStream()} may not work, + *

      + *
    1. this method may fail with {@code IllegalStateException} if {@code inputStream()} was already called
      2. + *
    2. subsequent {@code inputStream()} calls may fail with {@code IllegalStateException}
      3. + *
    + * The byte order of the returned {@code ByteBuffer} is {@code BIG_ENDIAN}. + * + * @return all contained data as a ByteBuffer + * + * @throws IOException + * if any I/O error occurs + * @throws IllegalStateException + * if {@link #inputStream()} was already called once and cannot be called again. + */ + default ByteBuffer toByteBuffer() throws IOException, IllegalStateException { + return ByteBuffer.wrap(allBytes()); + } + /** * If this {@code ReadData} is a {@code SplittableReadData}, just returns {@code this}. *

    @@ -114,8 +136,6 @@ default void writeTo(OutputStream outputStream) throws IOException, IllegalState // ------------- Encoding / Decoding ---------------- // - // TODO: WIP, exploring API options... - /** * Returns a new ReadData that uses the given {@code Codec} to encode this * ReadData. From d9f42220f98462440829ee6eb318744de6f78fa7 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 13 Feb 2025 10:18:26 +0100 Subject: [PATCH 169/423] clean up --- .../java/org/janelia/saalfeldlab/n5/Bzip2Compression.java | 6 +++--- src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 9f084601c..c153d7a33 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -30,11 +30,11 @@ import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; -import org.janelia.saalfeldlab.n5.readdata.OutputStreamEncoder; +import org.janelia.saalfeldlab.n5.readdata.OutputStreamEncoder.EncodedOutputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("bzip2") -public class Bzip2Compression implements DefaultBlockReader, DefaultBlockWriter, Compression { +public class Bzip2Compression implements Compression { private static final long serialVersionUID = -4873117458390529118L; @@ -70,7 +70,7 @@ public ReadData decode(final ReadData readData, final int decodedLength) throws public ReadData encode(final ReadData readData) { return readData.encode(out -> { final BZip2CompressorOutputStream deflater = new BZip2CompressorOutputStream(out, blockSize); - return new OutputStreamEncoder.EncodedOutputStream(deflater, deflater::finish); + return new EncodedOutputStream(deflater, deflater::finish); }); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index bc32f93f3..e4477775e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -30,7 +30,7 @@ import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; -import org.janelia.saalfeldlab.n5.readdata.OutputStreamEncoder; +import org.janelia.saalfeldlab.n5.readdata.OutputStreamEncoder.EncodedOutputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("xz") @@ -70,7 +70,7 @@ public ReadData decode(final ReadData readData, final int decodedLength) throws public ReadData encode(final ReadData readData) { return readData.encode(out -> { final XZCompressorOutputStream deflater = new XZCompressorOutputStream(out, preset); - return new OutputStreamEncoder.EncodedOutputStream(deflater, deflater::finish); + return new EncodedOutputStream(deflater, deflater::finish); }); } } From b70e6b5ba1b2a91e857c61ee07811f45777b12f4 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 13 Feb 2025 13:11:33 +0100 Subject: [PATCH 170/423] Use OutputStream wrapper to intercept close() in EncodedReadData We still need a custom interface "OutputStreamOperator" because we want to throw IOException and UnaryOperator::apply doesn't. --- .../saalfeldlab/n5/Bzip2Compression.java | 6 +-- .../saalfeldlab/n5/GzipCompression.java | 9 +--- .../saalfeldlab/n5/Lz4Compression.java | 8 ++-- .../janelia/saalfeldlab/n5/XzCompression.java | 6 +-- .../n5/readdata/EncodedReadData.java | 42 +++++++++++++++---- .../n5/readdata/OutputStreamEncoder.java | 33 --------------- .../saalfeldlab/n5/readdata/ReadData.java | 4 +- 7 files changed, 45 insertions(+), 63 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/OutputStreamEncoder.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index c153d7a33..354560279 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -30,7 +30,6 @@ import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; -import org.janelia.saalfeldlab.n5.readdata.OutputStreamEncoder.EncodedOutputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("bzip2") @@ -68,9 +67,6 @@ public ReadData decode(final ReadData readData, final int decodedLength) throws @Override public ReadData encode(final ReadData readData) { - return readData.encode(out -> { - final BZip2CompressorOutputStream deflater = new BZip2CompressorOutputStream(out, blockSize); - return new EncodedOutputStream(deflater, deflater::finish); - }); + return readData.encode(out -> new BZip2CompressorOutputStream(out, blockSize)); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index 949d54fa8..f9dab5c5b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -34,7 +34,6 @@ import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; import org.apache.commons.compress.compressors.gzip.GzipParameters; import org.janelia.saalfeldlab.n5.Compression.CompressionType; -import org.janelia.saalfeldlab.n5.readdata.OutputStreamEncoder.EncodedOutputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("gzip") @@ -94,15 +93,11 @@ public ReadData decode(final ReadData readData, final int decodedLength) throws @Override public ReadData encode(final ReadData readData) { if (useZlib) { - return readData.encode(out -> { - final DeflaterOutputStream deflater = new DeflaterOutputStream(out, new Deflater(level)); - return new EncodedOutputStream(deflater, deflater::finish); - }); + return readData.encode(out -> new DeflaterOutputStream(out, new Deflater(level))); } else { return readData.encode(out -> { parameters.setCompressionLevel(level); - final GzipCompressorOutputStream deflater = new GzipCompressorOutputStream(out, parameters); - return new EncodedOutputStream(deflater, deflater::finish); + return new GzipCompressorOutputStream(out, parameters); }); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index 2a8e9c2c6..e8fdafcc8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -25,14 +25,15 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.FilterOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; -import org.janelia.saalfeldlab.n5.readdata.OutputStreamEncoder.EncodedOutputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("lz4") @@ -70,9 +71,6 @@ public ReadData decode(final ReadData readData, final int decodedLength) throws @Override public ReadData encode(final ReadData readData) { - return readData.encode(out -> { - final LZ4BlockOutputStream deflater = new LZ4BlockOutputStream(out, blockSize); - return new EncodedOutputStream(deflater, deflater::finish); - }); + return readData.encode(out -> new LZ4BlockOutputStream(out, blockSize)); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index e4477775e..eb174873a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -30,7 +30,6 @@ import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; -import org.janelia.saalfeldlab.n5.readdata.OutputStreamEncoder.EncodedOutputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("xz") @@ -68,9 +67,6 @@ public ReadData decode(final ReadData readData, final int decodedLength) throws @Override public ReadData encode(final ReadData readData) { - return readData.encode(out -> { - final XZCompressorOutputStream deflater = new XZCompressorOutputStream(out, preset); - return new EncodedOutputStream(deflater, deflater::finish); - }); + return readData.encode(out -> new XZCompressorOutputStream(out, preset)); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java index 18a221419..bc56f4e0d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java @@ -1,21 +1,37 @@ package org.janelia.saalfeldlab.n5.readdata; import java.io.ByteArrayOutputStream; +import java.io.FilterOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Objects; class EncodedReadData implements ReadData { - private final ReadData source; + /** + * Like {@code UnaryOperator}, but {@code apply} throws {@code IOException}. + */ + @FunctionalInterface + public interface OutputStreamOperator { - private final OutputStreamEncoder encoder; + OutputStream apply(OutputStream o) throws IOException; - EncodedReadData(final ReadData data, final OutputStreamEncoder encoder) { + default OutputStreamOperator andThen(OutputStreamOperator after) { + Objects.requireNonNull(after); + return o -> after.apply(apply(o)); + } + } + + EncodedReadData(final ReadData data, final OutputStreamOperator encoder) { this.source = data; - this.encoder = encoder; + this.encoder = interceptClose.andThen(encoder)::apply; } + private final ReadData source; + + private final OutputStreamOperator encoder; + private ByteArraySplittableReadData bytes; @Override @@ -48,9 +64,21 @@ public void writeTo(final OutputStream outputStream) throws IOException, Illegal if (bytes != null) { outputStream.write(bytes.allBytes()); } else { - final OutputStreamEncoder.EncodedOutputStream deflater = encoder.encode(outputStream); - source.writeTo(deflater.outputStream()); - deflater.finish(); + try (final OutputStream deflater = encoder.apply(outputStream)) { + source.writeTo(deflater); + } } } + + /** + * {@code UnaryOperator} that wraps {@code OutputStream} to intercept {@code + * close()} and call {@code flush()} instead + */ + private static OutputStreamOperator interceptClose = o -> new FilterOutputStream(o) { + + @Override + public void close() throws IOException { + out.flush(); + } + }; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/OutputStreamEncoder.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/OutputStreamEncoder.java deleted file mode 100644 index 3eed7ada8..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/OutputStreamEncoder.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.janelia.saalfeldlab.n5.readdata; - -import java.io.IOException; -import java.io.OutputStream; - -public interface OutputStreamEncoder { - - EncodedOutputStream encode(OutputStream out) throws IOException; - - interface Finisher { - - void finish() throws IOException; - } - - final class EncodedOutputStream { - - private final OutputStream outputStream; - private final Finisher finisher; - - public EncodedOutputStream(final OutputStream outputStream, final Finisher finisher) { - this.outputStream = outputStream; - this.finisher = finisher; - } - - OutputStream outputStream() { - return outputStream; - } - - void finish() throws IOException { - finisher.finish(); - } - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 9f6a8eae6..2e0642c64 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -4,8 +4,10 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.function.UnaryOperator; import org.janelia.saalfeldlab.n5.BytesCodec; import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.readdata.EncodedReadData.OutputStreamOperator; /** * An abstraction over {@code byte[]} data. @@ -161,7 +163,7 @@ default ReadData encode(BytesCodec codec) throws IOException { * * @return encoded ReadData */ - default ReadData encode(OutputStreamEncoder encoder) { + default ReadData encode(OutputStreamOperator encoder) { return new EncodedReadData(this, encoder); } From a2e9596706265abf2a40d73337f1539ef2409427 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 13 Feb 2025 13:14:17 +0100 Subject: [PATCH 171/423] ReadData.encode/decode methods forwarding to BytesCodec --- .../saalfeldlab/n5/DefaultBlockReader.java | 4 +- .../saalfeldlab/n5/DefaultBlockWriter.java | 6 +-- .../saalfeldlab/n5/readdata/ReadData.java | 41 ------------------- 3 files changed, 5 insertions(+), 46 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 241f30c8b..11bd5aa1f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -81,8 +81,8 @@ static DataBlock readBlock( final int numBytes = dataType.isVarLength() ? numElements : (numElements * dataType.bytesPerElement()); - final ReadData data = ReadData.from(in) - .decode(datasetAttributes.getCompression(), numBytes); + final ReadData data = datasetAttributes.getCompression().decode( + ReadData.from(in), numBytes); dataBlock.readData(ByteOrder.BIG_ENDIAN, data); return dataBlock; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index 4d94745fb..962e98752 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -78,9 +78,9 @@ else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSiz dos.flush(); - dataBlock.writeData(ByteOrder.BIG_ENDIAN) - .encode(datasetAttributes.getCompression()) - .writeTo(out); + datasetAttributes.getCompression().encode( + dataBlock.writeData(ByteOrder.BIG_ENDIAN) + ).writeTo(out); out.flush(); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 2e0642c64..64c791a2b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -133,27 +133,9 @@ default void writeTo(OutputStream outputStream) throws IOException, IllegalState outputStream.write(allBytes()); } - // - // // ------------- Encoding / Decoding ---------------- // - /** - * Returns a new ReadData that uses the given {@code Codec} to encode this - * ReadData. - * - * @param codec - * Codec to use for encoding - * - * @return encoded ReadData - * - * @throws IOException - * if any I/O error occurs - */ - default ReadData encode(BytesCodec codec) throws IOException { - return codec.encode(this); - } - /** * Returns a new ReadData that uses the given {@code OutputStreamEncoder} to * encode this ReadData. @@ -167,29 +149,6 @@ default ReadData encode(OutputStreamOperator encoder) { return new EncodedReadData(this, encoder); } - /** - * Returns a new ReadData that uses the given {@code codec} to decode this - * ReadData. - *

    - * The returned ReadData reports {@link #length()}{@code == decodedLength}. - * Decoding may be lazy or eager, depending on the {@code BytesCodec}. - * - * @param codec - * Codec to use for decoding - * @param decodedLength - * length of the decoded data (-1 if unknown). - * - * @return decoded ReadData - * - * @throws IOException - * if any I/O error occurs - */ - default ReadData decode(BytesCodec codec, int decodedLength) throws IOException { - return codec.decode(this, decodedLength); - } - - // - // // --------------- Factory Methods ------------------ // From 018d31186f269f1d7efc0d5701e3ef96d1e65928 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 13 Feb 2025 13:32:22 +0100 Subject: [PATCH 172/423] remove BytesCodec interface and put encode/decode into Compression for now --- .../janelia/saalfeldlab/n5/BytesCodec.java | 42 ------------------ .../janelia/saalfeldlab/n5/Compression.java | 43 ++++++++++++++++++- .../saalfeldlab/n5/readdata/ReadData.java | 2 - 3 files changed, 42 insertions(+), 45 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java deleted file mode 100644 index 2d69211c3..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/BytesCodec.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - -public interface BytesCodec { - - /** - * Decode the given {@code readData}. - *

    - * The returned decoded {@code ReadData} reports {@link ReadData#length() - * length()}{@code == decodedLength}. Decoding may be lazy or eager, - * depending on the {@code BytesCodec} implementation. - * - * @param readData - * data to decode - * @param decodedLength - * length of the decoded data (-1 if unknown) - * - * @return decoded ReadData - * - * @throws IOException - * if any I/O error occurs - */ - ReadData decode(ReadData readData, int decodedLength) throws IOException; - - /** - * Encode the given {@code readData}. - *

    - * Encoding may be lazy or eager, depending on the {@code BytesCodec} - * implementation. - * - * @param readData - * data to encode - * - * @return encoded ReadData - * - * @throws IOException - * if any I/O error occurs - */ - ReadData encode(ReadData readData) throws IOException; -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index d9cac2fb7..cb61de922 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -25,12 +25,14 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.IOException; import java.io.Serializable; import java.lang.annotation.ElementType; import java.lang.annotation.Inherited; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.scijava.annotations.Indexable; /** @@ -38,7 +40,7 @@ * * @author Stephan Saalfeld */ -public interface Compression extends BytesCodec, Serializable { +public interface Compression extends Serializable { /** * Annotation for runtime discovery of compression schemes. @@ -70,4 +72,43 @@ default String getType() { else return compressionType.value(); } + + // -------------------------------------------------- + // + + /** + * Decode the given {@code readData}. + *

    + * The returned decoded {@code ReadData} reports {@link ReadData#length() + * length()}{@code == decodedLength}. Decoding may be lazy or eager, + * depending on the {@code BytesCodec} implementation. + * + * @param readData + * data to decode + * @param decodedLength + * length of the decoded data (-1 if unknown) + * + * @return decoded ReadData + * + * @throws IOException + * if any I/O error occurs + */ + ReadData decode(ReadData readData, int decodedLength) throws IOException; + + /** + * Encode the given {@code readData}. + *

    + * Encoding may be lazy or eager, depending on the {@code BytesCodec} + * implementation. + * + * @param readData + * data to encode + * + * @return encoded ReadData + * + * @throws IOException + * if any I/O error occurs + */ + ReadData encode(ReadData readData) throws IOException; + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 64c791a2b..19f0537a2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -4,8 +4,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.function.UnaryOperator; -import org.janelia.saalfeldlab.n5.BytesCodec; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.readdata.EncodedReadData.OutputStreamOperator; From 2bcdab883ed583d9c5ba67474184e218427c0ccf Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 13 Feb 2025 09:37:44 +0100 Subject: [PATCH 173/423] idea --- .../java/org/janelia/saalfeldlab/n5/DataBlock.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 5a93b4267..3539fb76d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -105,6 +105,17 @@ public interface DataBlock { // TODO: rename? "serialize"? "write"? ReadData writeData(ByteOrder byteOrder); + +// DataBlockCodec getDataBlockCodec(); +// +// interface DataBlockCodec { +// +// ReadData serialize(DataBlock dataBlock) throws IOException; +// +// void deserialize(ReadData readData, DataBlock dataBlock) throws IOException; +// } + + /** * Returns the number of elements in this {@link DataBlock}. This number is * not necessarily equal {@link #getNumElements(int[]) From 0d66ee69dd7866e379ff3eef1ab0df1953d60cde Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 14 Feb 2025 17:03:15 +0100 Subject: [PATCH 174/423] Add LazyReadData and OutputStreamWriter When data is requested from the LazyReadData, the LazyReadData will ask its OutputStreamWriter to write the data to a ByteArrayOutputStream. When the LazyReadData itself is written to an OutputStream, it will pass that OutputStream to its OutputStreamWriter (without loading the data into a byte[] array first). --- .../org/janelia/saalfeldlab/n5/Codecs.java | 4 ++ .../saalfeldlab/n5/readdata/LazyReadData.java | 53 +++++++++++++++++++ .../saalfeldlab/n5/readdata/ReadData.java | 9 ++++ 3 files changed, 66 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/Codecs.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java new file mode 100644 index 000000000..f69f544d2 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java @@ -0,0 +1,4 @@ +package org.janelia.saalfeldlab.n5; + +public class Codecs { +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java new file mode 100644 index 000000000..fe3d8f7a1 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -0,0 +1,53 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.ByteArrayOutputStream; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Objects; + +class LazyReadData implements ReadData { + + LazyReadData(final OutputStreamWriter writer) { + this.writer = writer; + } + + private final OutputStreamWriter writer; + + private ByteArraySplittableReadData bytes; + + @Override + public SplittableReadData splittable() throws IOException { + if (bytes == null) { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); + writeTo(baos); + bytes = new ByteArraySplittableReadData(baos.toByteArray()); + } + return bytes; + } + + @Override + public long length() throws IOException { + return splittable().length(); + } + + @Override + public InputStream inputStream() throws IOException, IllegalStateException { + return splittable().inputStream(); + } + + @Override + public byte[] allBytes() throws IOException, IllegalStateException { + return splittable().allBytes(); + } + + @Override + public void writeTo(final OutputStream outputStream) throws IOException, IllegalStateException { + if (bytes != null) { + outputStream.write(bytes.allBytes()); + } else { + writer.writeTo(outputStream); + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 19f0537a2..60d3bc840 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -241,4 +241,13 @@ static SplittableReadData from(final ByteBuffer data) { throw new UnsupportedOperationException("TODO. Direct ByteBuffer not supported yet."); } } + + @FunctionalInterface + interface OutputStreamWriter { + void writeTo(OutputStream outputStream) throws IOException, IllegalStateException; + } + + static ReadData from(OutputStreamWriter generator) { + return new LazyReadData(generator); + } } From 8683dfa3e775e119149813347e70ab2646ddf0c9 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 14 Feb 2025 17:03:51 +0100 Subject: [PATCH 175/423] WIP Codecs --- .../saalfeldlab/n5/AbstractDataBlock.java | 24 ++ .../saalfeldlab/n5/ByteArrayDataBlock.java | 24 +- .../org/janelia/saalfeldlab/n5/Codecs.java | 369 ++++++++++++++++++ .../org/janelia/saalfeldlab/n5/DataBlock.java | 17 +- .../saalfeldlab/n5/DefaultBlockReader.java | 7 +- .../saalfeldlab/n5/DefaultBlockWriter.java | 5 +- .../org/janelia/saalfeldlab/n5/N5Reader.java | 8 +- .../saalfeldlab/n5/ShortArrayDataBlock.java | 28 +- .../saalfeldlab/n5/StringDataBlock.java | 28 ++ 9 files changed, 486 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index 822ceddab..9a201a435 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -25,6 +25,10 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.IOException; +import java.nio.ByteOrder; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + /** * Abstract base class for {@link DataBlock} implementations. * @@ -63,4 +67,24 @@ public T getData() { return data; } + + static class DefaultDataBlockCodec implements DataCodec { + + @Override + public ReadData serialize(final DataBlock dataBlock) throws IOException { + return dataBlock.writeData(ByteOrder.BIG_ENDIAN); + } + + @Override + public void deserialize(final ReadData readData, final DataBlock dataBlock) throws IOException { + dataBlock.readData(ByteOrder.BIG_ENDIAN, readData); + } + } + + private static DefaultDataBlockCodec defaultCodec = new DefaultDataBlockCodec<>(); + + @Override + public DataCodec getDataCodec() { + return defaultCodec; + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index a97c515d1..7fc2b55e6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -39,12 +39,12 @@ public ByteArrayDataBlock(final int[] size, final long[] gridPosition, final byt @Override public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - new DataInputStream(readData.inputStream()).readFully(data); + throw new UnsupportedOperationException(); } @Override public ReadData writeData(final ByteOrder byteOrder) { - return ReadData.from(data); + throw new UnsupportedOperationException(); } @Override @@ -52,4 +52,24 @@ public int getNumElements() { return data.length; } + + public static class DefaultCodec implements DataCodec { + + @Override + public ReadData serialize(final DataBlock dataBlock) throws IOException { + return ReadData.from(dataBlock.getData()); + } + + @Override + public void deserialize(final ReadData readData, final DataBlock dataBlock) throws IOException { + new DataInputStream(readData.inputStream()).readFully(dataBlock.getData()); + } + + static DefaultCodec INSTANCE = new DefaultCodec(); + } + + @Override + public DataCodec getDataCodec() { + return DefaultCodec.INSTANCE; + } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java index f69f544d2..129e38938 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java @@ -1,4 +1,373 @@ package org.janelia.saalfeldlab.n5; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.function.IntFunction; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +import static org.janelia.saalfeldlab.n5.Codecs.ChunkHeader.MODE_DEFAULT; +import static org.janelia.saalfeldlab.n5.Codecs.ChunkHeader.MODE_VARLENGTH; + public class Codecs { + + + + + + + + public static abstract class DataCodec { + + public static final DataCodec BYTE = new ByteDataCodec(); + public static final DataCodec SHORT = new ShortDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec INT = new IntDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec LONG = new LongDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec FLOAT = new FloatDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec DOUBLE = new DoubleDataCodec(ByteOrder.BIG_ENDIAN); + + public static final DataCodec SHORT_LITTLE_ENDIAN = new ShortDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec INT_LITTLE_ENDIAN = new IntDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec LONG_LITTLE_ENDIAN = new LongDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec FLOAT_LITTLE_ENDIAN = new FloatDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec DOUBLE_LITTLE_ENDIAN = new DoubleDataCodec(ByteOrder.LITTLE_ENDIAN); + + public abstract ReadData serialize(A data) throws IOException; + + public abstract void deserialize(ReadData readData, A data) throws IOException; + + public int bytesPerElement() { + return bytesPerElement; + } + + public A createData(final int numElements) { + return dataFactory.apply(numElements); + } + + // ---------------- implementations ----------------- + // + + private final int bytesPerElement; + private final IntFunction dataFactory; + + private DataCodec(int bytesPerElement, IntFunction dataFactory) { + this.bytesPerElement = bytesPerElement; + this.dataFactory = dataFactory; + } + + private static final class ByteDataCodec extends DataCodec { + + private ByteDataCodec() { + super(Byte.BYTES, byte[]::new); + } + + @Override + public ReadData serialize(final byte[] data) { + return ReadData.from(data); + } + + @Override + public void deserialize(final ReadData readData, final byte[] data) throws IOException { + new DataInputStream(readData.inputStream()).readFully(data); + } + } + + private static final class ShortDataCodec extends DataCodec { + + private final ByteOrder order; + + ShortDataCodec(ByteOrder order) { + super(Short.BYTES, short[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final short[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * data.length); + serialized.order(order).asShortBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public void deserialize(final ReadData readData, final short[] data) throws IOException { + readData.toByteBuffer().order(order).asShortBuffer().get(data); + } + } + + private static final class IntDataCodec extends DataCodec { + + private final ByteOrder order; + + IntDataCodec(ByteOrder order) { + super(Integer.BYTES, int[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final int[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * data.length); + serialized.order(order).asIntBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public void deserialize(final ReadData readData, final int[] data) throws IOException { + readData.toByteBuffer().order(order).asIntBuffer().get(data); + } + } + + private static final class LongDataCodec extends DataCodec { + + private final ByteOrder order; + + LongDataCodec(ByteOrder order) { + super(Long.BYTES, long[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final long[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Long.BYTES * data.length); + serialized.order(order).asLongBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public void deserialize(final ReadData readData, final long[] data) throws IOException { + readData.toByteBuffer().order(order).asLongBuffer().get(data); + } + } + + private static final class FloatDataCodec extends DataCodec { + + private final ByteOrder order; + + FloatDataCodec(ByteOrder order) { + super(Float.BYTES, float[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final float[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Float.BYTES * data.length); + serialized.order(order).asFloatBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public void deserialize(final ReadData readData, final float[] data) throws IOException { + readData.toByteBuffer().order(order).asFloatBuffer().get(data); + } + } + + private static final class DoubleDataCodec extends DataCodec { + + private final ByteOrder order; + + DoubleDataCodec(ByteOrder order) { + super(Double.BYTES, double[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final double[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Double.BYTES * data.length); + serialized.order(order).asDoubleBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public void deserialize(final ReadData readData, final double[] data) throws IOException { + readData.toByteBuffer().order(order).asDoubleBuffer().get(data); + } + } + } + + + + + + + + + + + + + interface DataBlockCodec { + + ReadData encode(B dataBlock, Compression compression) throws IOException; + + B decode(ReadData readData, long[] gridPosition, Compression compression) throws IOException; + } + + public static final DataBlockCodec BYTE = + new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new); + public static final DataBlockCodec SHORT = + new DefaultDataBlockCodec<>(DataCodec.SHORT, ShortArrayDataBlock::new); + + /** + * DataBlockCodec for all N5 data types, except STRING and OBJECT + */ + static class DefaultDataBlockCodec> implements DataBlockCodec { + + interface DataBlockFactory> { + + B createDataBlock(int[] blockSize, long[] gridPosition, A data); + } + + private final DataCodec dataCodec; + + private final DataBlockFactory dataBlockFactory; + + DefaultDataBlockCodec( + final DataCodec dataCodec, + final DataBlockFactory dataBlockFactory) { + this.dataCodec = dataCodec; + this.dataBlockFactory = dataBlockFactory; + } + + @Override + public ReadData encode(final B dataBlock, final Compression compression) throws IOException { + return ReadData.from(out -> { + new ChunkHeader(dataBlock.getSize(), dataBlock.getNumElements()).writeTo(out); + compression.encode(dataCodec.serialize(dataBlock.getData())).writeTo(out); + out.flush(); + }); + } + + @Override + public B decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { + try(final InputStream in = readData.inputStream()) { + final ChunkHeader header = ChunkHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); + final A data = dataCodec.createData(header.numElements()); + final int numBytes = header.numElements() * dataCodec.bytesPerElement(); + final ReadData decompressed = compression.decode(ReadData.from(in), numBytes); + dataCodec.deserialize(decompressed, data); + return dataBlockFactory.createDataBlock(header.blockSize(), gridPosition, data); + } + } + } + + + + + + + + + + + + + static class ChunkHeader { + + public static final short MODE_DEFAULT = 0; + public static final short MODE_VARLENGTH = 1; + public static final short MODE_OBJECT = 2; + + private final short mode; + private final int[] blockSize; + private final int numElements; + + ChunkHeader(final short mode, final int[] blockSize, final int numElements) { + this.mode = mode; + this.blockSize = blockSize; + this.numElements = numElements; + } + + ChunkHeader(final int[] blockSize, final int numElements) { + this.mode = (DataBlock.getNumElements(blockSize) == numElements) ? MODE_DEFAULT : MODE_VARLENGTH; + this.blockSize = blockSize; + this.numElements = numElements; + } + + public short mode() { + return mode; + } + + public int[] blockSize() { + return blockSize; + } + + public int numElements() { + return numElements; + } + + private static int[] readBlockSize(final DataInputStream dis) throws IOException { + final int nDim = dis.readShort(); + final int[] blockSize = new int[nDim]; + for (int d = 0; d < nDim; ++d) + blockSize[d] = dis.readInt(); + return blockSize; + } + + private static void writeBlockSize(final int[] blockSize, final DataOutputStream dos) throws IOException { + dos.writeShort(blockSize.length); + for (final int size : blockSize) + dos.writeInt(size); + } + + void writeTo(final OutputStream out) throws IOException { + final DataOutputStream dos = new DataOutputStream(out); + dos.writeShort(mode); + switch (mode) { + case MODE_DEFAULT:// default + writeBlockSize(blockSize, dos); + break; + case MODE_VARLENGTH:// varlength + writeBlockSize(blockSize, dos); + dos.writeInt(numElements); + break; + case MODE_OBJECT: // object + dos.writeInt(numElements); + break; + default: + throw new IOException("unexpected mode: " + mode); + } + dos.flush(); + } + + static ChunkHeader readFrom(final InputStream in) throws IOException { + return readFrom(in, null); + } + + static ChunkHeader readFrom(final InputStream in, short... allowedModes) throws IOException { + final DataInputStream dis = new DataInputStream(in); + final short mode = dis.readShort(); + final int[] blockSize; + final int numElements; + switch (mode) { + case MODE_DEFAULT:// default + blockSize = readBlockSize(dis); + numElements = DataBlock.getNumElements(blockSize); + break; + case MODE_VARLENGTH:// varlength + blockSize = readBlockSize(dis); + numElements = dis.readInt(); + break; + case MODE_OBJECT: // object + blockSize = null; + numElements = dis.readInt(); + break; + default: + throw new IOException("unexpected mode: " + mode); + } + + boolean modeIsOk = allowedModes == null || allowedModes.length == 0; + for (int i = 0; !modeIsOk && i < allowedModes.length; ++i) { + if (mode == allowedModes[i]) + modeIsOk = true; + } + if (!modeIsOk) { + throw new IOException("unexpected mode: " + mode); + } + + return new ChunkHeader(mode, blockSize, numElements); + } + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 3539fb76d..293d30c40 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -105,15 +105,16 @@ public interface DataBlock { // TODO: rename? "serialize"? "write"? ReadData writeData(ByteOrder byteOrder); + // TODO revise, clean up + DataCodec getDataCodec(); -// DataBlockCodec getDataBlockCodec(); -// -// interface DataBlockCodec { -// -// ReadData serialize(DataBlock dataBlock) throws IOException; -// -// void deserialize(ReadData readData, DataBlock dataBlock) throws IOException; -// } + // TODO revise, clean up + interface DataCodec { + + ReadData serialize(DataBlock dataBlock) throws IOException; + + void deserialize(ReadData readData, DataBlock dataBlock) throws IOException; + } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 11bd5aa1f..4847985c0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -28,7 +28,7 @@ import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.ByteOrder; +import org.janelia.saalfeldlab.n5.DataBlock.DataCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** @@ -83,8 +83,9 @@ static DataBlock readBlock( : (numElements * dataType.bytesPerElement()); final ReadData data = datasetAttributes.getCompression().decode( ReadData.from(in), numBytes); - dataBlock.readData(ByteOrder.BIG_ENDIAN, data); + final DataCodec codec = dataBlock.getDataCodec(); + codec.deserialize(data, dataBlock); return dataBlock; } -} \ No newline at end of file +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index 962e98752..351663425 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -28,7 +28,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.nio.ByteOrder; +import org.janelia.saalfeldlab.n5.DataBlock.DataCodec; /** * Default implementation of block writing (N5 format). @@ -78,8 +78,9 @@ else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSiz dos.flush(); + final DataCodec codec = dataBlock.getDataCodec(); datasetAttributes.getCompression().encode( - dataBlock.writeData(ByteOrder.BIG_ENDIAN) + codec.serialize(dataBlock) ).writeTo(out); out.flush(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 9d8b10a9f..21e104080 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -25,14 +25,12 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.Serializable; import java.io.UncheckedIOException; import java.lang.reflect.Type; import java.net.URI; -import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -43,7 +41,7 @@ import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.DataBlock.DataCodec; /** * A simple structured container for hierarchies of chunked @@ -324,8 +322,8 @@ default T readSerializedBlock( if (block == null) return null; - final ReadData serialized = block.writeData(ByteOrder.BIG_ENDIAN); - try (ObjectInputStream in = new ObjectInputStream(serialized.inputStream())) { + final DataCodec codec = block.getDataCodec(); + try (ObjectInputStream in = new ObjectInputStream(codec.serialize(block).inputStream())) { return (T) in.readObject(); } catch (final IOException | UncheckedIOException e) { throw new N5Exception.N5IOException(e); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 5dcafc10e..5a1ff60e9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -39,14 +39,12 @@ public ShortArrayDataBlock(final int[] size, final long[] gridPosition, final sh @Override public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - readData.toByteBuffer().order(byteOrder).asShortBuffer().get(data); + throw new UnsupportedOperationException(); } @Override public ReadData writeData(final ByteOrder byteOrder) { - final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * data.length); - serialized.order(byteOrder).asShortBuffer().put(data); - return ReadData.from(serialized); + throw new UnsupportedOperationException(); } @Override @@ -54,4 +52,26 @@ public int getNumElements() { return data.length; } + + static class DefaultCodec implements DataCodec { + + @Override + public ReadData serialize(final DataBlock dataBlock) throws IOException { + final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * dataBlock.getNumElements()); + serialized.order(ByteOrder.BIG_ENDIAN).asShortBuffer().put(dataBlock.getData()); + return ReadData.from(serialized); + } + + @Override + public void deserialize(final ReadData readData, final DataBlock dataBlock) throws IOException { + readData.toByteBuffer().order(ByteOrder.BIG_ENDIAN).asShortBuffer().get(dataBlock.getData()); + } + + static DefaultCodec INSTANCE = new DefaultCodec(); + } + + @Override + public DataCodec getDataCodec() { + return DefaultCodec.INSTANCE; + } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index d7973012b..f39717c52 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -72,4 +72,32 @@ public int getNumElements() { public String[] getData() { return actualData; } + + + + + + + + static class DefaultCodec implements DataCodec { + @Override + public ReadData serialize(final DataBlock dataBlock) throws IOException { + final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * dataBlock.getNumElements()); + serialized.order(ByteOrder.BIG_ENDIAN).asShortBuffer().put(dataBlock.getData()); + return ReadData.from(serialized); + } + + @Override + public void deserialize(final ReadData readData, final DataBlock dataBlock) throws IOException { + readData.toByteBuffer().order(ByteOrder.BIG_ENDIAN).asShortBuffer().get(dataBlock.getData()); + } + + static DefaultCodec INSTANCE = new DefaultCodec(); + } + + @Override + public DataCodec getDataCodec() { + return DefaultCodec.INSTANCE; + } + } From e52d0fc743e516106fbbedb5127d1ec3c62c53ba Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 14 Feb 2025 17:46:49 +0100 Subject: [PATCH 176/423] WIP Codecs --- .../org/janelia/saalfeldlab/n5/Codecs.java | 59 ++++++++++++++++--- .../org/janelia/saalfeldlab/n5/N5Reader.java | 9 ++- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java index 129e38938..ddd5b26b3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java @@ -11,6 +11,7 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import static org.janelia.saalfeldlab.n5.Codecs.ChunkHeader.MODE_DEFAULT; +import static org.janelia.saalfeldlab.n5.Codecs.ChunkHeader.MODE_OBJECT; import static org.janelia.saalfeldlab.n5.Codecs.ChunkHeader.MODE_VARLENGTH; public class Codecs { @@ -198,17 +199,20 @@ public void deserialize(final ReadData readData, final double[] data) throws IOE - interface DataBlockCodec { + public interface DataBlockCodec { ReadData encode(B dataBlock, Compression compression) throws IOException; B decode(ReadData readData, long[] gridPosition, Compression compression) throws IOException; } - public static final DataBlockCodec BYTE = - new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new); - public static final DataBlockCodec SHORT = - new DefaultDataBlockCodec<>(DataCodec.SHORT, ShortArrayDataBlock::new); + public static final DataBlockCodec BYTE = new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new); + public static final DataBlockCodec SHORT = new DefaultDataBlockCodec<>(DataCodec.SHORT, ShortArrayDataBlock::new); + public static final DataBlockCodec INT = new DefaultDataBlockCodec<>(DataCodec.INT, IntArrayDataBlock::new); + public static final DataBlockCodec LONG = new DefaultDataBlockCodec<>(DataCodec.LONG, LongArrayDataBlock::new); + public static final DataBlockCodec FLOAT = new DefaultDataBlockCodec<>(DataCodec.FLOAT, FloatArrayDataBlock::new); + public static final DataBlockCodec DOUBLE = new DefaultDataBlockCodec<>(DataCodec.DOUBLE, DoubleArrayDataBlock::new); + public static final DataBlockCodec OBJECT = new ObjectDataBlockCodec(); /** * DataBlockCodec for all N5 data types, except STRING and OBJECT @@ -253,16 +257,51 @@ public B decode(final ReadData readData, final long[] gridPosition, final Compre } } + // TODO: String + static class StringDataBlockCodec implements DataBlockCodec { + @Override + public ReadData encode(final StringDataBlock dataBlock, final Compression compression) throws IOException { + // TODO should not use dataBlock.getNumElements() for header, because that will just be the String[] data length in the future + + return null; + } + @Override + public StringDataBlock decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { + return null; + } + } + /** + * TODO javadoc + */ + static class ObjectDataBlockCodec implements DataBlockCodec { + @Override + public ReadData encode(final ByteArrayDataBlock dataBlock, final Compression compression) throws IOException { + return ReadData.from(out -> { + new ChunkHeader(null, dataBlock.getNumElements()).writeTo(out); + compression.encode(ReadData.from(dataBlock.getData())).writeTo(out); + out.flush(); + }); + } - + @Override + public ByteArrayDataBlock decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { + try(final InputStream in = readData.inputStream()) { + final ChunkHeader header = ChunkHeader.readFrom(in, MODE_OBJECT); + final byte[] data = new byte[header.numElements()]; + final ReadData decompressed = compression.decode(ReadData.from(in), data.length); + new DataInputStream(decompressed.inputStream()).readFully(data); + return new ByteArrayDataBlock(null, gridPosition, data); + } + } + } static class ChunkHeader { @@ -281,7 +320,13 @@ static class ChunkHeader { } ChunkHeader(final int[] blockSize, final int numElements) { - this.mode = (DataBlock.getNumElements(blockSize) == numElements) ? MODE_DEFAULT : MODE_VARLENGTH; + if (blockSize == null) { + this.mode = MODE_OBJECT; + } else if (DataBlock.getNumElements(blockSize) == numElements) { + this.mode = MODE_DEFAULT; + } else { + this.mode = MODE_VARLENGTH; + } this.blockSize = blockSize; this.numElements = numElements; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 21e104080..2d0cd216d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -25,6 +25,7 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.Serializable; @@ -322,8 +323,12 @@ default T readSerializedBlock( if (block == null) return null; - final DataCodec codec = block.getDataCodec(); - try (ObjectInputStream in = new ObjectInputStream(codec.serialize(block).inputStream())) { + + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.toByteBuffer().array()); + try (ObjectInputStream in = new ObjectInputStream(byteArrayInputStream)) { + +// final DataCodec codec = block.getDataCodec(); +// try (ObjectInputStream in = new ObjectInputStream(codec.serialize(block).inputStream())) { return (T) in.readObject(); } catch (final IOException | UncheckedIOException e) { throw new N5Exception.N5IOException(e); From 6cd52d0771b1bc5cd25919cc969781eb73c75eea Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 14 Feb 2025 22:12:29 +0100 Subject: [PATCH 177/423] WIP Codecs --- .../org/janelia/saalfeldlab/n5/Codecs.java | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java index ddd5b26b3..cfb550c65 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java @@ -7,6 +7,8 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.function.IntFunction; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -257,23 +259,35 @@ public B decode(final ReadData readData, final long[] gridPosition, final Compre } } - // TODO: String + /** + * TODO javadoc + */ static class StringDataBlockCodec implements DataBlockCodec { + private static final Charset ENCODING = StandardCharsets.UTF_8; + private static final String NULLCHAR = "\0"; + @Override public ReadData encode(final StringDataBlock dataBlock, final Compression compression) throws IOException { - - // TODO should not use dataBlock.getNumElements() for header, because that will just be the String[] data length in the future - - - - - return null; + return ReadData.from(out -> { + final String flattenedArray = String.join(NULLCHAR, dataBlock.getData()) + NULLCHAR; + final byte[] serializedData = flattenedArray.getBytes(ENCODING); + new ChunkHeader(dataBlock.getSize(), serializedData.length).writeTo(out); + compression.encode(ReadData.from(serializedData)).writeTo(out); + out.flush(); + }); } @Override public StringDataBlock decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { - return null; + try(final InputStream in = readData.inputStream()) { + final ChunkHeader header = ChunkHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); + final ReadData decompressed = compression.decode(ReadData.from(in), header.numElements()); + final byte[] serializedData = decompressed.allBytes(); + final String rawChars = new String(serializedData, ENCODING); + final String[] actualData = rawChars.split(NULLCHAR); + return new StringDataBlock(header.blockSize(), gridPosition, actualData); + } } } From 1be51a5d2321b26bbd6187ce6b5c76571a1b0b35 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 14 Feb 2025 22:55:43 +0100 Subject: [PATCH 178/423] WIP use Codecs, remove serialization methods from DataBlock --- .../saalfeldlab/n5/AbstractDataBlock.java | 24 ------ .../saalfeldlab/n5/ByteArrayDataBlock.java | 35 -------- .../org/janelia/saalfeldlab/n5/Codecs.java | 5 +- .../org/janelia/saalfeldlab/n5/DataBlock.java | 56 +------------ .../org/janelia/saalfeldlab/n5/DataType.java | 84 ++++++++----------- .../saalfeldlab/n5/DefaultBlockReader.java | 35 +------- .../saalfeldlab/n5/DefaultBlockWriter.java | 33 ++------ .../saalfeldlab/n5/DoubleArrayDataBlock.java | 17 ---- .../saalfeldlab/n5/FloatArrayDataBlock.java | 17 ---- .../saalfeldlab/n5/IntArrayDataBlock.java | 19 +---- .../saalfeldlab/n5/LongArrayDataBlock.java | 17 ---- .../org/janelia/saalfeldlab/n5/N5Reader.java | 9 +- .../saalfeldlab/n5/ShortArrayDataBlock.java | 37 -------- .../saalfeldlab/n5/StringDataBlock.java | 67 +-------------- 14 files changed, 54 insertions(+), 401 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index 9a201a435..822ceddab 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -25,10 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - /** * Abstract base class for {@link DataBlock} implementations. * @@ -67,24 +63,4 @@ public T getData() { return data; } - - static class DefaultDataBlockCodec implements DataCodec { - - @Override - public ReadData serialize(final DataBlock dataBlock) throws IOException { - return dataBlock.writeData(ByteOrder.BIG_ENDIAN); - } - - @Override - public void deserialize(final ReadData readData, final DataBlock dataBlock) throws IOException { - dataBlock.readData(ByteOrder.BIG_ENDIAN, readData); - } - } - - private static DefaultDataBlockCodec defaultCodec = new DefaultDataBlockCodec<>(); - - @Override - public DataCodec getDataCodec() { - return defaultCodec; - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index 7fc2b55e6..78539c281 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -25,11 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInputStream; -import java.io.IOException; -import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - public class ByteArrayDataBlock extends AbstractDataBlock { public ByteArrayDataBlock(final int[] size, final long[] gridPosition, final byte[] data) { @@ -37,39 +32,9 @@ public ByteArrayDataBlock(final int[] size, final long[] gridPosition, final byt super(size, gridPosition, data); } - @Override - public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public ReadData writeData(final ByteOrder byteOrder) { - throw new UnsupportedOperationException(); - } - @Override public int getNumElements() { return data.length; } - - public static class DefaultCodec implements DataCodec { - - @Override - public ReadData serialize(final DataBlock dataBlock) throws IOException { - return ReadData.from(dataBlock.getData()); - } - - @Override - public void deserialize(final ReadData readData, final DataBlock dataBlock) throws IOException { - new DataInputStream(readData.inputStream()).readFully(dataBlock.getData()); - } - - static DefaultCodec INSTANCE = new DefaultCodec(); - } - - @Override - public DataCodec getDataCodec() { - return DefaultCodec.INSTANCE; - } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java index cfb550c65..66285e123 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java @@ -112,7 +112,7 @@ private static final class IntDataCodec extends DataCodec { @Override public ReadData serialize(final int[] data) { - final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * data.length); + final ByteBuffer serialized = ByteBuffer.allocate(Integer.BYTES * data.length); serialized.order(order).asIntBuffer().put(data); return ReadData.from(serialized); } @@ -201,7 +201,7 @@ public void deserialize(final ReadData readData, final double[] data) throws IOE - public interface DataBlockCodec { + public interface DataBlockCodec> { ReadData encode(B dataBlock, Compression compression) throws IOException; @@ -214,6 +214,7 @@ public interface DataBlockCodec { public static final DataBlockCodec LONG = new DefaultDataBlockCodec<>(DataCodec.LONG, LongArrayDataBlock::new); public static final DataBlockCodec FLOAT = new DefaultDataBlockCodec<>(DataCodec.FLOAT, FloatArrayDataBlock::new); public static final DataBlockCodec DOUBLE = new DefaultDataBlockCodec<>(DataCodec.DOUBLE, DoubleArrayDataBlock::new); + public static final DataBlockCodec STRING = new StringDataBlockCodec(); public static final DataBlockCodec OBJECT = new ObjectDataBlockCodec(); /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 293d30c40..65a12a49a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -25,15 +25,9 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - /** * Interface for data blocks. A data block has data, a position on the block - * grid, a size, and can read itself from and write itself into a - * {@link ByteBuffer}. + * grid, and a size. * * @param type of the data contained in the DataBlock * @@ -69,54 +63,6 @@ public interface DataBlock { */ T getData(); - /** - * Read (deserialize) the data object of this data block from a {@link ReadData}. - *

    - * The {@code ReadData} may or may not map directly to the data - * object of this data block. I.e. modifying the {@code ReadData} after - * calling this method may or may not change the data of this data block. - * modifying the data object of this data block after calling this method - * may or may not change the content of the {@code ReadData}. - * - * @param byteOrder - * ByteOrder to use for serialization - * @param readData - * data to deserialize - */ - // TODO: include ByteOrder in ReadData - // TODO: rename? "readFrom"? "deserializeFrom"? - void readData(ByteOrder byteOrder, ReadData readData) throws IOException; - - /** - * Creates a {@link ReadData} that contains the serialized data object of - * this data block. - *

    - * The {@code ReadData} may or may not map directly to the data - * object of this data block. I.e. modifying the {@code ReadData} after - * calling this method may or may not change the data of this data block. - * modifying the data object of this data block after calling this method - * may or may not change the content of the {@code ReadData}. - * - * @param byteOrder - * ByteOrder to use for serialization - * - * @return serialized {@code ReadData} - */ - // TODO: rename? "serialize"? "write"? - ReadData writeData(ByteOrder byteOrder); - - // TODO revise, clean up - DataCodec getDataCodec(); - - // TODO revise, clean up - interface DataCodec { - - ReadData serialize(DataBlock dataBlock) throws IOException; - - void deserialize(ReadData readData, DataBlock dataBlock) throws IOException; - } - - /** * Returns the number of elements in this {@link DataBlock}. This number is * not necessarily equal {@link #getNumElements(int[]) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index c3144a51f..93a159fa4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -34,6 +34,7 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import org.janelia.saalfeldlab.n5.Codecs.DataBlockCodec; /** * Enumerates available data types. @@ -44,112 +45,101 @@ public enum DataType { UINT8( "uint8", - Byte.BYTES, (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, - new byte[numElements]) - ), + new byte[numElements]), + Codecs.BYTE), UINT16( "uint16", - Short.BYTES, (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( blockSize, gridPosition, - new short[numElements]) - ), + new short[numElements]), + Codecs.SHORT), UINT32( "uint32", - Integer.BYTES, (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( blockSize, gridPosition, - new int[numElements]) - ), + new int[numElements]), + Codecs.INT), UINT64( "uint64", - Long.BYTES, (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( blockSize, gridPosition, - new long[numElements]) - ), + new long[numElements]), + Codecs.LONG), INT8( "int8", - Byte.BYTES, (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, - new byte[numElements]) - ), + new byte[numElements]), + Codecs.BYTE), INT16( "int16", - Short.BYTES, (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( blockSize, gridPosition, - new short[numElements]) - ), + new short[numElements]), + Codecs.SHORT), INT32( "int32", - Integer.BYTES, (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( blockSize, gridPosition, - new int[numElements]) - ), + new int[numElements]), + Codecs.INT), INT64( "int64", - Long.BYTES, (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( blockSize, gridPosition, - new long[numElements]) - ), + new long[numElements]), + Codecs.LONG), FLOAT32( "float32", - Float.BYTES, (blockSize, gridPosition, numElements) -> new FloatArrayDataBlock( blockSize, gridPosition, - new float[numElements]) - ), + new float[numElements]), + Codecs.FLOAT), FLOAT64( "float64", - Double.BYTES, (blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock( blockSize, gridPosition, - new double[numElements]) - ), + new double[numElements]), + Codecs.DOUBLE), STRING( "string", - -1, (blockSize, gridPosition, numElements) -> new StringDataBlock( blockSize, gridPosition, - null) - ), + new String[numElements]), + Codecs.STRING), OBJECT( "object", - 1, (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, - new byte[numElements]) - ); + new byte[numElements]), + Codecs.OBJECT); + private final String label; private final DataBlockFactory dataBlockFactory; - private final int bytesPerElement; + private final DataBlockCodec defaultCodec; - DataType(final String label, final int bytesPerElement, final DataBlockFactory dataBlockFactory) { + DataType(final String label, final DataBlockFactory dataBlockFactory, final DataBlockCodec defaultCodec) { this.label = label; this.dataBlockFactory = dataBlockFactory; - this.bytesPerElement = bytesPerElement; + this.defaultCodec = defaultCodec; } @Override @@ -199,16 +189,14 @@ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosi } /** - * TODO: javadoc - * explain that STRING and OBJECT are a bit weird ... - * -1 means varlength + * Get the default {@link DataBlockCodec} for {@link DataBlock DataBlocks} + * of this {@code DataType}. The default codec is used for de/serializing + * blocks to N5 format. + * + * @return the default {@code DataBlockCodec} */ - public int bytesPerElement() { - return bytesPerElement; - } - - public boolean isVarLength() { - return bytesPerElement < 0; + public > DataBlockCodec defaultCodec() { + return (DataBlockCodec) defaultCodec; } private interface DataBlockFactory { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 4847985c0..a4601552f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -25,10 +25,9 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; -import org.janelia.saalfeldlab.n5.DataBlock.DataCodec; +import org.janelia.saalfeldlab.n5.Codecs.DataBlockCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** @@ -58,34 +57,8 @@ static DataBlock readBlock( final long[] gridPosition) throws IOException { final DataType dataType = datasetAttributes.getDataType(); - final DataInputStream dis = new DataInputStream(in); - final short mode = dis.readShort(); - final int numElements; - final DataBlock dataBlock; - if (mode != 2) { - final int nDim = dis.readShort(); - final int[] blockSize = new int[nDim]; - for (int d = 0; d < nDim; ++d) - blockSize[d] = dis.readInt(); - if (mode == 0) { - numElements = DataBlock.getNumElements(blockSize); - } else { - numElements = dis.readInt(); - } - dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); - } else { - numElements = dis.readInt(); - dataBlock = dataType.createDataBlock(null, gridPosition, numElements); - } - - final int numBytes = dataType.isVarLength() - ? numElements - : (numElements * dataType.bytesPerElement()); - final ReadData data = datasetAttributes.getCompression().decode( - ReadData.from(in), numBytes); - final DataCodec codec = dataBlock.getDataCodec(); - codec.deserialize(data, dataBlock); - - return dataBlock; + final Compression compression = datasetAttributes.getCompression(); + final DataBlockCodec codec = dataType.defaultCodec(); + return codec.decode(ReadData.from(in), gridPosition, compression); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index 351663425..ddd09a5a6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -25,10 +25,9 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataOutputStream; import java.io.IOException; import java.io.OutputStream; -import org.janelia.saalfeldlab.n5.DataBlock.DataCodec; +import org.janelia.saalfeldlab.n5.Codecs.DataBlockCodec; /** * Default implementation of block writing (N5 format). @@ -56,32 +55,10 @@ static void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - final DataOutputStream dos = new DataOutputStream(out); - - final int mode; - if (datasetAttributes.getDataType() == DataType.OBJECT || dataBlock.getSize() == null) - mode = 2; - else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSize())) - mode = 0; - else - mode = 1; - dos.writeShort(mode); - - if (mode != 2) { - dos.writeShort(datasetAttributes.getNumDimensions()); - for (final int size : dataBlock.getSize()) - dos.writeInt(size); - } - - if (mode != 0) - dos.writeInt(dataBlock.getNumElements()); - - dos.flush(); - - final DataCodec codec = dataBlock.getDataCodec(); - datasetAttributes.getCompression().encode( - codec.serialize(dataBlock) - ).writeTo(out); + final DataType dataType = datasetAttributes.getDataType(); + final Compression compression = datasetAttributes.getCompression(); + final DataBlockCodec> codec = dataType.defaultCodec(); + codec.encode(dataBlock, compression).writeTo(out); out.flush(); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 29b066f9d..393feaddb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -25,11 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - public class DoubleArrayDataBlock extends AbstractDataBlock { public DoubleArrayDataBlock(final int[] size, final long[] gridPosition, final double[] data) { @@ -37,18 +32,6 @@ public DoubleArrayDataBlock(final int[] size, final long[] gridPosition, final d super(size, gridPosition, data); } - @Override - public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - readData.toByteBuffer().order(byteOrder).asDoubleBuffer().get(data); - } - - @Override - public ReadData writeData(final ByteOrder byteOrder) { - final ByteBuffer serialized = ByteBuffer.allocate(Double.BYTES * data.length); - serialized.order(byteOrder).asDoubleBuffer().put(data); - return ReadData.from(serialized); - } - @Override public int getNumElements() { return data.length; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index 3f37c4ed0..c2b1bb501 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -25,11 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - public class FloatArrayDataBlock extends AbstractDataBlock { public FloatArrayDataBlock(final int[] size, final long[] gridPosition, final float[] data) { @@ -37,18 +32,6 @@ public FloatArrayDataBlock(final int[] size, final long[] gridPosition, final fl super(size, gridPosition, data); } - @Override - public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - readData.toByteBuffer().order(byteOrder).asFloatBuffer().get(data); - } - - @Override - public ReadData writeData(final ByteOrder byteOrder) { - final ByteBuffer serialized = ByteBuffer.allocate(Float.BYTES * data.length); - serialized.order(byteOrder).asFloatBuffer().put(data); - return ReadData.from(serialized); - } - @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 26614adf5..355a7a13f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -25,30 +25,13 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - - public class IntArrayDataBlock extends AbstractDataBlock { + public IntArrayDataBlock(final int[] size, final long[] gridPosition, final int[] data) { super(size, gridPosition, data); } - @Override - public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - readData.toByteBuffer().order(byteOrder).asIntBuffer().get(data); - } - - @Override - public ReadData writeData(final ByteOrder byteOrder) { - final ByteBuffer serialized = ByteBuffer.allocate(Integer.BYTES * data.length); - serialized.order(byteOrder).asIntBuffer().put(data); - return ReadData.from(serialized); - } - @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index 346e83339..2669e0f8f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -25,11 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - public class LongArrayDataBlock extends AbstractDataBlock { public LongArrayDataBlock(final int[] size, final long[] gridPosition, final long[] data) { @@ -37,18 +32,6 @@ public LongArrayDataBlock(final int[] size, final long[] gridPosition, final lon super(size, gridPosition, data); } - @Override - public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - readData.toByteBuffer().order(byteOrder).asLongBuffer().get(data); - } - - @Override - public ReadData writeData(final ByteOrder byteOrder) { - final ByteBuffer serialized = ByteBuffer.allocate(Long.BYTES * data.length); - serialized.order(byteOrder).asLongBuffer().put(data); - return ReadData.from(serialized); - } - @Override public int getNumElements() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 2d0cd216d..c1d79e59e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -42,7 +42,6 @@ import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.janelia.saalfeldlab.n5.DataBlock.DataCodec; /** * A simple structured container for hierarchies of chunked @@ -319,16 +318,12 @@ default T readSerializedBlock( final DatasetAttributes attributes, final long... gridPosition) throws N5Exception, ClassNotFoundException { - final DataBlock block = readBlock(dataset, attributes, gridPosition); + final DataBlock block = (DataBlock) readBlock(dataset, attributes, gridPosition); if (block == null) return null; - - final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.toByteBuffer().array()); + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.getData()); try (ObjectInputStream in = new ObjectInputStream(byteArrayInputStream)) { - -// final DataCodec codec = block.getDataCodec(); -// try (ObjectInputStream in = new ObjectInputStream(codec.serialize(block).inputStream())) { return (T) in.readObject(); } catch (final IOException | UncheckedIOException e) { throw new N5Exception.N5IOException(e); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 5a1ff60e9..64c842d83 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -25,11 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - public class ShortArrayDataBlock extends AbstractDataBlock { public ShortArrayDataBlock(final int[] size, final long[] gridPosition, final short[] data) { @@ -37,41 +32,9 @@ public ShortArrayDataBlock(final int[] size, final long[] gridPosition, final sh super(size, gridPosition, data); } - @Override - public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public ReadData writeData(final ByteOrder byteOrder) { - throw new UnsupportedOperationException(); - } - @Override public int getNumElements() { return data.length; } - - static class DefaultCodec implements DataCodec { - - @Override - public ReadData serialize(final DataBlock dataBlock) throws IOException { - final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * dataBlock.getNumElements()); - serialized.order(ByteOrder.BIG_ENDIAN).asShortBuffer().put(dataBlock.getData()); - return ReadData.from(serialized); - } - - @Override - public void deserialize(final ReadData readData, final DataBlock dataBlock) throws IOException { - readData.toByteBuffer().order(ByteOrder.BIG_ENDIAN).asShortBuffer().get(dataBlock.getData()); - } - - static DefaultCodec INSTANCE = new DefaultCodec(); - } - - @Override - public DataCodec getDataCodec() { - return DefaultCodec.INSTANCE; - } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index f39717c52..2e3bd22b6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -25,79 +25,16 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.nio.ByteOrder; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - public class StringDataBlock extends AbstractDataBlock { - protected static final Charset ENCODING = StandardCharsets.UTF_8; - protected static final String NULLCHAR = "\0"; - protected byte[] serializedData; - protected String[] actualData; - public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { - super(size, gridPosition, null); - actualData = data; - } - @Override - public void readData(final ByteOrder byteOrder, final ReadData readData) throws IOException { - serializedData = readData.allBytes(); - final String rawChars = new String(serializedData, ENCODING); - actualData = rawChars.split(NULLCHAR); - } - - private byte[] serialize() { - if (serializedData == null) { - final String flattenedArray = String.join(NULLCHAR, actualData) + NULLCHAR; - serializedData = flattenedArray.getBytes(ENCODING); - } - return serializedData; - } - - @Override - public ReadData writeData(final ByteOrder byteOrder) { - return ReadData.from(serialize()); + super(size, gridPosition, data); } @Override public int getNumElements() { - return serialize().length; - } - @Override - public String[] getData() { - return actualData; + return data.length; } - - - - - - - - static class DefaultCodec implements DataCodec { - @Override - public ReadData serialize(final DataBlock dataBlock) throws IOException { - final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * dataBlock.getNumElements()); - serialized.order(ByteOrder.BIG_ENDIAN).asShortBuffer().put(dataBlock.getData()); - return ReadData.from(serialized); - } - - @Override - public void deserialize(final ReadData readData, final DataBlock dataBlock) throws IOException { - readData.toByteBuffer().order(ByteOrder.BIG_ENDIAN).asShortBuffer().get(dataBlock.getData()); - } - - static DefaultCodec INSTANCE = new DefaultCodec(); - } - - @Override - public DataCodec getDataCodec() { - return DefaultCodec.INSTANCE; - } - } From d8d07195b88948c1e8114574020885c7e736910f Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 14 Feb 2025 23:17:26 +0100 Subject: [PATCH 179/423] Use ProxyOutputStream. FilterOutputStream is slow --- .../java/org/janelia/saalfeldlab/n5/Lz4Compression.java | 6 +----- .../janelia/saalfeldlab/n5/readdata/EncodedReadData.java | 4 ++-- .../saalfeldlab/n5/readdata/KeyValueAccessReadData.java | 4 ++-- .../org/janelia/saalfeldlab/n5/readdata/LazyReadData.java | 2 -- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index e8fdafcc8..3aa58b3cb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -25,15 +25,11 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.FilterOutputStream; import java.io.IOException; - import java.io.InputStream; -import java.io.OutputStream; -import org.janelia.saalfeldlab.n5.Compression.CompressionType; - import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; +import org.janelia.saalfeldlab.n5.Compression.CompressionType; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("lz4") diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java index bc56f4e0d..28af79f22 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java @@ -1,11 +1,11 @@ package org.janelia.saalfeldlab.n5.readdata; import java.io.ByteArrayOutputStream; -import java.io.FilterOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Objects; +import org.apache.commons.io.output.ProxyOutputStream; class EncodedReadData implements ReadData { @@ -74,7 +74,7 @@ public void writeTo(final OutputStream outputStream) throws IOException, Illegal * {@code UnaryOperator} that wraps {@code OutputStream} to intercept {@code * close()} and call {@code flush()} instead */ - private static OutputStreamOperator interceptClose = o -> new FilterOutputStream(o) { + private static OutputStreamOperator interceptClose = o -> new ProxyOutputStream(o) { @Override public void close() throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java index 8ad582646..ea114c061 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java @@ -1,8 +1,8 @@ package org.janelia.saalfeldlab.n5.readdata; -import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import org.apache.commons.io.input.ProxyInputStream; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; @@ -31,7 +31,7 @@ class KeyValueAccessReadData extends AbstractInputStreamReadData { @Override public InputStream inputStream() throws IOException { final LockedChannel channel = keyValueAccess.lockForReading(normalPath); - return new FilterInputStream(channel.newInputStream()) { + return new ProxyInputStream(channel.newInputStream()) { @Override public void close() throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java index fe3d8f7a1..b87282d31 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -1,11 +1,9 @@ package org.janelia.saalfeldlab.n5.readdata; import java.io.ByteArrayOutputStream; -import java.io.FilterOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.Objects; class LazyReadData implements ReadData { From a4fb38e54d75f7270f00865b78f085cdd9822e66 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 15 Feb 2025 00:15:03 +0100 Subject: [PATCH 180/423] use only array type in DataBlockCodec generics --- .../org/janelia/saalfeldlab/n5/Codecs.java | 66 +++++++++---------- .../org/janelia/saalfeldlab/n5/DataType.java | 5 +- .../saalfeldlab/n5/DefaultBlockWriter.java | 2 +- 3 files changed, 38 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java index 66285e123..c2020bdad 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java @@ -24,7 +24,7 @@ public class Codecs { - public static abstract class DataCodec { + public static abstract class DataCodec { public static final DataCodec BYTE = new ByteDataCodec(); public static final DataCodec SHORT = new ShortDataCodec(ByteOrder.BIG_ENDIAN); @@ -39,15 +39,15 @@ public static abstract class DataCodec { public static final DataCodec FLOAT_LITTLE_ENDIAN = new FloatDataCodec(ByteOrder.LITTLE_ENDIAN); public static final DataCodec DOUBLE_LITTLE_ENDIAN = new DoubleDataCodec(ByteOrder.LITTLE_ENDIAN); - public abstract ReadData serialize(A data) throws IOException; + public abstract ReadData serialize(T data) throws IOException; - public abstract void deserialize(ReadData readData, A data) throws IOException; + public abstract void deserialize(ReadData readData, T data) throws IOException; public int bytesPerElement() { return bytesPerElement; } - public A createData(final int numElements) { + public T createData(final int numElements) { return dataFactory.apply(numElements); } @@ -55,9 +55,9 @@ public A createData(final int numElements) { // private final int bytesPerElement; - private final IntFunction dataFactory; + private final IntFunction dataFactory; - private DataCodec(int bytesPerElement, IntFunction dataFactory) { + private DataCodec(int bytesPerElement, IntFunction dataFactory) { this.bytesPerElement = bytesPerElement; this.dataFactory = dataFactory; } @@ -201,45 +201,45 @@ public void deserialize(final ReadData readData, final double[] data) throws IOE - public interface DataBlockCodec> { + public interface DataBlockCodec { - ReadData encode(B dataBlock, Compression compression) throws IOException; + ReadData encode(DataBlock dataBlock, Compression compression) throws IOException; - B decode(ReadData readData, long[] gridPosition, Compression compression) throws IOException; + DataBlock decode(ReadData readData, long[] gridPosition, Compression compression) throws IOException; } - public static final DataBlockCodec BYTE = new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new); - public static final DataBlockCodec SHORT = new DefaultDataBlockCodec<>(DataCodec.SHORT, ShortArrayDataBlock::new); - public static final DataBlockCodec INT = new DefaultDataBlockCodec<>(DataCodec.INT, IntArrayDataBlock::new); - public static final DataBlockCodec LONG = new DefaultDataBlockCodec<>(DataCodec.LONG, LongArrayDataBlock::new); - public static final DataBlockCodec FLOAT = new DefaultDataBlockCodec<>(DataCodec.FLOAT, FloatArrayDataBlock::new); - public static final DataBlockCodec DOUBLE = new DefaultDataBlockCodec<>(DataCodec.DOUBLE, DoubleArrayDataBlock::new); - public static final DataBlockCodec STRING = new StringDataBlockCodec(); - public static final DataBlockCodec OBJECT = new ObjectDataBlockCodec(); + public static final DataBlockCodec BYTE = new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new); + public static final DataBlockCodec SHORT = new DefaultDataBlockCodec<>(DataCodec.SHORT, ShortArrayDataBlock::new); + public static final DataBlockCodec INT = new DefaultDataBlockCodec<>(DataCodec.INT, IntArrayDataBlock::new); + public static final DataBlockCodec LONG = new DefaultDataBlockCodec<>(DataCodec.LONG, LongArrayDataBlock::new); + public static final DataBlockCodec FLOAT = new DefaultDataBlockCodec<>(DataCodec.FLOAT, FloatArrayDataBlock::new); + public static final DataBlockCodec DOUBLE = new DefaultDataBlockCodec<>(DataCodec.DOUBLE, DoubleArrayDataBlock::new); + public static final DataBlockCodec STRING = new StringDataBlockCodec(); + public static final DataBlockCodec OBJECT = new ObjectDataBlockCodec(); /** * DataBlockCodec for all N5 data types, except STRING and OBJECT */ - static class DefaultDataBlockCodec> implements DataBlockCodec { + static class DefaultDataBlockCodec implements DataBlockCodec { - interface DataBlockFactory> { + interface DataBlockFactory { - B createDataBlock(int[] blockSize, long[] gridPosition, A data); + DataBlock createDataBlock(int[] blockSize, long[] gridPosition, T data); } - private final DataCodec dataCodec; + private final DataCodec dataCodec; - private final DataBlockFactory dataBlockFactory; + private final DataBlockFactory dataBlockFactory; DefaultDataBlockCodec( - final DataCodec dataCodec, - final DataBlockFactory dataBlockFactory) { + final DataCodec dataCodec, + final DataBlockFactory dataBlockFactory) { this.dataCodec = dataCodec; this.dataBlockFactory = dataBlockFactory; } @Override - public ReadData encode(final B dataBlock, final Compression compression) throws IOException { + public ReadData encode(final DataBlock dataBlock, final Compression compression) throws IOException { return ReadData.from(out -> { new ChunkHeader(dataBlock.getSize(), dataBlock.getNumElements()).writeTo(out); compression.encode(dataCodec.serialize(dataBlock.getData())).writeTo(out); @@ -248,10 +248,10 @@ public ReadData encode(final B dataBlock, final Compression compression) throws } @Override - public B decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { + public DataBlock decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { try(final InputStream in = readData.inputStream()) { final ChunkHeader header = ChunkHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); - final A data = dataCodec.createData(header.numElements()); + final T data = dataCodec.createData(header.numElements()); final int numBytes = header.numElements() * dataCodec.bytesPerElement(); final ReadData decompressed = compression.decode(ReadData.from(in), numBytes); dataCodec.deserialize(decompressed, data); @@ -263,13 +263,13 @@ public B decode(final ReadData readData, final long[] gridPosition, final Compre /** * TODO javadoc */ - static class StringDataBlockCodec implements DataBlockCodec { + static class StringDataBlockCodec implements DataBlockCodec { private static final Charset ENCODING = StandardCharsets.UTF_8; private static final String NULLCHAR = "\0"; @Override - public ReadData encode(final StringDataBlock dataBlock, final Compression compression) throws IOException { + public ReadData encode(final DataBlock dataBlock, final Compression compression) throws IOException { return ReadData.from(out -> { final String flattenedArray = String.join(NULLCHAR, dataBlock.getData()) + NULLCHAR; final byte[] serializedData = flattenedArray.getBytes(ENCODING); @@ -280,7 +280,7 @@ public ReadData encode(final StringDataBlock dataBlock, final Compression compre } @Override - public StringDataBlock decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { + public DataBlock decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { try(final InputStream in = readData.inputStream()) { final ChunkHeader header = ChunkHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); final ReadData decompressed = compression.decode(ReadData.from(in), header.numElements()); @@ -295,10 +295,10 @@ public StringDataBlock decode(final ReadData readData, final long[] gridPosition /** * TODO javadoc */ - static class ObjectDataBlockCodec implements DataBlockCodec { + static class ObjectDataBlockCodec implements DataBlockCodec { @Override - public ReadData encode(final ByteArrayDataBlock dataBlock, final Compression compression) throws IOException { + public ReadData encode(final DataBlock dataBlock, final Compression compression) throws IOException { return ReadData.from(out -> { new ChunkHeader(null, dataBlock.getNumElements()).writeTo(out); compression.encode(ReadData.from(dataBlock.getData())).writeTo(out); @@ -307,7 +307,7 @@ public ReadData encode(final ByteArrayDataBlock dataBlock, final Compression com } @Override - public ByteArrayDataBlock decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { + public DataBlock decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { try(final InputStream in = readData.inputStream()) { final ChunkHeader header = ChunkHeader.readFrom(in, MODE_OBJECT); final byte[] data = new byte[header.numElements()]; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 93a159fa4..067f9b87f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -193,9 +193,12 @@ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosi * of this {@code DataType}. The default codec is used for de/serializing * blocks to N5 format. * + * @param + * the returned coded is cast to {@code DataBlockCodec} for convenience + * (that is, the caller doesn't have to do the cast explicitly). * @return the default {@code DataBlockCodec} */ - public > DataBlockCodec defaultCodec() { + public DataBlockCodec defaultCodec() { return (DataBlockCodec) defaultCodec; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index ddd09a5a6..c697acde9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -57,7 +57,7 @@ static void writeBlock( final DataType dataType = datasetAttributes.getDataType(); final Compression compression = datasetAttributes.getCompression(); - final DataBlockCodec> codec = dataType.defaultCodec(); + final DataBlockCodec codec = dataType.defaultCodec(); codec.encode(dataBlock, compression).writeTo(out); out.flush(); } From fedfbc7f4977099a883e7246eb85018c0883b019 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 15 Feb 2025 00:27:09 +0100 Subject: [PATCH 181/423] refactor --- .../org/janelia/saalfeldlab/n5/Codecs.java | 203 +----------------- .../org/janelia/saalfeldlab/n5/DataType.java | 24 +-- .../saalfeldlab/n5/codec/DataCodec.java | 182 ++++++++++++++++ 3 files changed, 204 insertions(+), 205 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java index c2020bdad..c96cb8f0a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java @@ -5,11 +5,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.function.IntFunction; +import org.janelia.saalfeldlab.n5.codec.DataCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; import static org.janelia.saalfeldlab.n5.Codecs.ChunkHeader.MODE_DEFAULT; @@ -19,203 +17,22 @@ public class Codecs { - - - - - - public static abstract class DataCodec { - - public static final DataCodec BYTE = new ByteDataCodec(); - public static final DataCodec SHORT = new ShortDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec INT = new IntDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec LONG = new LongDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec FLOAT = new FloatDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec DOUBLE = new DoubleDataCodec(ByteOrder.BIG_ENDIAN); - - public static final DataCodec SHORT_LITTLE_ENDIAN = new ShortDataCodec(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec INT_LITTLE_ENDIAN = new IntDataCodec(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec LONG_LITTLE_ENDIAN = new LongDataCodec(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec FLOAT_LITTLE_ENDIAN = new FloatDataCodec(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec DOUBLE_LITTLE_ENDIAN = new DoubleDataCodec(ByteOrder.LITTLE_ENDIAN); - - public abstract ReadData serialize(T data) throws IOException; - - public abstract void deserialize(ReadData readData, T data) throws IOException; - - public int bytesPerElement() { - return bytesPerElement; - } - - public T createData(final int numElements) { - return dataFactory.apply(numElements); - } - - // ---------------- implementations ----------------- - // - - private final int bytesPerElement; - private final IntFunction dataFactory; - - private DataCodec(int bytesPerElement, IntFunction dataFactory) { - this.bytesPerElement = bytesPerElement; - this.dataFactory = dataFactory; - } - - private static final class ByteDataCodec extends DataCodec { - - private ByteDataCodec() { - super(Byte.BYTES, byte[]::new); - } - - @Override - public ReadData serialize(final byte[] data) { - return ReadData.from(data); - } - - @Override - public void deserialize(final ReadData readData, final byte[] data) throws IOException { - new DataInputStream(readData.inputStream()).readFully(data); - } - } - - private static final class ShortDataCodec extends DataCodec { - - private final ByteOrder order; - - ShortDataCodec(ByteOrder order) { - super(Short.BYTES, short[]::new); - this.order = order; - } - - @Override - public ReadData serialize(final short[] data) { - final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * data.length); - serialized.order(order).asShortBuffer().put(data); - return ReadData.from(serialized); - } - - @Override - public void deserialize(final ReadData readData, final short[] data) throws IOException { - readData.toByteBuffer().order(order).asShortBuffer().get(data); - } - } - - private static final class IntDataCodec extends DataCodec { - - private final ByteOrder order; - - IntDataCodec(ByteOrder order) { - super(Integer.BYTES, int[]::new); - this.order = order; - } - - @Override - public ReadData serialize(final int[] data) { - final ByteBuffer serialized = ByteBuffer.allocate(Integer.BYTES * data.length); - serialized.order(order).asIntBuffer().put(data); - return ReadData.from(serialized); - } - - @Override - public void deserialize(final ReadData readData, final int[] data) throws IOException { - readData.toByteBuffer().order(order).asIntBuffer().get(data); - } - } - - private static final class LongDataCodec extends DataCodec { - - private final ByteOrder order; - - LongDataCodec(ByteOrder order) { - super(Long.BYTES, long[]::new); - this.order = order; - } - - @Override - public ReadData serialize(final long[] data) { - final ByteBuffer serialized = ByteBuffer.allocate(Long.BYTES * data.length); - serialized.order(order).asLongBuffer().put(data); - return ReadData.from(serialized); - } - - @Override - public void deserialize(final ReadData readData, final long[] data) throws IOException { - readData.toByteBuffer().order(order).asLongBuffer().get(data); - } - } - - private static final class FloatDataCodec extends DataCodec { - - private final ByteOrder order; - - FloatDataCodec(ByteOrder order) { - super(Float.BYTES, float[]::new); - this.order = order; - } - - @Override - public ReadData serialize(final float[] data) { - final ByteBuffer serialized = ByteBuffer.allocate(Float.BYTES * data.length); - serialized.order(order).asFloatBuffer().put(data); - return ReadData.from(serialized); - } - - @Override - public void deserialize(final ReadData readData, final float[] data) throws IOException { - readData.toByteBuffer().order(order).asFloatBuffer().get(data); - } - } - - private static final class DoubleDataCodec extends DataCodec { - - private final ByteOrder order; - - DoubleDataCodec(ByteOrder order) { - super(Double.BYTES, double[]::new); - this.order = order; - } - - @Override - public ReadData serialize(final double[] data) { - final ByteBuffer serialized = ByteBuffer.allocate(Double.BYTES * data.length); - serialized.order(order).asDoubleBuffer().put(data); - return ReadData.from(serialized); - } - - @Override - public void deserialize(final ReadData readData, final double[] data) throws IOException { - readData.toByteBuffer().order(order).asDoubleBuffer().get(data); - } - } - } - - - - - - - - - - - - public interface DataBlockCodec { ReadData encode(DataBlock dataBlock, Compression compression) throws IOException; DataBlock decode(ReadData readData, long[] gridPosition, Compression compression) throws IOException; + + DataBlockCodec BYTE = new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new); + DataBlockCodec SHORT = new DefaultDataBlockCodec<>(DataCodec.SHORT, ShortArrayDataBlock::new); + DataBlockCodec INT = new DefaultDataBlockCodec<>(DataCodec.INT, IntArrayDataBlock::new); + DataBlockCodec LONG = new DefaultDataBlockCodec<>(DataCodec.LONG, LongArrayDataBlock::new); + DataBlockCodec FLOAT = new DefaultDataBlockCodec<>(DataCodec.FLOAT, FloatArrayDataBlock::new); + DataBlockCodec DOUBLE = new DefaultDataBlockCodec<>(DataCodec.DOUBLE, DoubleArrayDataBlock::new); + DataBlockCodec STRING = new StringDataBlockCodec(); + DataBlockCodec OBJECT = new ObjectDataBlockCodec(); } - public static final DataBlockCodec BYTE = new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new); - public static final DataBlockCodec SHORT = new DefaultDataBlockCodec<>(DataCodec.SHORT, ShortArrayDataBlock::new); - public static final DataBlockCodec INT = new DefaultDataBlockCodec<>(DataCodec.INT, IntArrayDataBlock::new); - public static final DataBlockCodec LONG = new DefaultDataBlockCodec<>(DataCodec.LONG, LongArrayDataBlock::new); - public static final DataBlockCodec FLOAT = new DefaultDataBlockCodec<>(DataCodec.FLOAT, FloatArrayDataBlock::new); - public static final DataBlockCodec DOUBLE = new DefaultDataBlockCodec<>(DataCodec.DOUBLE, DoubleArrayDataBlock::new); - public static final DataBlockCodec STRING = new StringDataBlockCodec(); - public static final DataBlockCodec OBJECT = new ObjectDataBlockCodec(); /** * DataBlockCodec for all N5 data types, except STRING and OBJECT diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 067f9b87f..9e247b947 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -49,84 +49,84 @@ public enum DataType { blockSize, gridPosition, new byte[numElements]), - Codecs.BYTE), + DataBlockCodec.BYTE), UINT16( "uint16", (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( blockSize, gridPosition, new short[numElements]), - Codecs.SHORT), + DataBlockCodec.SHORT), UINT32( "uint32", (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( blockSize, gridPosition, new int[numElements]), - Codecs.INT), + DataBlockCodec.INT), UINT64( "uint64", (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( blockSize, gridPosition, new long[numElements]), - Codecs.LONG), + DataBlockCodec.LONG), INT8( "int8", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, new byte[numElements]), - Codecs.BYTE), + DataBlockCodec.BYTE), INT16( "int16", (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( blockSize, gridPosition, new short[numElements]), - Codecs.SHORT), + DataBlockCodec.SHORT), INT32( "int32", (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( blockSize, gridPosition, new int[numElements]), - Codecs.INT), + DataBlockCodec.INT), INT64( "int64", (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( blockSize, gridPosition, new long[numElements]), - Codecs.LONG), + DataBlockCodec.LONG), FLOAT32( "float32", (blockSize, gridPosition, numElements) -> new FloatArrayDataBlock( blockSize, gridPosition, new float[numElements]), - Codecs.FLOAT), + DataBlockCodec.FLOAT), FLOAT64( "float64", (blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock( blockSize, gridPosition, new double[numElements]), - Codecs.DOUBLE), + DataBlockCodec.DOUBLE), STRING( "string", (blockSize, gridPosition, numElements) -> new StringDataBlock( blockSize, gridPosition, new String[numElements]), - Codecs.STRING), + DataBlockCodec.STRING), OBJECT( "object", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, new byte[numElements]), - Codecs.OBJECT); + DataBlockCodec.OBJECT); private final String label; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java new file mode 100644 index 000000000..ddf013d70 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -0,0 +1,182 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.function.IntFunction; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +/** + * TODO javadoc + * ... writes data contained in a DataBlock to byte[]. + * ... versions for BIG_ENDIAN and LITTLE_ENDIAN byte order. + * + * @param + * type of the data contained in the DataBlock + */ +public abstract class DataCodec { + + public static final DataCodec BYTE = new ByteDataCodec(); + public static final DataCodec SHORT = new ShortDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec INT = new IntDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec LONG = new LongDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec FLOAT = new FloatDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec DOUBLE = new DoubleDataCodec(ByteOrder.BIG_ENDIAN); + + public static final DataCodec SHORT_LITTLE_ENDIAN = new ShortDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec INT_LITTLE_ENDIAN = new IntDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec LONG_LITTLE_ENDIAN = new LongDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec FLOAT_LITTLE_ENDIAN = new FloatDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec DOUBLE_LITTLE_ENDIAN = new DoubleDataCodec(ByteOrder.LITTLE_ENDIAN); + + public abstract ReadData serialize(T data) throws IOException; + + public abstract void deserialize(ReadData readData, T data) throws IOException; + + public int bytesPerElement() { + return bytesPerElement; + } + + public T createData(final int numElements) { + return dataFactory.apply(numElements); + } + + // ---------------- implementations ----------------- + // + + private final int bytesPerElement; + private final IntFunction dataFactory; + + private DataCodec(int bytesPerElement, IntFunction dataFactory) { + this.bytesPerElement = bytesPerElement; + this.dataFactory = dataFactory; + } + + private static final class ByteDataCodec extends DataCodec { + + private ByteDataCodec() { + super(Byte.BYTES, byte[]::new); + } + + @Override + public ReadData serialize(final byte[] data) { + return ReadData.from(data); + } + + @Override + public void deserialize(final ReadData readData, final byte[] data) throws IOException { + new DataInputStream(readData.inputStream()).readFully(data); + } + } + + private static final class ShortDataCodec extends DataCodec { + + private final ByteOrder order; + + ShortDataCodec(ByteOrder order) { + super(Short.BYTES, short[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final short[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * data.length); + serialized.order(order).asShortBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public void deserialize(final ReadData readData, final short[] data) throws IOException { + readData.toByteBuffer().order(order).asShortBuffer().get(data); + } + } + + private static final class IntDataCodec extends DataCodec { + + private final ByteOrder order; + + IntDataCodec(ByteOrder order) { + super(Integer.BYTES, int[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final int[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Integer.BYTES * data.length); + serialized.order(order).asIntBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public void deserialize(final ReadData readData, final int[] data) throws IOException { + readData.toByteBuffer().order(order).asIntBuffer().get(data); + } + } + + private static final class LongDataCodec extends DataCodec { + + private final ByteOrder order; + + LongDataCodec(ByteOrder order) { + super(Long.BYTES, long[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final long[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Long.BYTES * data.length); + serialized.order(order).asLongBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public void deserialize(final ReadData readData, final long[] data) throws IOException { + readData.toByteBuffer().order(order).asLongBuffer().get(data); + } + } + + private static final class FloatDataCodec extends DataCodec { + + private final ByteOrder order; + + FloatDataCodec(ByteOrder order) { + super(Float.BYTES, float[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final float[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Float.BYTES * data.length); + serialized.order(order).asFloatBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public void deserialize(final ReadData readData, final float[] data) throws IOException { + readData.toByteBuffer().order(order).asFloatBuffer().get(data); + } + } + + private static final class DoubleDataCodec extends DataCodec { + + private final ByteOrder order; + + DoubleDataCodec(ByteOrder order) { + super(Double.BYTES, double[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final double[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Double.BYTES * data.length); + serialized.order(order).asDoubleBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public void deserialize(final ReadData readData, final double[] data) throws IOException { + readData.toByteBuffer().order(order).asDoubleBuffer().get(data); + } + } +} From 2a0cd24854eba5552163bf7fe9b9e22c8426f6a7 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 15 Feb 2025 13:09:05 +0100 Subject: [PATCH 182/423] refactor --- .../org/janelia/saalfeldlab/n5/DataType.java | 27 ++++++----- .../saalfeldlab/n5/DefaultBlockReader.java | 2 +- .../saalfeldlab/n5/DefaultBlockWriter.java | 2 +- .../saalfeldlab/n5/{ => codec}/Codecs.java | 47 ++++++++++--------- .../saalfeldlab/n5/codec/DataBlockCodec.java | 13 +++++ .../saalfeldlab/n5/codec/DataCodec.java | 12 ++--- 6 files changed, 59 insertions(+), 44 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/{ => codec}/Codecs.java (79%) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 9e247b947..a0a80d43d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -34,7 +34,8 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; -import org.janelia.saalfeldlab.n5.Codecs.DataBlockCodec; +import org.janelia.saalfeldlab.n5.codec.Codecs; +import org.janelia.saalfeldlab.n5.codec.DataBlockCodec; /** * Enumerates available data types. @@ -49,84 +50,84 @@ public enum DataType { blockSize, gridPosition, new byte[numElements]), - DataBlockCodec.BYTE), + Codecs.BYTE), UINT16( "uint16", (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( blockSize, gridPosition, new short[numElements]), - DataBlockCodec.SHORT), + Codecs.SHORT), UINT32( "uint32", (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( blockSize, gridPosition, new int[numElements]), - DataBlockCodec.INT), + Codecs.INT), UINT64( "uint64", (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( blockSize, gridPosition, new long[numElements]), - DataBlockCodec.LONG), + Codecs.LONG), INT8( "int8", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, new byte[numElements]), - DataBlockCodec.BYTE), + Codecs.BYTE), INT16( "int16", (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( blockSize, gridPosition, new short[numElements]), - DataBlockCodec.SHORT), + Codecs.SHORT), INT32( "int32", (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( blockSize, gridPosition, new int[numElements]), - DataBlockCodec.INT), + Codecs.INT), INT64( "int64", (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( blockSize, gridPosition, new long[numElements]), - DataBlockCodec.LONG), + Codecs.LONG), FLOAT32( "float32", (blockSize, gridPosition, numElements) -> new FloatArrayDataBlock( blockSize, gridPosition, new float[numElements]), - DataBlockCodec.FLOAT), + Codecs.FLOAT), FLOAT64( "float64", (blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock( blockSize, gridPosition, new double[numElements]), - DataBlockCodec.DOUBLE), + Codecs.DOUBLE), STRING( "string", (blockSize, gridPosition, numElements) -> new StringDataBlock( blockSize, gridPosition, new String[numElements]), - DataBlockCodec.STRING), + Codecs.STRING), OBJECT( "object", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, new byte[numElements]), - DataBlockCodec.OBJECT); + Codecs.OBJECT); private final String label; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index a4601552f..43532e4e0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -27,7 +27,7 @@ import java.io.IOException; import java.io.InputStream; -import org.janelia.saalfeldlab.n5.Codecs.DataBlockCodec; +import org.janelia.saalfeldlab.n5.codec.DataBlockCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index c697acde9..1a284fbc9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -27,7 +27,7 @@ import java.io.IOException; import java.io.OutputStream; -import org.janelia.saalfeldlab.n5.Codecs.DataBlockCodec; +import org.janelia.saalfeldlab.n5.codec.DataBlockCodec; /** * Default implementation of block writing (N5 format). diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codecs.java similarity index 79% rename from src/main/java/org/janelia/saalfeldlab/n5/Codecs.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/Codecs.java index c96cb8f0a..92990c17f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codecs.java @@ -1,4 +1,4 @@ -package org.janelia.saalfeldlab.n5; +package org.janelia.saalfeldlab.n5.codec; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -7,32 +7,33 @@ import java.io.OutputStream; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import org.janelia.saalfeldlab.n5.codec.DataCodec; +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.Compression; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock; +import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; +import org.janelia.saalfeldlab.n5.IntArrayDataBlock; +import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; +import org.janelia.saalfeldlab.n5.StringDataBlock; import org.janelia.saalfeldlab.n5.readdata.ReadData; -import static org.janelia.saalfeldlab.n5.Codecs.ChunkHeader.MODE_DEFAULT; -import static org.janelia.saalfeldlab.n5.Codecs.ChunkHeader.MODE_OBJECT; -import static org.janelia.saalfeldlab.n5.Codecs.ChunkHeader.MODE_VARLENGTH; +import static org.janelia.saalfeldlab.n5.codec.Codecs.ChunkHeader.MODE_DEFAULT; +import static org.janelia.saalfeldlab.n5.codec.Codecs.ChunkHeader.MODE_OBJECT; +import static org.janelia.saalfeldlab.n5.codec.Codecs.ChunkHeader.MODE_VARLENGTH; public class Codecs { + public static final DataBlockCodec BYTE = new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new); + public static final DataBlockCodec SHORT = new DefaultDataBlockCodec<>(DataCodec.SHORT_BIG_ENDIAN, ShortArrayDataBlock::new); + public static final DataBlockCodec INT = new DefaultDataBlockCodec<>(DataCodec.INT_BIG_ENDIAN, IntArrayDataBlock::new); + public static final DataBlockCodec LONG = new DefaultDataBlockCodec<>(DataCodec.LONG_BIG_ENDIAN, LongArrayDataBlock::new); + public static final DataBlockCodec FLOAT = new DefaultDataBlockCodec<>(DataCodec.FLOAT_BIG_ENDIAN, FloatArrayDataBlock::new); + public static final DataBlockCodec DOUBLE = new DefaultDataBlockCodec<>(DataCodec.DOUBLE_BIG_ENDIAN, DoubleArrayDataBlock::new); + public static final DataBlockCodec STRING = new StringDataBlockCodec(); + public static final DataBlockCodec OBJECT = new ObjectDataBlockCodec(); - public interface DataBlockCodec { - - ReadData encode(DataBlock dataBlock, Compression compression) throws IOException; - - DataBlock decode(ReadData readData, long[] gridPosition, Compression compression) throws IOException; - - DataBlockCodec BYTE = new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new); - DataBlockCodec SHORT = new DefaultDataBlockCodec<>(DataCodec.SHORT, ShortArrayDataBlock::new); - DataBlockCodec INT = new DefaultDataBlockCodec<>(DataCodec.INT, IntArrayDataBlock::new); - DataBlockCodec LONG = new DefaultDataBlockCodec<>(DataCodec.LONG, LongArrayDataBlock::new); - DataBlockCodec FLOAT = new DefaultDataBlockCodec<>(DataCodec.FLOAT, FloatArrayDataBlock::new); - DataBlockCodec DOUBLE = new DefaultDataBlockCodec<>(DataCodec.DOUBLE, DoubleArrayDataBlock::new); - DataBlockCodec STRING = new StringDataBlockCodec(); - DataBlockCodec OBJECT = new ObjectDataBlockCodec(); - } - + private Codecs() {} /** * DataBlockCodec for all N5 data types, except STRING and OBJECT @@ -78,7 +79,7 @@ public DataBlock decode(final ReadData readData, final long[] gridPosition, f } /** - * TODO javadoc + * DataBlockCodec for N5 data type STRING */ static class StringDataBlockCodec implements DataBlockCodec { @@ -110,7 +111,7 @@ public DataBlock decode(final ReadData readData, final long[] gridPosi } /** - * TODO javadoc + * DataBlockCodec for N5 data type OBJECT */ static class ObjectDataBlockCodec implements DataBlockCodec { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java new file mode 100644 index 000000000..1543f20ee --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java @@ -0,0 +1,13 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.io.IOException; +import org.janelia.saalfeldlab.n5.Compression; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +public interface DataBlockCodec { + + ReadData encode(DataBlock dataBlock, Compression compression) throws IOException; + + DataBlock decode(ReadData readData, long[] gridPosition, Compression compression) throws IOException; +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java index ddf013d70..316f9411e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -17,12 +17,12 @@ */ public abstract class DataCodec { - public static final DataCodec BYTE = new ByteDataCodec(); - public static final DataCodec SHORT = new ShortDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec INT = new IntDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec LONG = new LongDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec FLOAT = new FloatDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec DOUBLE = new DoubleDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec BYTE = new ByteDataCodec(); + public static final DataCodec SHORT_BIG_ENDIAN = new ShortDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec INT_BIG_ENDIAN = new IntDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec LONG_BIG_ENDIAN = new LongDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec FLOAT_BIG_ENDIAN = new FloatDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec DOUBLE_BIG_ENDIAN = new DoubleDataCodec(ByteOrder.BIG_ENDIAN); public static final DataCodec SHORT_LITTLE_ENDIAN = new ShortDataCodec(ByteOrder.LITTLE_ENDIAN); public static final DataCodec INT_LITTLE_ENDIAN = new IntDataCodec(ByteOrder.LITTLE_ENDIAN); From 72986216b5e699c0956e27b38c37af1d770d62c6 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 16 Feb 2025 11:35:34 +0100 Subject: [PATCH 183/423] convenience DataCodec methods to get codec with approprioate endianness --- .../saalfeldlab/n5/codec/DataCodec.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java index 316f9411e..3160c4689 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -30,6 +30,26 @@ public abstract class DataCodec { public static final DataCodec FLOAT_LITTLE_ENDIAN = new FloatDataCodec(ByteOrder.LITTLE_ENDIAN); public static final DataCodec DOUBLE_LITTLE_ENDIAN = new DoubleDataCodec(ByteOrder.LITTLE_ENDIAN); + public static DataCodec SHORT(ByteOrder order) { + return order == ByteOrder.BIG_ENDIAN ? SHORT_BIG_ENDIAN : SHORT_LITTLE_ENDIAN; + } + + public static DataCodec INT(ByteOrder order) { + return order == ByteOrder.BIG_ENDIAN ? INT_BIG_ENDIAN : INT_LITTLE_ENDIAN; + } + + public static DataCodec LONG(ByteOrder order) { + return order == ByteOrder.BIG_ENDIAN ? LONG_BIG_ENDIAN : LONG_LITTLE_ENDIAN; + } + + public static DataCodec FLOAT(ByteOrder order) { + return order == ByteOrder.BIG_ENDIAN ? FLOAT_BIG_ENDIAN : FLOAT_LITTLE_ENDIAN; + } + + public static DataCodec DOUBLE(ByteOrder order) { + return order == ByteOrder.BIG_ENDIAN ? DOUBLE_BIG_ENDIAN : DOUBLE_LITTLE_ENDIAN; + } + public abstract ReadData serialize(T data) throws IOException; public abstract void deserialize(ReadData readData, T data) throws IOException; From fa88b72515272a146d4c897fa5282a5069a808de Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 16 Feb 2025 12:02:19 +0100 Subject: [PATCH 184/423] typo --- src/main/java/org/janelia/saalfeldlab/n5/DataType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index a0a80d43d..ef792e819 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -195,7 +195,7 @@ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosi * blocks to N5 format. * * @param - * the returned coded is cast to {@code DataBlockCodec} for convenience + * the returned codec is cast to {@code DataBlockCodec} for convenience * (that is, the caller doesn't have to do the cast explicitly). * @return the default {@code DataBlockCodec} */ From 4290e0b414df26ae694cf82448d68c9f1c48e9eb Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 16 Feb 2025 16:03:54 +0100 Subject: [PATCH 185/423] refactor --- .../saalfeldlab/n5/AbstractDataBlock.java | 22 ++++++++++++--- .../saalfeldlab/n5/ByteArrayDataBlock.java | 8 +----- .../org/janelia/saalfeldlab/n5/DataBlock.java | 11 ++++++++ .../saalfeldlab/n5/DoubleArrayDataBlock.java | 7 +---- .../saalfeldlab/n5/FloatArrayDataBlock.java | 8 +----- .../saalfeldlab/n5/IntArrayDataBlock.java | 8 +----- .../saalfeldlab/n5/LongArrayDataBlock.java | 8 +----- .../saalfeldlab/n5/ShortArrayDataBlock.java | 8 +----- .../saalfeldlab/n5/StringDataBlock.java | 8 +----- .../janelia/saalfeldlab/n5/codec/Codecs.java | 6 +---- .../saalfeldlab/n5/codec/DataCodec.java | 27 ++++++++++--------- 11 files changed, 52 insertions(+), 69 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index 822ceddab..9fe705f81 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -25,6 +25,8 @@ */ package org.janelia.saalfeldlab.n5; +import java.util.function.ToIntFunction; + /** * Abstract base class for {@link DataBlock} implementations. * @@ -35,15 +37,21 @@ */ public abstract class AbstractDataBlock implements DataBlock { - protected final int[] size; - protected final long[] gridPosition; - protected final T data; + private final int[] size; + private final long[] gridPosition; + private final T data; + private final ToIntFunction numElements; - public AbstractDataBlock(final int[] size, final long[] gridPosition, final T data) { + public AbstractDataBlock( + final int[] size, + final long[] gridPosition, + final T data, + final ToIntFunction numElements) { this.size = size; this.gridPosition = gridPosition; this.data = data; + this.numElements = numElements; } @Override @@ -63,4 +71,10 @@ public T getData() { return data; } + + @Override + public int getNumElements() { + + return numElements.applyAsInt(data); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index 78539c281..23857e596 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -29,12 +29,6 @@ public class ByteArrayDataBlock extends AbstractDataBlock { public ByteArrayDataBlock(final int[] size, final long[] gridPosition, final byte[] data) { - super(size, gridPosition, data); - } - - @Override - public int getNumElements() { - - return data.length; + super(size, gridPosition, data, a -> a.length); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 65a12a49a..cb8f5b685 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -86,4 +86,15 @@ static int getNumElements(final int[] size) { n *= size[i]; return n; } + + /** + * Factory for creating {@code DataBlock}. + * + * @param + * type of the data contained in the DataBlock + */ + interface DataBlockFactory { + + DataBlock createDataBlock(int[] blockSize, long[] gridPosition, T data); + } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 393feaddb..df5333257 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -29,11 +29,6 @@ public class DoubleArrayDataBlock extends AbstractDataBlock { public DoubleArrayDataBlock(final int[] size, final long[] gridPosition, final double[] data) { - super(size, gridPosition, data); - } - - @Override - public int getNumElements() { - return data.length; + super(size, gridPosition, data, a -> a.length); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index c2b1bb501..71065a0e7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -29,12 +29,6 @@ public class FloatArrayDataBlock extends AbstractDataBlock { public FloatArrayDataBlock(final int[] size, final long[] gridPosition, final float[] data) { - super(size, gridPosition, data); - } - - @Override - public int getNumElements() { - - return data.length; + super(size, gridPosition, data, a -> a.length); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 355a7a13f..56b32f309 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -29,12 +29,6 @@ public class IntArrayDataBlock extends AbstractDataBlock { public IntArrayDataBlock(final int[] size, final long[] gridPosition, final int[] data) { - super(size, gridPosition, data); - } - - @Override - public int getNumElements() { - - return data.length; + super(size, gridPosition, data, a -> a.length); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index 2669e0f8f..960ccd5ec 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -29,12 +29,6 @@ public class LongArrayDataBlock extends AbstractDataBlock { public LongArrayDataBlock(final int[] size, final long[] gridPosition, final long[] data) { - super(size, gridPosition, data); - } - - @Override - public int getNumElements() { - - return data.length; + super(size, gridPosition, data, a -> a.length); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 64c842d83..856c8cfe5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -29,12 +29,6 @@ public class ShortArrayDataBlock extends AbstractDataBlock { public ShortArrayDataBlock(final int[] size, final long[] gridPosition, final short[] data) { - super(size, gridPosition, data); - } - - @Override - public int getNumElements() { - - return data.length; + super(size, gridPosition, data, a -> a.length); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 2e3bd22b6..291934d24 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -29,12 +29,6 @@ public class StringDataBlock extends AbstractDataBlock { public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { - super(size, gridPosition, data); - } - - @Override - public int getNumElements() { - - return data.length; + super(size, gridPosition, data, a -> a.length); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codecs.java index 92990c17f..19d992304 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codecs.java @@ -10,6 +10,7 @@ import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataBlock.DataBlockFactory; import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock; import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; import org.janelia.saalfeldlab.n5.IntArrayDataBlock; @@ -40,11 +41,6 @@ private Codecs() {} */ static class DefaultDataBlockCodec implements DataBlockCodec { - interface DataBlockFactory { - - DataBlock createDataBlock(int[] blockSize, long[] gridPosition, T data); - } - private final DataCodec dataCodec; private final DataBlockFactory dataBlockFactory; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java index 3160c4689..e10fd7f44 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -17,6 +17,21 @@ */ public abstract class DataCodec { + public abstract ReadData serialize(T data) throws IOException; + + public abstract void deserialize(ReadData readData, T data) throws IOException; + + public int bytesPerElement() { + return bytesPerElement; + } + + public T createData(final int numElements) { + return dataFactory.apply(numElements); + } + + // ------------------- instances -------------------- + // + public static final DataCodec BYTE = new ByteDataCodec(); public static final DataCodec SHORT_BIG_ENDIAN = new ShortDataCodec(ByteOrder.BIG_ENDIAN); public static final DataCodec INT_BIG_ENDIAN = new IntDataCodec(ByteOrder.BIG_ENDIAN); @@ -50,18 +65,6 @@ public static DataCodec DOUBLE(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? DOUBLE_BIG_ENDIAN : DOUBLE_LITTLE_ENDIAN; } - public abstract ReadData serialize(T data) throws IOException; - - public abstract void deserialize(ReadData readData, T data) throws IOException; - - public int bytesPerElement() { - return bytesPerElement; - } - - public T createData(final int numElements) { - return dataFactory.apply(numElements); - } - // ---------------- implementations ----------------- // From ffea7a8842442050404b4b3a558d514e3077462e Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 16 Feb 2025 16:04:33 +0100 Subject: [PATCH 186/423] refactor --- .../org/janelia/saalfeldlab/n5/DataType.java | 26 +++++++++---------- .../n5/codec/{Codecs.java => N5Codecs.java} | 10 +++---- 2 files changed, 18 insertions(+), 18 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/codec/{Codecs.java => N5Codecs.java} (96%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index ef792e819..6a90418b2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -34,7 +34,7 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; -import org.janelia.saalfeldlab.n5.codec.Codecs; +import org.janelia.saalfeldlab.n5.codec.N5Codecs; import org.janelia.saalfeldlab.n5.codec.DataBlockCodec; /** @@ -50,84 +50,84 @@ public enum DataType { blockSize, gridPosition, new byte[numElements]), - Codecs.BYTE), + N5Codecs.BYTE), UINT16( "uint16", (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( blockSize, gridPosition, new short[numElements]), - Codecs.SHORT), + N5Codecs.SHORT), UINT32( "uint32", (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( blockSize, gridPosition, new int[numElements]), - Codecs.INT), + N5Codecs.INT), UINT64( "uint64", (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( blockSize, gridPosition, new long[numElements]), - Codecs.LONG), + N5Codecs.LONG), INT8( "int8", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, new byte[numElements]), - Codecs.BYTE), + N5Codecs.BYTE), INT16( "int16", (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( blockSize, gridPosition, new short[numElements]), - Codecs.SHORT), + N5Codecs.SHORT), INT32( "int32", (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( blockSize, gridPosition, new int[numElements]), - Codecs.INT), + N5Codecs.INT), INT64( "int64", (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( blockSize, gridPosition, new long[numElements]), - Codecs.LONG), + N5Codecs.LONG), FLOAT32( "float32", (blockSize, gridPosition, numElements) -> new FloatArrayDataBlock( blockSize, gridPosition, new float[numElements]), - Codecs.FLOAT), + N5Codecs.FLOAT), FLOAT64( "float64", (blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock( blockSize, gridPosition, new double[numElements]), - Codecs.DOUBLE), + N5Codecs.DOUBLE), STRING( "string", (blockSize, gridPosition, numElements) -> new StringDataBlock( blockSize, gridPosition, new String[numElements]), - Codecs.STRING), + N5Codecs.STRING), OBJECT( "object", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( blockSize, gridPosition, new byte[numElements]), - Codecs.OBJECT); + N5Codecs.OBJECT); private final String label; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java similarity index 96% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/Codecs.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index 19d992304..684e5fc88 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -19,11 +19,11 @@ import org.janelia.saalfeldlab.n5.StringDataBlock; import org.janelia.saalfeldlab.n5.readdata.ReadData; -import static org.janelia.saalfeldlab.n5.codec.Codecs.ChunkHeader.MODE_DEFAULT; -import static org.janelia.saalfeldlab.n5.codec.Codecs.ChunkHeader.MODE_OBJECT; -import static org.janelia.saalfeldlab.n5.codec.Codecs.ChunkHeader.MODE_VARLENGTH; +import static org.janelia.saalfeldlab.n5.codec.N5Codecs.ChunkHeader.MODE_DEFAULT; +import static org.janelia.saalfeldlab.n5.codec.N5Codecs.ChunkHeader.MODE_OBJECT; +import static org.janelia.saalfeldlab.n5.codec.N5Codecs.ChunkHeader.MODE_VARLENGTH; -public class Codecs { +public class N5Codecs { public static final DataBlockCodec BYTE = new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new); public static final DataBlockCodec SHORT = new DefaultDataBlockCodec<>(DataCodec.SHORT_BIG_ENDIAN, ShortArrayDataBlock::new); @@ -34,7 +34,7 @@ public class Codecs { public static final DataBlockCodec STRING = new StringDataBlockCodec(); public static final DataBlockCodec OBJECT = new ObjectDataBlockCodec(); - private Codecs() {} + private N5Codecs() {} /** * DataBlockCodec for all N5 data types, except STRING and OBJECT From d8b7c1f4596c80fbe225ea7a9cb5aa35f8f85d49 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 16 Feb 2025 16:29:02 +0100 Subject: [PATCH 187/423] Let DatasetAttributes provide the DataBlockCodec avoids the Compression argument to encode/decode methods --- .../org/janelia/saalfeldlab/n5/DataType.java | 22 ++++---- .../saalfeldlab/n5/DatasetAttributes.java | 25 +++++++++ .../saalfeldlab/n5/DefaultBlockReader.java | 6 +-- .../saalfeldlab/n5/DefaultBlockWriter.java | 6 +-- .../saalfeldlab/n5/codec/DataBlockCodec.java | 5 +- .../saalfeldlab/n5/codec/N5Codecs.java | 51 +++++++++++++------ 6 files changed, 78 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 6a90418b2..0aece9d19 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -36,6 +36,7 @@ import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.codec.N5Codecs; import org.janelia.saalfeldlab.n5.codec.DataBlockCodec; +import org.janelia.saalfeldlab.n5.codec.N5Codecs.DataBlockCodecFactory; /** * Enumerates available data types. @@ -134,13 +135,13 @@ public enum DataType { private final DataBlockFactory dataBlockFactory; - private final DataBlockCodec defaultCodec; + private final DataBlockCodecFactory dataBlockCodecFactory; - DataType(final String label, final DataBlockFactory dataBlockFactory, final DataBlockCodec defaultCodec) { + DataType(final String label, final DataBlockFactory dataBlockFactory, final DataBlockCodecFactory dataBlockCodecFactory) { this.label = label; this.dataBlockFactory = dataBlockFactory; - this.defaultCodec = defaultCodec; + this.dataBlockCodecFactory = dataBlockCodecFactory; } @Override @@ -190,17 +191,16 @@ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosi } /** - * Get the default {@link DataBlockCodec} for {@link DataBlock DataBlocks} - * of this {@code DataType}. The default codec is used for de/serializing - * blocks to N5 format. + * Get the default {@link DataBlockCodec}, with the specified {@code + * compression}, for {@link DataBlock DataBlocks} of this {@code DataType}. + * The default codec is used for de/serializing blocks to N5 format. + * + * @param compression * - * @param - * the returned codec is cast to {@code DataBlockCodec} for convenience - * (that is, the caller doesn't have to do the cast explicitly). * @return the default {@code DataBlockCodec} */ - public DataBlockCodec defaultCodec() { - return (DataBlockCodec) defaultCodec; + public DataBlockCodec createDataBlockCodec(final Compression compression) { + return dataBlockCodecFactory.createDataBlockCodec(compression); } private interface DataBlockFactory { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index f4aea9fe5..dc9b51fed 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -28,6 +28,7 @@ import java.io.Serializable; import java.util.Arrays; import java.util.HashMap; +import org.janelia.saalfeldlab.n5.codec.DataBlockCodec; /** * Mandatory dataset attributes: @@ -58,6 +59,7 @@ public class DatasetAttributes implements Serializable { private final int[] blockSize; private final DataType dataType; private final Compression compression; + private final DataBlockCodec dataBlockCodec; public DatasetAttributes( final long[] dimensions, @@ -65,10 +67,21 @@ public DatasetAttributes( final DataType dataType, final Compression compression) { + this(dimensions, blockSize, dataType, compression, dataType.createDataBlockCodec(compression)); + } + + protected DatasetAttributes( + final long[] dimensions, + final int[] blockSize, + final DataType dataType, + final Compression compression, + final DataBlockCodec dataBlockCodec) { + this.dimensions = dimensions; this.blockSize = blockSize; this.dataType = dataType; this.compression = compression; + this.dataBlockCodec = dataBlockCodec; } public long[] getDimensions() { @@ -96,6 +109,18 @@ public DataType getDataType() { return dataType; } + /** + * Get the {@link DataBlockCodec} for this dataset. + * + * @param + * the returned codec is cast to {@code DataBlockCodec} for convenience + * (that is, the caller doesn't have to do the cast explicitly). + * @return the {@code DataBlockCodec} for this dataset + */ + public DataBlockCodec getDataBlockCodec() { + return (DataBlockCodec) dataBlockCodec; + } + public HashMap asMap() { final HashMap map = new HashMap<>(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 43532e4e0..5b248ca80 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -56,9 +56,7 @@ static DataBlock readBlock( final DatasetAttributes datasetAttributes, final long[] gridPosition) throws IOException { - final DataType dataType = datasetAttributes.getDataType(); - final Compression compression = datasetAttributes.getCompression(); - final DataBlockCodec codec = dataType.defaultCodec(); - return codec.decode(ReadData.from(in), gridPosition, compression); + final DataBlockCodec codec = datasetAttributes.getDataBlockCodec(); + return codec.decode(ReadData.from(in), gridPosition); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index 1a284fbc9..d91458401 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -55,10 +55,8 @@ static void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - final DataType dataType = datasetAttributes.getDataType(); - final Compression compression = datasetAttributes.getCompression(); - final DataBlockCodec codec = dataType.defaultCodec(); - codec.encode(dataBlock, compression).writeTo(out); + final DataBlockCodec codec = datasetAttributes.getDataBlockCodec(); + codec.encode(dataBlock).writeTo(out); out.flush(); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java index 1543f20ee..d71c858be 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java @@ -1,13 +1,12 @@ package org.janelia.saalfeldlab.n5.codec; import java.io.IOException; -import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.readdata.ReadData; public interface DataBlockCodec { - ReadData encode(DataBlock dataBlock, Compression compression) throws IOException; + ReadData encode(DataBlock dataBlock) throws IOException; - DataBlock decode(ReadData readData, long[] gridPosition, Compression compression) throws IOException; + DataBlock decode(ReadData readData, long[] gridPosition) throws IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index 684e5fc88..9120f102d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -25,17 +25,22 @@ public class N5Codecs { - public static final DataBlockCodec BYTE = new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new); - public static final DataBlockCodec SHORT = new DefaultDataBlockCodec<>(DataCodec.SHORT_BIG_ENDIAN, ShortArrayDataBlock::new); - public static final DataBlockCodec INT = new DefaultDataBlockCodec<>(DataCodec.INT_BIG_ENDIAN, IntArrayDataBlock::new); - public static final DataBlockCodec LONG = new DefaultDataBlockCodec<>(DataCodec.LONG_BIG_ENDIAN, LongArrayDataBlock::new); - public static final DataBlockCodec FLOAT = new DefaultDataBlockCodec<>(DataCodec.FLOAT_BIG_ENDIAN, FloatArrayDataBlock::new); - public static final DataBlockCodec DOUBLE = new DefaultDataBlockCodec<>(DataCodec.DOUBLE_BIG_ENDIAN, DoubleArrayDataBlock::new); - public static final DataBlockCodec STRING = new StringDataBlockCodec(); - public static final DataBlockCodec OBJECT = new ObjectDataBlockCodec(); + public static final DataBlockCodecFactory BYTE = c -> new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new, c); + public static final DataBlockCodecFactory SHORT = c -> new DefaultDataBlockCodec<>(DataCodec.SHORT_BIG_ENDIAN, ShortArrayDataBlock::new, c); + public static final DataBlockCodecFactory INT = c -> new DefaultDataBlockCodec<>(DataCodec.INT_BIG_ENDIAN, IntArrayDataBlock::new, c); + public static final DataBlockCodecFactory LONG = c -> new DefaultDataBlockCodec<>(DataCodec.LONG_BIG_ENDIAN, LongArrayDataBlock::new, c); + public static final DataBlockCodecFactory FLOAT = c -> new DefaultDataBlockCodec<>(DataCodec.FLOAT_BIG_ENDIAN, FloatArrayDataBlock::new, c); + public static final DataBlockCodecFactory DOUBLE = c -> new DefaultDataBlockCodec<>(DataCodec.DOUBLE_BIG_ENDIAN, DoubleArrayDataBlock::new, c); + public static final DataBlockCodecFactory STRING = StringDataBlockCodec::new; + public static final DataBlockCodecFactory OBJECT = ObjectDataBlockCodec::new; private N5Codecs() {} + public interface DataBlockCodecFactory { + + DataBlockCodec createDataBlockCodec(Compression compression); + } + /** * DataBlockCodec for all N5 data types, except STRING and OBJECT */ @@ -45,15 +50,19 @@ static class DefaultDataBlockCodec implements DataBlockCodec { private final DataBlockFactory dataBlockFactory; + private final Compression compression; + DefaultDataBlockCodec( final DataCodec dataCodec, - final DataBlockFactory dataBlockFactory) { + final DataBlockFactory dataBlockFactory, + final Compression compression) { this.dataCodec = dataCodec; this.dataBlockFactory = dataBlockFactory; + this.compression = compression; } @Override - public ReadData encode(final DataBlock dataBlock, final Compression compression) throws IOException { + public ReadData encode(final DataBlock dataBlock) throws IOException { return ReadData.from(out -> { new ChunkHeader(dataBlock.getSize(), dataBlock.getNumElements()).writeTo(out); compression.encode(dataCodec.serialize(dataBlock.getData())).writeTo(out); @@ -62,7 +71,7 @@ public ReadData encode(final DataBlock dataBlock, final Compression compressi } @Override - public DataBlock decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { + public DataBlock decode(final ReadData readData, final long[] gridPosition) throws IOException { try(final InputStream in = readData.inputStream()) { final ChunkHeader header = ChunkHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); final T data = dataCodec.createData(header.numElements()); @@ -82,8 +91,14 @@ static class StringDataBlockCodec implements DataBlockCodec { private static final Charset ENCODING = StandardCharsets.UTF_8; private static final String NULLCHAR = "\0"; + private final Compression compression; + + StringDataBlockCodec(final Compression compression) { + this.compression = compression; + } + @Override - public ReadData encode(final DataBlock dataBlock, final Compression compression) throws IOException { + public ReadData encode(final DataBlock dataBlock) throws IOException { return ReadData.from(out -> { final String flattenedArray = String.join(NULLCHAR, dataBlock.getData()) + NULLCHAR; final byte[] serializedData = flattenedArray.getBytes(ENCODING); @@ -94,7 +109,7 @@ public ReadData encode(final DataBlock dataBlock, final Compression co } @Override - public DataBlock decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { + public DataBlock decode(final ReadData readData, final long[] gridPosition) throws IOException { try(final InputStream in = readData.inputStream()) { final ChunkHeader header = ChunkHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); final ReadData decompressed = compression.decode(ReadData.from(in), header.numElements()); @@ -111,8 +126,14 @@ public DataBlock decode(final ReadData readData, final long[] gridPosi */ static class ObjectDataBlockCodec implements DataBlockCodec { + private final Compression compression; + + ObjectDataBlockCodec(final Compression compression) { + this.compression = compression; + } + @Override - public ReadData encode(final DataBlock dataBlock, final Compression compression) throws IOException { + public ReadData encode(final DataBlock dataBlock) throws IOException { return ReadData.from(out -> { new ChunkHeader(null, dataBlock.getNumElements()).writeTo(out); compression.encode(ReadData.from(dataBlock.getData())).writeTo(out); @@ -121,7 +142,7 @@ public ReadData encode(final DataBlock dataBlock, final Compression comp } @Override - public DataBlock decode(final ReadData readData, final long[] gridPosition, final Compression compression) throws IOException { + public DataBlock decode(final ReadData readData, final long[] gridPosition) throws IOException { try(final InputStream in = readData.inputStream()) { final ChunkHeader header = ChunkHeader.readFrom(in, MODE_OBJECT); final byte[] data = new byte[header.numElements()]; From f191b7d62d4759ccd8ab83b118594dde30db1bb5 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 19 Feb 2025 10:27:12 +0100 Subject: [PATCH 188/423] refactor, add ReadData.materialize() This is in preparation for moving SplittableReadData into a separate PR --- .../saalfeldlab/n5/codec/DataBlockCodec.java | 6 ++ .../saalfeldlab/n5/codec/DataCodec.java | 10 ++- .../readdata/AbstractInputStreamReadData.java | 7 +- .../readdata/ByteArraySplittableReadData.java | 5 ++ .../n5/readdata/EncodedReadData.java | 84 ------------------- .../saalfeldlab/n5/readdata/LazyReadData.java | 41 ++++++++- .../saalfeldlab/n5/readdata/ReadData.java | 31 ++++++- 7 files changed, 91 insertions(+), 93 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java index d71c858be..4141be663 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java @@ -4,6 +4,12 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.readdata.ReadData; +/** + * De/serialize {@link DataBlock DataBlock} from/to {@link ReadData}. + * + * @param + * type of the data contained in the DataBlock + */ public interface DataBlockCodec { ReadData encode(DataBlock dataBlock) throws IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java index e10fd7f44..9f306ecbd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -5,12 +5,16 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.function.IntFunction; +import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** - * TODO javadoc - * ... writes data contained in a DataBlock to byte[]. - * ... versions for BIG_ENDIAN and LITTLE_ENDIAN byte order. + * De/serialize the {@link DataBlock#getData() data} contained in a {@code + * DataBlock} from/to a sequence of bytes. + *

    + * Static fields {@code BYTE}, {@code SHORT_BIG_ENDIAN}, {@code + * SHORT_LITTLE_ENDIAN}, etc. contain {@code DataCodec}s for all primitive array + * types and big-endian / little-endian byte order. * * @param * type of the data contained in the DataBlock diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java index 0340e70e0..1dfb815ac 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java @@ -11,6 +11,11 @@ abstract class AbstractInputStreamReadData implements ReadData { @Override public SplittableReadData splittable() throws IOException { + return materialize(); + } + + @Override + public SplittableReadData materialize() throws IOException { if (bytes == null) { final byte[] data; final int length = (int) length(); @@ -27,6 +32,6 @@ public SplittableReadData splittable() throws IOException { @Override public byte[] allBytes() throws IOException, IllegalStateException { - return splittable().allBytes(); + return materialize().allBytes(); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java index 1fadedfff..d3f807497 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java @@ -45,6 +45,11 @@ public SplittableReadData splittable() throws IOException { return this; } + @Override + public SplittableReadData materialize() throws IOException { + return this; + } + @Override public SplittableReadData split(final long offset, final long length) throws IOException { if (offset < 0 || offset > this.length || length < 0) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java deleted file mode 100644 index 28af79f22..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/EncodedReadData.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.janelia.saalfeldlab.n5.readdata; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Objects; -import org.apache.commons.io.output.ProxyOutputStream; - -class EncodedReadData implements ReadData { - - /** - * Like {@code UnaryOperator}, but {@code apply} throws {@code IOException}. - */ - @FunctionalInterface - public interface OutputStreamOperator { - - OutputStream apply(OutputStream o) throws IOException; - - default OutputStreamOperator andThen(OutputStreamOperator after) { - Objects.requireNonNull(after); - return o -> after.apply(apply(o)); - } - } - - EncodedReadData(final ReadData data, final OutputStreamOperator encoder) { - this.source = data; - this.encoder = interceptClose.andThen(encoder)::apply; - } - - private final ReadData source; - - private final OutputStreamOperator encoder; - - private ByteArraySplittableReadData bytes; - - @Override - public SplittableReadData splittable() throws IOException { - if (bytes == null) { - final ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); - writeTo(baos); - bytes = new ByteArraySplittableReadData(baos.toByteArray()); - } - return bytes; - } - - @Override - public long length() throws IOException { - return splittable().length(); - } - - @Override - public InputStream inputStream() throws IOException, IllegalStateException { - return splittable().inputStream(); - } - - @Override - public byte[] allBytes() throws IOException, IllegalStateException { - return splittable().allBytes(); - } - - @Override - public void writeTo(final OutputStream outputStream) throws IOException, IllegalStateException { - if (bytes != null) { - outputStream.write(bytes.allBytes()); - } else { - try (final OutputStream deflater = encoder.apply(outputStream)) { - source.writeTo(deflater); - } - } - } - - /** - * {@code UnaryOperator} that wraps {@code OutputStream} to intercept {@code - * close()} and call {@code flush()} instead - */ - private static OutputStreamOperator interceptClose = o -> new ProxyOutputStream(o) { - - @Override - public void close() throws IOException { - out.flush(); - } - }; -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java index b87282d31..f71c7b301 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import org.apache.commons.io.output.ProxyOutputStream; class LazyReadData implements ReadData { @@ -11,12 +12,34 @@ class LazyReadData implements ReadData { this.writer = writer; } + /** + * Construct a {@code LazyReadData} that uses the given {@code OutputStreamEncoder} to + * encode the given {@code ReadData}. + * + * @param data + * the ReadData to encode + * @param encoder + * OutputStreamEncoder to use for encoding + */ + LazyReadData(final ReadData data, final OutputStreamOperator encoder) { + this(outputStream -> { + try (final OutputStream deflater = encoder.apply(interceptClose.apply(outputStream))) { + data.writeTo(deflater); + } + }); + } + private final OutputStreamWriter writer; private ByteArraySplittableReadData bytes; @Override public SplittableReadData splittable() throws IOException { + return materialize(); + } + + @Override + public SplittableReadData materialize() throws IOException { if (bytes == null) { final ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); writeTo(baos); @@ -27,17 +50,17 @@ public SplittableReadData splittable() throws IOException { @Override public long length() throws IOException { - return splittable().length(); + return materialize().length(); } @Override public InputStream inputStream() throws IOException, IllegalStateException { - return splittable().inputStream(); + return materialize().inputStream(); } @Override public byte[] allBytes() throws IOException, IllegalStateException { - return splittable().allBytes(); + return materialize().allBytes(); } @Override @@ -48,4 +71,16 @@ public void writeTo(final OutputStream outputStream) throws IOException, Illegal writer.writeTo(outputStream); } } + + /** + * {@code UnaryOperator} that wraps {@code OutputStream} to intercept {@code + * close()} and call {@code flush()} instead + */ + private static OutputStreamOperator interceptClose = o -> new ProxyOutputStream(o) { + + @Override + public void close() throws IOException { + out.flush(); + } + }; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 60d3bc840..f8a811598 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -5,7 +5,6 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.readdata.EncodedReadData.OutputStreamOperator; /** * An abstraction over {@code byte[]} data. @@ -98,6 +97,16 @@ default ByteBuffer toByteBuffer() throws IOException, IllegalStateException { return ByteBuffer.wrap(allBytes()); } + /** + * Read the underlying data into a {@code byte[]} array, and return it as a {@code ReadData}. + * (If this {@code ReadData} is already in a {@code byte[]} array or {@code + * ByteBuffer}, just return {@code this}.) + *

    + * The returned {@code ReadData} has a known {@link #length} and multiple + * {@link #inputStream InputStreams} can be opened on it. + */ + SplittableReadData materialize() throws IOException; + /** * If this {@code ReadData} is a {@code SplittableReadData}, just returns {@code this}. *

    @@ -144,7 +153,17 @@ default void writeTo(OutputStream outputStream) throws IOException, IllegalState * @return encoded ReadData */ default ReadData encode(OutputStreamOperator encoder) { - return new EncodedReadData(this, encoder); + return new LazyReadData(this, encoder); + } + + /** + * Like {@code UnaryOperator}, but {@code apply} throws {@code IOException}. + */ + @FunctionalInterface + interface OutputStreamOperator { + + OutputStream apply(OutputStream o) throws IOException; + } // --------------- Factory Methods ------------------ @@ -247,6 +266,14 @@ interface OutputStreamWriter { void writeTo(OutputStream outputStream) throws IOException, IllegalStateException; } + /** + * Create a new {@code ReadData} that is lazily generated by the given {@link OutputStreamWriter}. + * + * @param generator + * generates the data + * + * @return a new ReadData + */ static ReadData from(OutputStreamWriter generator) { return new LazyReadData(generator); } From 1154bf9f5510ecce82e89509d1fe2fb837df1de2 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 19 Feb 2025 10:35:29 +0100 Subject: [PATCH 189/423] Remove SplittableReadData interface --- .../readdata/AbstractInputStreamReadData.java | 7 +------ .../readdata/ByteArraySplittableReadData.java | 19 ++----------------- .../saalfeldlab/n5/readdata/LazyReadData.java | 7 +------ .../saalfeldlab/n5/readdata/ReadData.java | 19 ++++--------------- .../n5/readdata/SplittableReadData.java | 8 -------- 5 files changed, 8 insertions(+), 52 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java index 1dfb815ac..6f636ad75 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java @@ -10,12 +10,7 @@ abstract class AbstractInputStreamReadData implements ReadData { private ByteArraySplittableReadData bytes; @Override - public SplittableReadData splittable() throws IOException { - return materialize(); - } - - @Override - public SplittableReadData materialize() throws IOException { + public ReadData materialize() throws IOException { if (bytes == null) { final byte[] data; final int length = (int) length(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java index d3f807497..a46ef5f64 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java @@ -5,7 +5,7 @@ import java.io.InputStream; import java.util.Arrays; -class ByteArraySplittableReadData implements SplittableReadData { +class ByteArraySplittableReadData implements ReadData { private final byte[] data; private final int offset; @@ -41,22 +41,7 @@ public byte[] allBytes() { } @Override - public SplittableReadData splittable() throws IOException { + public ReadData materialize() throws IOException { return this; } - - @Override - public SplittableReadData materialize() throws IOException { - return this; - } - - @Override - public SplittableReadData split(final long offset, final long length) throws IOException { - if (offset < 0 || offset > this.length || length < 0) { - throw new IndexOutOfBoundsException(); - } - final int o = this.offset + (int) offset; - final int l = Math.min((int) length, this.length - o); - return new ByteArraySplittableReadData(data, o, l); - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java index f71c7b301..b67d99e13 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -34,12 +34,7 @@ class LazyReadData implements ReadData { private ByteArraySplittableReadData bytes; @Override - public SplittableReadData splittable() throws IOException { - return materialize(); - } - - @Override - public SplittableReadData materialize() throws IOException { + public ReadData materialize() throws IOException { if (bytes == null) { final ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); writeTo(baos); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index f8a811598..21f2ab3e3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -105,18 +105,7 @@ default ByteBuffer toByteBuffer() throws IOException, IllegalStateException { * The returned {@code ReadData} has a known {@link #length} and multiple * {@link #inputStream InputStreams} can be opened on it. */ - SplittableReadData materialize() throws IOException; - - /** - * If this {@code ReadData} is a {@code SplittableReadData}, just returns {@code this}. - *

    - * Otherwise, if the underlying data is an {@code InputStream}, all data is read and - * wrapped as a {@code ByteArraySplittableReadData}. - *

    - * The returned {@code SplittableReadData} has a known {@link #length} - * and multiple {@link #inputStream}s can be opened on it. - */ - SplittableReadData splittable() throws IOException; + ReadData materialize() throws IOException; /** * Write the contained data into an {@code OutputStream}. @@ -229,7 +218,7 @@ static ReadData from(final KeyValueAccess keyValueAccess, final String normalPat * * @return a new ReadData */ - static SplittableReadData from(final byte[] data, final int offset, final int length) { + static ReadData from(final byte[] data, final int offset, final int length) { return new ByteArraySplittableReadData(data, offset, length); } @@ -241,7 +230,7 @@ static SplittableReadData from(final byte[] data, final int offset, final int le * * @return a new ReadData */ - static SplittableReadData from(final byte[] data) { + static ReadData from(final byte[] data) { return from(data, 0, data.length); } @@ -253,7 +242,7 @@ static SplittableReadData from(final byte[] data) { * * @return a new ReadData */ - static SplittableReadData from(final ByteBuffer data) { + static ReadData from(final ByteBuffer data) { if (data.hasArray()) { return from(data.array(), 0, data.limit()); } else { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java deleted file mode 100644 index 396359c98..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.janelia.saalfeldlab.n5.readdata; - -import java.io.IOException; - -public interface SplittableReadData extends ReadData { - - ReadData split(final long offset, final long length) throws IOException; -} From 31f351e14274209157815e80b7dfed2a747b71f7 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 19 Feb 2025 10:47:19 +0100 Subject: [PATCH 190/423] fix javadoc error --- .../java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java index 4141be663..c840ace99 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java @@ -5,7 +5,7 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; /** - * De/serialize {@link DataBlock DataBlock} from/to {@link ReadData}. + * De/serialize {@link DataBlock} from/to {@link ReadData}. * * @param * type of the data contained in the DataBlock From 4db22cfd276944c6b0003110e9b82961a2f6e9b8 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 26 Feb 2025 22:13:31 +0100 Subject: [PATCH 191/423] rename ChunkHeader to BlockHeader --- .../saalfeldlab/n5/codec/N5Codecs.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index 9120f102d..3dec98271 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -19,9 +19,9 @@ import org.janelia.saalfeldlab.n5.StringDataBlock; import org.janelia.saalfeldlab.n5.readdata.ReadData; -import static org.janelia.saalfeldlab.n5.codec.N5Codecs.ChunkHeader.MODE_DEFAULT; -import static org.janelia.saalfeldlab.n5.codec.N5Codecs.ChunkHeader.MODE_OBJECT; -import static org.janelia.saalfeldlab.n5.codec.N5Codecs.ChunkHeader.MODE_VARLENGTH; +import static org.janelia.saalfeldlab.n5.codec.N5Codecs.BlockHeader.MODE_DEFAULT; +import static org.janelia.saalfeldlab.n5.codec.N5Codecs.BlockHeader.MODE_OBJECT; +import static org.janelia.saalfeldlab.n5.codec.N5Codecs.BlockHeader.MODE_VARLENGTH; public class N5Codecs { @@ -64,7 +64,7 @@ static class DefaultDataBlockCodec implements DataBlockCodec { @Override public ReadData encode(final DataBlock dataBlock) throws IOException { return ReadData.from(out -> { - new ChunkHeader(dataBlock.getSize(), dataBlock.getNumElements()).writeTo(out); + new BlockHeader(dataBlock.getSize(), dataBlock.getNumElements()).writeTo(out); compression.encode(dataCodec.serialize(dataBlock.getData())).writeTo(out); out.flush(); }); @@ -73,7 +73,7 @@ public ReadData encode(final DataBlock dataBlock) throws IOException { @Override public DataBlock decode(final ReadData readData, final long[] gridPosition) throws IOException { try(final InputStream in = readData.inputStream()) { - final ChunkHeader header = ChunkHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); + final BlockHeader header = BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); final T data = dataCodec.createData(header.numElements()); final int numBytes = header.numElements() * dataCodec.bytesPerElement(); final ReadData decompressed = compression.decode(ReadData.from(in), numBytes); @@ -102,7 +102,7 @@ public ReadData encode(final DataBlock dataBlock) throws IOException { return ReadData.from(out -> { final String flattenedArray = String.join(NULLCHAR, dataBlock.getData()) + NULLCHAR; final byte[] serializedData = flattenedArray.getBytes(ENCODING); - new ChunkHeader(dataBlock.getSize(), serializedData.length).writeTo(out); + new BlockHeader(dataBlock.getSize(), serializedData.length).writeTo(out); compression.encode(ReadData.from(serializedData)).writeTo(out); out.flush(); }); @@ -111,7 +111,7 @@ public ReadData encode(final DataBlock dataBlock) throws IOException { @Override public DataBlock decode(final ReadData readData, final long[] gridPosition) throws IOException { try(final InputStream in = readData.inputStream()) { - final ChunkHeader header = ChunkHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); + final BlockHeader header = BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); final ReadData decompressed = compression.decode(ReadData.from(in), header.numElements()); final byte[] serializedData = decompressed.allBytes(); final String rawChars = new String(serializedData, ENCODING); @@ -135,7 +135,7 @@ static class ObjectDataBlockCodec implements DataBlockCodec { @Override public ReadData encode(final DataBlock dataBlock) throws IOException { return ReadData.from(out -> { - new ChunkHeader(null, dataBlock.getNumElements()).writeTo(out); + new BlockHeader(null, dataBlock.getNumElements()).writeTo(out); compression.encode(ReadData.from(dataBlock.getData())).writeTo(out); out.flush(); }); @@ -144,7 +144,7 @@ public ReadData encode(final DataBlock dataBlock) throws IOException { @Override public DataBlock decode(final ReadData readData, final long[] gridPosition) throws IOException { try(final InputStream in = readData.inputStream()) { - final ChunkHeader header = ChunkHeader.readFrom(in, MODE_OBJECT); + final BlockHeader header = BlockHeader.readFrom(in, MODE_OBJECT); final byte[] data = new byte[header.numElements()]; final ReadData decompressed = compression.decode(ReadData.from(in), data.length); new DataInputStream(decompressed.inputStream()).readFully(data); @@ -153,7 +153,7 @@ public DataBlock decode(final ReadData readData, final long[] gridPositi } } - static class ChunkHeader { + static class BlockHeader { public static final short MODE_DEFAULT = 0; public static final short MODE_VARLENGTH = 1; @@ -163,13 +163,13 @@ static class ChunkHeader { private final int[] blockSize; private final int numElements; - ChunkHeader(final short mode, final int[] blockSize, final int numElements) { + BlockHeader(final short mode, final int[] blockSize, final int numElements) { this.mode = mode; this.blockSize = blockSize; this.numElements = numElements; } - ChunkHeader(final int[] blockSize, final int numElements) { + BlockHeader(final int[] blockSize, final int numElements) { if (blockSize == null) { this.mode = MODE_OBJECT; } else if (DataBlock.getNumElements(blockSize) == numElements) { @@ -227,11 +227,11 @@ void writeTo(final OutputStream out) throws IOException { dos.flush(); } - static ChunkHeader readFrom(final InputStream in) throws IOException { + static BlockHeader readFrom(final InputStream in) throws IOException { return readFrom(in, null); } - static ChunkHeader readFrom(final InputStream in, short... allowedModes) throws IOException { + static BlockHeader readFrom(final InputStream in, short... allowedModes) throws IOException { final DataInputStream dis = new DataInputStream(in); final short mode = dis.readShort(); final int[] blockSize; @@ -262,7 +262,7 @@ static ChunkHeader readFrom(final InputStream in, short... allowedModes) throws throw new IOException("unexpected mode: " + mode); } - return new ChunkHeader(mode, blockSize, numElements); + return new BlockHeader(mode, blockSize, numElements); } } } From 2648d380f070320cff8f3083565d5ccd4b0e9076 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 26 Feb 2025 23:01:50 +0100 Subject: [PATCH 192/423] Remove unnecessary flush()s, instead close OutputStream where it is constructed --- .../org/janelia/saalfeldlab/n5/DefaultBlockWriter.java | 1 - .../org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java | 8 ++++++-- .../java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java | 3 --- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index d91458401..ed91c2abf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -57,6 +57,5 @@ static void writeBlock( final DataBlockCodec codec = datasetAttributes.getDataBlockCodec(); codec.encode(dataBlock).writeTo(out); - out.flush(); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 38c754be9..f16cee6e0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; +import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.Arrays; import java.util.List; @@ -217,8 +218,11 @@ default void writeBlock( final DataBlock dataBlock) throws N5Exception { final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition()); - try (final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath)) { - DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); + try ( + final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath); + final OutputStream out = lock.newOutputStream() + ) { + DefaultBlockWriter.writeBlock(out, datasetAttributes, dataBlock); } catch (final IOException | UncheckedIOException e) { throw new N5IOException( "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index 3dec98271..f08ab1c40 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -66,7 +66,6 @@ public ReadData encode(final DataBlock dataBlock) throws IOException { return ReadData.from(out -> { new BlockHeader(dataBlock.getSize(), dataBlock.getNumElements()).writeTo(out); compression.encode(dataCodec.serialize(dataBlock.getData())).writeTo(out); - out.flush(); }); } @@ -104,7 +103,6 @@ public ReadData encode(final DataBlock dataBlock) throws IOException { final byte[] serializedData = flattenedArray.getBytes(ENCODING); new BlockHeader(dataBlock.getSize(), serializedData.length).writeTo(out); compression.encode(ReadData.from(serializedData)).writeTo(out); - out.flush(); }); } @@ -137,7 +135,6 @@ public ReadData encode(final DataBlock dataBlock) throws IOException { return ReadData.from(out -> { new BlockHeader(null, dataBlock.getNumElements()).writeTo(out); compression.encode(ReadData.from(dataBlock.getData())).writeTo(out); - out.flush(); }); } From 46c7eab42fc70a475fc9bc2880aa6f2d00b22995 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 5 Mar 2025 16:40:52 -0500 Subject: [PATCH 193/423] refactor: don't pass `decodedLength` to ReadData --- .../java/org/janelia/saalfeldlab/n5/Bzip2Compression.java | 6 +++--- src/main/java/org/janelia/saalfeldlab/n5/Compression.java | 4 +--- .../java/org/janelia/saalfeldlab/n5/GzipCompression.java | 6 +++--- .../java/org/janelia/saalfeldlab/n5/Lz4Compression.java | 6 +++--- .../java/org/janelia/saalfeldlab/n5/RawCompression.java | 7 ++----- .../java/org/janelia/saalfeldlab/n5/XzCompression.java | 6 +++--- 6 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 354560279..f81bb5fb7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -60,9 +60,9 @@ public boolean equals(final Object other) { } @Override - public ReadData decode(final ReadData readData, final int decodedLength) throws IOException { - final InputStream inflater = new BZip2CompressorInputStream(readData.inputStream()); - return ReadData.from(inflater, decodedLength); + public ReadData decode(final ReadData readData) throws IOException { + + return ReadData.from(new BZip2CompressorInputStream(readData.inputStream())); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index cb61de922..1f08dd26a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -85,15 +85,13 @@ default String getType() { * * @param readData * data to decode - * @param decodedLength - * length of the decoded data (-1 if unknown) * * @return decoded ReadData * * @throws IOException * if any I/O error occurs */ - ReadData decode(ReadData readData, int decodedLength) throws IOException; + ReadData decode(ReadData readData) throws IOException; /** * Encode the given {@code readData}. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index f9dab5c5b..8a7fa3c6b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -85,9 +85,9 @@ private InputStream decode(final InputStream in) throws IOException { } @Override - public ReadData decode(final ReadData readData, final int decodedLength) throws IOException { - final InputStream inflater = decode(readData.inputStream()); - return ReadData.from(inflater, decodedLength); + public ReadData decode(final ReadData readData) throws IOException { + + return ReadData.from(decode(readData.inputStream())); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index 3aa58b3cb..10f7f9681 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -60,9 +60,9 @@ public boolean equals(final Object other) { } @Override - public ReadData decode(final ReadData readData, final int decodedLength) throws IOException { - final InputStream inflater = new LZ4BlockInputStream(readData.inputStream()); - return ReadData.from(inflater, decodedLength); + public ReadData decode(final ReadData readData) throws IOException { + + return ReadData.from(new LZ4BlockInputStream(readData.inputStream())); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index e895a6fb7..5588d1a9b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -35,10 +35,7 @@ public class RawCompression implements Compression { @Override public boolean equals(final Object other) { - if (other == null || other.getClass() != RawCompression.class) - return false; - else - return true; + return other != null && other.getClass() == RawCompression.class; } @Override @@ -47,7 +44,7 @@ public ReadData encode(final ReadData readData) { } @Override - public ReadData decode(final ReadData readData, int decodedLength) { + public ReadData decode(final ReadData readData) { return readData; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index eb174873a..fd9ad98a3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -60,9 +60,9 @@ public boolean equals(final Object other) { } @Override - public ReadData decode(final ReadData readData, final int decodedLength) throws IOException { - final InputStream inflater = new XZCompressorInputStream(readData.inputStream()); - return ReadData.from(inflater, decodedLength); + public ReadData decode(final ReadData readData) throws IOException { + + return ReadData.from(new XZCompressorInputStream(readData.inputStream())); } @Override From b03e897db8d2d89bc480c8595eaef93225f5b2cc Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 5 Mar 2025 16:48:34 -0500 Subject: [PATCH 194/423] refactor: move DataBlockFactory/DataBlockCodecFactory to N5Codecs --- .../org/janelia/saalfeldlab/n5/DataType.java | 165 ++---------------- .../saalfeldlab/n5/codec/N5Codecs.java | 41 +++++ 2 files changed, 57 insertions(+), 149 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 0aece9d19..8d260583c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -6,10 +6,10 @@ * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE @@ -34,10 +34,6 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; -import org.janelia.saalfeldlab.n5.codec.N5Codecs; -import org.janelia.saalfeldlab.n5.codec.DataBlockCodec; -import org.janelia.saalfeldlab.n5.codec.N5Codecs.DataBlockCodecFactory; - /** * Enumerates available data types. * @@ -45,103 +41,24 @@ */ public enum DataType { - UINT8( - "uint8", - (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( - blockSize, - gridPosition, - new byte[numElements]), - N5Codecs.BYTE), - UINT16( - "uint16", - (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( - blockSize, - gridPosition, - new short[numElements]), - N5Codecs.SHORT), - UINT32( - "uint32", - (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( - blockSize, - gridPosition, - new int[numElements]), - N5Codecs.INT), - UINT64( - "uint64", - (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( - blockSize, - gridPosition, - new long[numElements]), - N5Codecs.LONG), - INT8( - "int8", - (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( - blockSize, - gridPosition, - new byte[numElements]), - N5Codecs.BYTE), - INT16( - "int16", - (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( - blockSize, - gridPosition, - new short[numElements]), - N5Codecs.SHORT), - INT32( - "int32", - (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( - blockSize, - gridPosition, - new int[numElements]), - N5Codecs.INT), - INT64( - "int64", - (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( - blockSize, - gridPosition, - new long[numElements]), - N5Codecs.LONG), - FLOAT32( - "float32", - (blockSize, gridPosition, numElements) -> new FloatArrayDataBlock( - blockSize, - gridPosition, - new float[numElements]), - N5Codecs.FLOAT), - FLOAT64( - "float64", - (blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock( - blockSize, - gridPosition, - new double[numElements]), - N5Codecs.DOUBLE), - STRING( - "string", - (blockSize, gridPosition, numElements) -> new StringDataBlock( - blockSize, - gridPosition, - new String[numElements]), - N5Codecs.STRING), - OBJECT( - "object", - (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( - blockSize, - gridPosition, - new byte[numElements]), - N5Codecs.OBJECT); - + UINT8("uint8"), + UINT16("uint16"), + UINT32("uint32"), + UINT64("uint64"), + INT8("int8"), + INT16("int16"), + INT32("int32"), + INT64("int64"), + FLOAT32("float32"), + FLOAT64("float64"), + STRING("string"), + OBJECT("object"); private final String label; - private final DataBlockFactory dataBlockFactory; - - private final DataBlockCodecFactory dataBlockCodecFactory; - - DataType(final String label, final DataBlockFactory dataBlockFactory, final DataBlockCodecFactory dataBlockCodecFactory) { + DataType(final String label) { this.label = label; - this.dataBlockFactory = dataBlockFactory; - this.dataBlockCodecFactory = dataBlockCodecFactory; } @Override @@ -158,56 +75,6 @@ public static DataType fromString(final String string) { return null; } - /** - * Factory for {@link DataBlock DataBlocks}. - * - * @param blockSize - * the block size - * @param gridPosition - * the grid position - * @param numElements - * the number of elements (not necessarily one element per block - * element) - * @return the data block - */ - public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements) { - - return dataBlockFactory.createDataBlock(blockSize, gridPosition, numElements); - } - - /** - * Factory for {@link DataBlock DataBlocks} with one data element for each - * block element (e.g. pixel image). - * - * @param blockSize - * the block size - * @param gridPosition - * the grid position - * @return the data block - */ - public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition) { - - return dataBlockFactory.createDataBlock(blockSize, gridPosition, DataBlock.getNumElements(blockSize)); - } - - /** - * Get the default {@link DataBlockCodec}, with the specified {@code - * compression}, for {@link DataBlock DataBlocks} of this {@code DataType}. - * The default codec is used for de/serializing blocks to N5 format. - * - * @param compression - * - * @return the default {@code DataBlockCodec} - */ - public DataBlockCodec createDataBlockCodec(final Compression compression) { - return dataBlockCodecFactory.createDataBlockCodec(compression); - } - - private interface DataBlockFactory { - - DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements); - } - static public class JsonAdapter implements JsonDeserializer, JsonSerializer { @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index f08ab1c40..4b244006d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -36,6 +36,47 @@ public class N5Codecs { private N5Codecs() {} + public static DataBlockCodec createDataBlockCodec( + final DataType dataType, + final Compression compression) { + + final DataBlockCodecFactory factory; + switch (dataType) { + case UINT8: + case INT8: + factory = N5Codecs.BYTE; + break; + case UINT16: + case INT16: + factory = N5Codecs.SHORT; + break; + case UINT32: + case INT32: + factory = N5Codecs.INT; + break; + case UINT64: + case INT64: + factory = N5Codecs.LONG; + break; + case FLOAT32: + factory = N5Codecs.FLOAT; + break; + case FLOAT64: + factory = N5Codecs.DOUBLE; + break; + case STRING: + factory = N5Codecs.STRING; + break; + case OBJECT: + factory = N5Codecs.OBJECT; + break; + default: + throw new IllegalArgumentException("Unsupported data type: " + dataType); + } + final DataBlockCodecFactory tFactory = (DataBlockCodecFactory)factory; + return tFactory.createDataBlockCodec(compression); + } + public interface DataBlockCodecFactory { DataBlockCodec createDataBlockCodec(Compression compression); From f948f578479501e0e8a45e91904427428a7dc864 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 5 Mar 2025 16:54:56 -0500 Subject: [PATCH 195/423] feat: Add StringDataCodec, ObjectDataCodec, StringDataBlockCodec, ObjectDataBlockCodec DataCodecs now creat the access object and return it during `deserialize` --- .../saalfeldlab/n5/codec/DataCodec.java | 79 +++++++++++++++++-- .../saalfeldlab/n5/codec/N5Codecs.java | 17 ++-- 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java index 9f306ecbd..666be5840 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -4,6 +4,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.IntBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.function.IntFunction; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -23,7 +26,7 @@ public abstract class DataCodec { public abstract ReadData serialize(T data) throws IOException; - public abstract void deserialize(ReadData readData, T data) throws IOException; + public abstract T deserialize(ReadData readData, int numElements) throws IOException; public int bytesPerElement() { return bytesPerElement; @@ -49,6 +52,8 @@ public T createData(final int numElements) { public static final DataCodec FLOAT_LITTLE_ENDIAN = new FloatDataCodec(ByteOrder.LITTLE_ENDIAN); public static final DataCodec DOUBLE_LITTLE_ENDIAN = new DoubleDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec STRING = new StringDataCodec(); + public static final DataCodec OBJECT = new ObjectDataCodec(); public static DataCodec SHORT(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? SHORT_BIG_ENDIAN : SHORT_LITTLE_ENDIAN; } @@ -92,8 +97,10 @@ public ReadData serialize(final byte[] data) { } @Override - public void deserialize(final ReadData readData, final byte[] data) throws IOException { + public byte[] deserialize(final ReadData readData, int numElements) throws IOException { + final byte[] data = createData(numElements); new DataInputStream(readData.inputStream()).readFully(data); + return data; } } @@ -114,8 +121,11 @@ public ReadData serialize(final short[] data) { } @Override - public void deserialize(final ReadData readData, final short[] data) throws IOException { + public short[] deserialize(final ReadData readData, int numElements) throws IOException { + + final short[] data = createData(numElements); readData.toByteBuffer().order(order).asShortBuffer().get(data); + return data; } } @@ -136,8 +146,13 @@ public ReadData serialize(final int[] data) { } @Override - public void deserialize(final ReadData readData, final int[] data) throws IOException { - readData.toByteBuffer().order(order).asIntBuffer().get(data); + public int[] deserialize(final ReadData readData, int numElements) throws IOException { + + final int[] data = createData(numElements); + final ByteBuffer byteBuffer = readData.toByteBuffer(); + final IntBuffer intBuffer = byteBuffer.order(order).asIntBuffer(); + intBuffer.get(data); + return data; } } @@ -158,8 +173,10 @@ public ReadData serialize(final long[] data) { } @Override - public void deserialize(final ReadData readData, final long[] data) throws IOException { + public long[] deserialize(final ReadData readData, int numElements) throws IOException { + final long[] data = createData(numElements); readData.toByteBuffer().order(order).asLongBuffer().get(data); + return data; } } @@ -180,8 +197,10 @@ public ReadData serialize(final float[] data) { } @Override - public void deserialize(final ReadData readData, final float[] data) throws IOException { + public float[] deserialize(final ReadData readData, int numElements) throws IOException { + final float[] data = createData(numElements); readData.toByteBuffer().order(order).asFloatBuffer().get(data); + return data; } } @@ -202,8 +221,52 @@ public ReadData serialize(final double[] data) { } @Override - public void deserialize(final ReadData readData, final double[] data) throws IOException { + public double[] deserialize(final ReadData readData, int numElements) throws IOException { + + final double[] data = createData(numElements); readData.toByteBuffer().order(order).asDoubleBuffer().get(data); + return data; + } + } + + private static final class StringDataCodec extends DataCodec { + + StringDataCodec() { + super( -1, String[]::new); + } + + private static final Charset ENCODING = StandardCharsets.UTF_8; + private static final String NULLCHAR = "\0"; + + + @Override public ReadData serialize(String[] data) { + final String flattenedArray = String.join(NULLCHAR, data) + NULLCHAR; + return ReadData.from(flattenedArray.getBytes(ENCODING)); + } + + @Override public String[] deserialize(ReadData readData, int numElements) throws IOException { + final byte[] serializedData = readData.allBytes(); + final String rawChars = new String(serializedData, ENCODING); + return rawChars.split(NULLCHAR); + } + } + + private static final class ObjectDataCodec extends DataCodec { + + + ObjectDataCodec() { + super(-1, byte[]::new); + } + + @Override public ReadData serialize(byte[] data) { + + return ReadData.from(data); + } + + @Override public byte[] deserialize(ReadData readData, int numElements) throws IOException { + final byte[] data = createData(numElements); + new DataInputStream(readData.inputStream()).readFully(data); + return data; } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index 4b244006d..9b6945348 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -11,6 +11,7 @@ import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataBlock.DataBlockFactory; +import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock; import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; import org.janelia.saalfeldlab.n5.IntArrayDataBlock; @@ -25,14 +26,14 @@ public class N5Codecs { - public static final DataBlockCodecFactory BYTE = c -> new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new, c); - public static final DataBlockCodecFactory SHORT = c -> new DefaultDataBlockCodec<>(DataCodec.SHORT_BIG_ENDIAN, ShortArrayDataBlock::new, c); - public static final DataBlockCodecFactory INT = c -> new DefaultDataBlockCodec<>(DataCodec.INT_BIG_ENDIAN, IntArrayDataBlock::new, c); - public static final DataBlockCodecFactory LONG = c -> new DefaultDataBlockCodec<>(DataCodec.LONG_BIG_ENDIAN, LongArrayDataBlock::new, c); - public static final DataBlockCodecFactory FLOAT = c -> new DefaultDataBlockCodec<>(DataCodec.FLOAT_BIG_ENDIAN, FloatArrayDataBlock::new, c); - public static final DataBlockCodecFactory DOUBLE = c -> new DefaultDataBlockCodec<>(DataCodec.DOUBLE_BIG_ENDIAN, DoubleArrayDataBlock::new, c); - public static final DataBlockCodecFactory STRING = StringDataBlockCodec::new; - public static final DataBlockCodecFactory OBJECT = ObjectDataBlockCodec::new; + public static final DataBlockCodecFactory BYTE =c -> new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new, c); + public static final DataBlockCodecFactory SHORT =c -> new DefaultDataBlockCodec<>(DataCodec.SHORT_BIG_ENDIAN, ShortArrayDataBlock::new, c); + public static final DataBlockCodecFactory INT =c -> new DefaultDataBlockCodec<>(DataCodec.INT_BIG_ENDIAN, IntArrayDataBlock::new, c); + public static final DataBlockCodecFactory LONG =c -> new DefaultDataBlockCodec<>(DataCodec.LONG_BIG_ENDIAN, LongArrayDataBlock::new, c); + public static final DataBlockCodecFactory FLOAT =c -> new DefaultDataBlockCodec<>(DataCodec.FLOAT_BIG_ENDIAN, FloatArrayDataBlock::new, c); + public static final DataBlockCodecFactory DOUBLE =c -> new DefaultDataBlockCodec<>(DataCodec.DOUBLE_BIG_ENDIAN, DoubleArrayDataBlock::new, c); + public static final DataBlockCodecFactory STRING =c -> new StringDataBlockCodec(DataCodec.STRING, StringDataBlock::new, c); + public static final DataBlockCodecFactory OBJECT =c -> new ObjectDataBlockCodec(DataCodec.OBJECT, ByteArrayDataBlock::new, c); private N5Codecs() {} From ae85a55844d8090ce0e17b26fc0f65d22169ef4b Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 5 Mar 2025 16:55:21 -0500 Subject: [PATCH 196/423] refactor: add AbstractDataBlock to extract shared logic between Default/String/Object blocks --- .../saalfeldlab/n5/codec/N5Codecs.java | 166 +++++++++++------- 1 file changed, 99 insertions(+), 67 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index 9b6945348..cb9729e5a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -5,8 +5,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; + import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; @@ -83,112 +82,146 @@ public interface DataBlockCodecFactory { DataBlockCodec createDataBlockCodec(Compression compression); } - /** - * DataBlockCodec for all N5 data types, except STRING and OBJECT - */ - static class DefaultDataBlockCodec implements DataBlockCodec { + public abstract static class AbstractDataBlockCodec implements DataBlockCodec { - private final DataCodec dataCodec; + private static final int VAR_OBJ_BYTES_PER_ELEMENT = 1; + private final DataCodec dataCodec; private final DataBlockFactory dataBlockFactory; - private final Compression compression; - DefaultDataBlockCodec( + public AbstractDataBlockCodec( final DataCodec dataCodec, final DataBlockFactory dataBlockFactory, - final Compression compression) { + final Compression compression + ) { this.dataCodec = dataCodec; this.dataBlockFactory = dataBlockFactory; this.compression = compression; } - @Override - public ReadData encode(final DataBlock dataBlock) throws IOException { + public DataBlockFactory getDataBlockFactory() { + + return dataBlockFactory; + } + + public DataCodec getDataCodec() { + + return dataCodec; + } + + public Compression getCompression() { + + return compression; + } + + + abstract BlockHeader encodeBlockHeader(final DataBlock dataBlock, ReadData blockData) throws IOException; + + @Override public ReadData encode(DataBlock dataBlock) throws IOException { return ReadData.from(out -> { - new BlockHeader(dataBlock.getSize(), dataBlock.getNumElements()).writeTo(out); - compression.encode(dataCodec.serialize(dataBlock.getData())).writeTo(out); + final ReadData dataReadData = getDataCodec().serialize(dataBlock.getData()); + final ReadData encodedData = getCompression().encode(dataReadData); + final BlockHeader header = encodeBlockHeader(dataBlock, dataReadData); + + header.writeTo(out); + encodedData.writeTo(out); }); } + abstract BlockHeader decodeBlockHeader(final InputStream in) throws IOException; @Override public DataBlock decode(final ReadData readData, final long[] gridPosition) throws IOException { try(final InputStream in = readData.inputStream()) { - final BlockHeader header = BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); - final T data = dataCodec.createData(header.numElements()); - final int numBytes = header.numElements() * dataCodec.bytesPerElement(); - final ReadData decompressed = compression.decode(ReadData.from(in), numBytes); - dataCodec.deserialize(decompressed, data); - return dataBlockFactory.createDataBlock(header.blockSize(), gridPosition, data); + final BlockHeader header = decodeBlockHeader(in); + + final int bytesPerElement + = getDataCodec().bytesPerElement() == -1 + ? VAR_OBJ_BYTES_PER_ELEMENT + : getDataCodec().bytesPerElement(); + + final int numElements = header.numElements(); + final ReadData blockData = ReadData.from(in, numElements * bytesPerElement); + final ReadData decodeData = getCompression().decode(blockData); + final T data = getDataCodec().deserialize(decodeData, numElements); + return getDataBlockFactory().createDataBlock(header.blockSize(), gridPosition, data); } } } /** - * DataBlockCodec for N5 data type STRING + * DataBlockCodec for all N5 data types, except STRING and OBJECT */ - static class StringDataBlockCodec implements DataBlockCodec { + static class DefaultDataBlockCodec extends AbstractDataBlockCodec { - private static final Charset ENCODING = StandardCharsets.UTF_8; - private static final String NULLCHAR = "\0"; + DefaultDataBlockCodec( + final DataCodec dataCodec, + final DataBlockFactory dataBlockFactory, + final Compression compression) { - private final Compression compression; + super(dataCodec, dataBlockFactory, compression); + } + @Override + protected BlockHeader encodeBlockHeader(final DataBlock dataBlock, ReadData blockData) { - StringDataBlockCodec(final Compression compression) { - this.compression = compression; + return new BlockHeader(dataBlock.getSize(), dataBlock.getNumElements()); } @Override - public ReadData encode(final DataBlock dataBlock) throws IOException { - return ReadData.from(out -> { - final String flattenedArray = String.join(NULLCHAR, dataBlock.getData()) + NULLCHAR; - final byte[] serializedData = flattenedArray.getBytes(ENCODING); - new BlockHeader(dataBlock.getSize(), serializedData.length).writeTo(out); - compression.encode(ReadData.from(serializedData)).writeTo(out); - }); + protected BlockHeader decodeBlockHeader(final InputStream in) throws IOException { + + return BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); + } + } + + /** + * DataBlockCodec for N5 data type STRING + */ + static class StringDataBlockCodec extends AbstractDataBlockCodec { + + public StringDataBlockCodec( + final DataCodec dataCodec, + final DataBlockFactory dataBlockFactory, + final Compression compression) { + + super(dataCodec, dataBlockFactory, compression); } @Override - public DataBlock decode(final ReadData readData, final long[] gridPosition) throws IOException { - try(final InputStream in = readData.inputStream()) { - final BlockHeader header = BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); - final ReadData decompressed = compression.decode(ReadData.from(in), header.numElements()); - final byte[] serializedData = decompressed.allBytes(); - final String rawChars = new String(serializedData, ENCODING); - final String[] actualData = rawChars.split(NULLCHAR); - return new StringDataBlock(header.blockSize(), gridPosition, actualData); - } + protected BlockHeader encodeBlockHeader(final DataBlock dataBlock, ReadData blockData) throws IOException { + + return new BlockHeader(dataBlock.getSize(), (int)blockData.length()); + } + + @Override + protected BlockHeader decodeBlockHeader(final InputStream in) throws IOException { + + return BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); } } /** * DataBlockCodec for N5 data type OBJECT */ - static class ObjectDataBlockCodec implements DataBlockCodec { + static class ObjectDataBlockCodec extends AbstractDataBlockCodec { - private final Compression compression; + public ObjectDataBlockCodec( + final DataCodec dataCodec, + final DataBlockFactory dataBlockFactory, + final Compression compression) { - ObjectDataBlockCodec(final Compression compression) { - this.compression = compression; + super(dataCodec, dataBlockFactory, compression); } - @Override - public ReadData encode(final DataBlock dataBlock) throws IOException { - return ReadData.from(out -> { - new BlockHeader(null, dataBlock.getNumElements()).writeTo(out); - compression.encode(ReadData.from(dataBlock.getData())).writeTo(out); - }); + @Override protected BlockHeader encodeBlockHeader(DataBlock dataBlock, ReadData blockData) { + + return new BlockHeader(null, dataBlock.getNumElements()); } @Override - public DataBlock decode(final ReadData readData, final long[] gridPosition) throws IOException { - try(final InputStream in = readData.inputStream()) { - final BlockHeader header = BlockHeader.readFrom(in, MODE_OBJECT); - final byte[] data = new byte[header.numElements()]; - final ReadData decompressed = compression.decode(ReadData.from(in), data.length); - new DataInputStream(decompressed.inputStream()).readFully(data); - return new ByteArrayDataBlock(null, gridPosition, data); - } + protected BlockHeader decodeBlockHeader(final InputStream in) throws IOException { + + return BlockHeader.readFrom(in, MODE_OBJECT); } } @@ -266,9 +299,6 @@ void writeTo(final OutputStream out) throws IOException { dos.flush(); } - static BlockHeader readFrom(final InputStream in) throws IOException { - return readFrom(in, null); - } static BlockHeader readFrom(final InputStream in, short... allowedModes) throws IOException { final DataInputStream dis = new DataInputStream(in); @@ -289,16 +319,18 @@ static BlockHeader readFrom(final InputStream in, short... allowedModes) throws numElements = dis.readInt(); break; default: - throw new IOException("unexpected mode: " + mode); + throw new IOException("Unexpected mode: " + mode); } boolean modeIsOk = allowedModes == null || allowedModes.length == 0; for (int i = 0; !modeIsOk && i < allowedModes.length; ++i) { - if (mode == allowedModes[i]) + if (mode == allowedModes[i]) { modeIsOk = true; + break; + } } if (!modeIsOk) { - throw new IOException("unexpected mode: " + mode); + throw new IOException("Unexpected mode: " + mode); } return new BlockHeader(mode, blockSize, numElements); From 7b713cd7f93a76a9536799ae44852433b5655d0c Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 5 Mar 2025 16:59:15 -0500 Subject: [PATCH 197/423] refactor: DatasetAttributes responsible for DataBlockCodec creation N5BlockCodec uses dataType (and potentially other DatasetAttributes to wrap the desired DataBlockCodec --- .../saalfeldlab/n5/DatasetAttributes.java | 18 +++------ .../saalfeldlab/n5/codec/N5BlockCodec.java | 37 +++++++++++++++++++ 2 files changed, 42 insertions(+), 13 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index dc9b51fed..86f8d5620 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -28,7 +28,9 @@ import java.io.Serializable; import java.util.Arrays; import java.util.HashMap; + import org.janelia.saalfeldlab.n5.codec.DataBlockCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec.N5BlockCodecFactory; /** * Mandatory dataset attributes: @@ -58,8 +60,8 @@ public class DatasetAttributes implements Serializable { private final long[] dimensions; private final int[] blockSize; private final DataType dataType; - private final Compression compression; private final DataBlockCodec dataBlockCodec; + private final Compression compression; public DatasetAttributes( final long[] dimensions, @@ -67,21 +69,11 @@ public DatasetAttributes( final DataType dataType, final Compression compression) { - this(dimensions, blockSize, dataType, compression, dataType.createDataBlockCodec(compression)); - } - - protected DatasetAttributes( - final long[] dimensions, - final int[] blockSize, - final DataType dataType, - final Compression compression, - final DataBlockCodec dataBlockCodec) { - this.dimensions = dimensions; this.blockSize = blockSize; this.dataType = dataType; this.compression = compression; - this.dataBlockCodec = dataBlockCodec; + this.dataBlockCodec = N5BlockCodecFactory.fromDatasetAttributes(this); } public long[] getDimensions() { @@ -118,7 +110,7 @@ public DataType getDataType() { * @return the {@code DataBlockCodec} for this dataset */ public DataBlockCodec getDataBlockCodec() { - return (DataBlockCodec) dataBlockCodec; + return (DataBlockCodec)dataBlockCodec; } public HashMap asMap() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java new file mode 100644 index 000000000..de02400c4 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java @@ -0,0 +1,37 @@ +package org.janelia.saalfeldlab.n5.codec; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +import java.io.IOException; + +public class N5BlockCodec implements DataBlockCodec { + + private DataBlockCodec dataBlockCodec; + + public N5BlockCodec(DataBlockCodec dataBlockCodec) { + this.dataBlockCodec = dataBlockCodec; + } + + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { + + return dataBlockCodec.decode(readData, gridPosition); + } + + @Override public ReadData encode(DataBlock dataBlock) throws IOException { + + return dataBlockCodec.encode(dataBlock); + } + + public static class N5BlockCodecFactory { + + private N5BlockCodecFactory() {} + + public static N5BlockCodec fromDatasetAttributes(final DatasetAttributes attributes) { + + final DataBlockCodec dataBlockCodec = N5Codecs.createDataBlockCodec(attributes.getDataType(), attributes.getCompression()); + return new N5BlockCodec<>(dataBlockCodec); + } + } +} From 0db21be4fc6b4f325cf43d9cd93cc1b4ce36daec Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 8 Apr 2025 14:52:52 -0400 Subject: [PATCH 198/423] revert: add back createDataBlock logic in DataType --- .../org/janelia/saalfeldlab/n5/DataType.java | 131 ++++++++++++++++-- 1 file changed, 118 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 8d260583c..b10a15005 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -41,24 +41,92 @@ */ public enum DataType { - UINT8("uint8"), - UINT16("uint16"), - UINT32("uint32"), - UINT64("uint64"), - INT8("int8"), - INT16("int16"), - INT32("int32"), - INT64("int64"), - FLOAT32("float32"), - FLOAT64("float64"), - STRING("string"), - OBJECT("object"); + UINT8( + "uint8", + (blockSize, gridPosition, numElements) -> { + ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements]); + + + return new ByteArrayDataBlock( + blockSize, + gridPosition, + new byte[numElements]); + }), + UINT16( + "uint16", + (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( + blockSize, + gridPosition, + new short[numElements])), + UINT32( + "uint32", + (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( + blockSize, + gridPosition, + new int[numElements])), + UINT64( + "uint64", + (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( + blockSize, + gridPosition, + new long[numElements])), + INT8( + "int8", + (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( + blockSize, + gridPosition, + new byte[numElements])), + INT16( + "int16", + (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( + blockSize, + gridPosition, + new short[numElements])), + INT32( + "int32", + (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( + blockSize, + gridPosition, + new int[numElements])), + INT64( + "int64", + (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( + blockSize, + gridPosition, + new long[numElements])), + FLOAT32( + "float32", + (blockSize, gridPosition, numElements) -> new FloatArrayDataBlock( + blockSize, + gridPosition, + new float[numElements])), + FLOAT64( + "float64", + (blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock( + blockSize, + gridPosition, + new double[numElements])), + STRING( + "string", + (blockSize, gridPosition, numElements) -> new StringDataBlock( + blockSize, + gridPosition, + new String[numElements])), + OBJECT( + "object", + (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( + blockSize, + gridPosition, + new byte[numElements])); private final String label; - DataType(final String label) { + private final DataBlockFactory dataBlockFactory; + + DataType(final String label, final DataBlockFactory dataBlockFactory) { this.label = label; + this.dataBlockFactory = dataBlockFactory; } @Override @@ -75,6 +143,43 @@ public static DataType fromString(final String string) { return null; } + /** + * Factory for {@link DataBlock DataBlocks}. + * + * @param blockSize + * the block size + * @param gridPosition + * the grid position + * @param numElements + * the number of elements (not necessarily one element per block + * element) + * @return the data block + */ + public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements) { + + return dataBlockFactory.createDataBlock(blockSize, gridPosition, numElements); + } + + /** + * Factory for {@link DataBlock DataBlocks} with one data element for each + * block element (e.g. pixel image). + * + * @param blockSize + * the block size + * @param gridPosition + * the grid position + * @return the data block + */ + public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition) { + + return dataBlockFactory.createDataBlock(blockSize, gridPosition, DataBlock.getNumElements(blockSize)); + } + + private interface DataBlockFactory { + + DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements); + } + static public class JsonAdapter implements JsonDeserializer, JsonSerializer { @Override From 0f8b2c59e2dcd60f65593f380b61813e0f9ae151 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 8 Apr 2025 14:53:32 -0400 Subject: [PATCH 199/423] doc: retain javadoc from before refactor --- .../java/org/janelia/saalfeldlab/n5/DataBlock.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index cb8f5b685..7b6249f4b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -95,6 +95,20 @@ static int getNumElements(final int[] size) { */ interface DataBlockFactory { + + /** + * Factory for {@link DataBlock DataBlocks}. + * + * @param blockSize + * the block size + * @param gridPosition + * the grid position + * @param data + * the number of elements (not necessarily one element per block + * element) + * @return the data block + */ DataBlock createDataBlock(int[] blockSize, long[] gridPosition, T data); + } } \ No newline at end of file From 70ec77710cc1ec9453efa5c98611ae9a64060948 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 8 Apr 2025 14:55:32 -0400 Subject: [PATCH 200/423] revert: keep protected constructor with DataBlockCodec parameter refactor: inline createDataBlockCodec from constructor params --- .../janelia/saalfeldlab/n5/DatasetAttributes.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 86f8d5620..06cd568ef 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -30,7 +30,7 @@ import java.util.HashMap; import org.janelia.saalfeldlab.n5.codec.DataBlockCodec; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodec.N5BlockCodecFactory; +import org.janelia.saalfeldlab.n5.codec.N5Codecs; /** * Mandatory dataset attributes: @@ -69,11 +69,21 @@ public DatasetAttributes( final DataType dataType, final Compression compression) { + this(dimensions, blockSize, dataType, compression, N5Codecs.createDataBlockCodec(dataType, compression)); + } + + protected DatasetAttributes( + final long[] dimensions, + final int[] blockSize, + final DataType dataType, + final Compression compression, + final DataBlockCodec dataBlockCodec) { + this.dimensions = dimensions; this.blockSize = blockSize; this.dataType = dataType; this.compression = compression; - this.dataBlockCodec = N5BlockCodecFactory.fromDatasetAttributes(this); + this.dataBlockCodec = dataBlockCodec; } public long[] getDimensions() { From ab8b83b3e41512dce7a41bee0340dbf7b4169da4 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 8 Apr 2025 14:56:34 -0400 Subject: [PATCH 201/423] refactor: remove currently unused N5BlockCodec. Something like this may be needed when multiple codecs are supported --- .../saalfeldlab/n5/codec/N5BlockCodec.java | 37 ------------------- 1 file changed, 37 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java deleted file mode 100644 index de02400c4..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec; - -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - -import java.io.IOException; - -public class N5BlockCodec implements DataBlockCodec { - - private DataBlockCodec dataBlockCodec; - - public N5BlockCodec(DataBlockCodec dataBlockCodec) { - this.dataBlockCodec = dataBlockCodec; - } - - @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { - - return dataBlockCodec.decode(readData, gridPosition); - } - - @Override public ReadData encode(DataBlock dataBlock) throws IOException { - - return dataBlockCodec.encode(dataBlock); - } - - public static class N5BlockCodecFactory { - - private N5BlockCodecFactory() {} - - public static N5BlockCodec fromDatasetAttributes(final DatasetAttributes attributes) { - - final DataBlockCodec dataBlockCodec = N5Codecs.createDataBlockCodec(attributes.getDataType(), attributes.getCompression()); - return new N5BlockCodec<>(dataBlockCodec); - } - } -} From 2e056286a27668056a485d058727eb1bacbf68b8 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 8 Apr 2025 14:57:30 -0400 Subject: [PATCH 202/423] refactor: dont expose N5Codecs internals --- .../janelia/saalfeldlab/n5/codec/N5Codecs.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index cb9729e5a..4441d1b75 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -79,10 +79,19 @@ public static DataBlockCodec createDataBlockCodec( public interface DataBlockCodecFactory { + /** + * Get the default {@link DataBlockCodec}, with the specified {@code + * compression}, for {@link DataBlock DataBlocks} of this {@code DataType}. + * The default codec is used for de/serializing blocks to N5 format. + * + * @param compression + * + * @return the default {@code DataBlockCodec} + */ DataBlockCodec createDataBlockCodec(Compression compression); } - public abstract static class AbstractDataBlockCodec implements DataBlockCodec { + private abstract static class AbstractDataBlockCodec implements DataBlockCodec { private static final int VAR_OBJ_BYTES_PER_ELEMENT = 1; @@ -100,17 +109,17 @@ public AbstractDataBlockCodec( this.compression = compression; } - public DataBlockFactory getDataBlockFactory() { + private DataBlockFactory getDataBlockFactory() { return dataBlockFactory; } - public DataCodec getDataCodec() { + private DataCodec getDataCodec() { return dataCodec; } - public Compression getCompression() { + private Compression getCompression() { return compression; } From d4ebfefe2be06d87e92efdd4739f5d04c14498b2 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 8 Apr 2025 14:57:50 -0400 Subject: [PATCH 203/423] refactor: rename encodeBlockHeader -> createBlockHeader --- .../org/janelia/saalfeldlab/n5/codec/N5Codecs.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index 4441d1b75..611cbeb9b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -125,13 +125,13 @@ private Compression getCompression() { } - abstract BlockHeader encodeBlockHeader(final DataBlock dataBlock, ReadData blockData) throws IOException; + abstract BlockHeader createBlockHeader(final DataBlock dataBlock, ReadData blockData) throws IOException; @Override public ReadData encode(DataBlock dataBlock) throws IOException { return ReadData.from(out -> { final ReadData dataReadData = getDataCodec().serialize(dataBlock.getData()); final ReadData encodedData = getCompression().encode(dataReadData); - final BlockHeader header = encodeBlockHeader(dataBlock, dataReadData); + final BlockHeader header = createBlockHeader(dataBlock, dataReadData); header.writeTo(out); encodedData.writeTo(out); @@ -171,7 +171,7 @@ static class DefaultDataBlockCodec extends AbstractDataBlockCodec { super(dataCodec, dataBlockFactory, compression); } @Override - protected BlockHeader encodeBlockHeader(final DataBlock dataBlock, ReadData blockData) { + protected BlockHeader createBlockHeader(final DataBlock dataBlock, ReadData blockData) { return new BlockHeader(dataBlock.getSize(), dataBlock.getNumElements()); } @@ -197,7 +197,7 @@ public StringDataBlockCodec( } @Override - protected BlockHeader encodeBlockHeader(final DataBlock dataBlock, ReadData blockData) throws IOException { + protected BlockHeader createBlockHeader(final DataBlock dataBlock, ReadData blockData) throws IOException { return new BlockHeader(dataBlock.getSize(), (int)blockData.length()); } @@ -222,7 +222,7 @@ public ObjectDataBlockCodec( super(dataCodec, dataBlockFactory, compression); } - @Override protected BlockHeader encodeBlockHeader(DataBlock dataBlock, ReadData blockData) { + @Override protected BlockHeader createBlockHeader(DataBlock dataBlock, ReadData blockData) { return new BlockHeader(null, dataBlock.getNumElements()); } From 0dcade45bf3a53466aca68c8a1850c1881171e9d Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 8 Apr 2025 14:58:13 -0400 Subject: [PATCH 204/423] feat: add ZarrStringDataCodec support --- .../saalfeldlab/n5/codec/DataCodec.java | 90 +++++++++++++++---- 1 file changed, 71 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java index 666be5840..55b8f7bd1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -7,6 +7,7 @@ import java.nio.IntBuffer; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.function.IntFunction; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -39,20 +40,22 @@ public T createData(final int numElements) { // ------------------- instances -------------------- // - public static final DataCodec BYTE = new ByteDataCodec(); - public static final DataCodec SHORT_BIG_ENDIAN = new ShortDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec INT_BIG_ENDIAN = new IntDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec LONG_BIG_ENDIAN = new LongDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec FLOAT_BIG_ENDIAN = new FloatDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec DOUBLE_BIG_ENDIAN = new DoubleDataCodec(ByteOrder.BIG_ENDIAN); - - public static final DataCodec SHORT_LITTLE_ENDIAN = new ShortDataCodec(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec INT_LITTLE_ENDIAN = new IntDataCodec(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec LONG_LITTLE_ENDIAN = new LongDataCodec(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec FLOAT_LITTLE_ENDIAN = new FloatDataCodec(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec DOUBLE_LITTLE_ENDIAN = new DoubleDataCodec(ByteOrder.LITTLE_ENDIAN); - - public static final DataCodec STRING = new StringDataCodec(); + public static final DataCodec BYTE = new ByteDataCodec(); + public static final DataCodec SHORT_BIG_ENDIAN = new ShortDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec INT_BIG_ENDIAN = new IntDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec LONG_BIG_ENDIAN = new LongDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec FLOAT_BIG_ENDIAN = new FloatDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec DOUBLE_BIG_ENDIAN = new DoubleDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec ZARR_STRING_BIG_STRING = new ZarrStringDataCodec(ByteOrder.BIG_ENDIAN); + + public static final DataCodec SHORT_LITTLE_ENDIAN = new ShortDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec INT_LITTLE_ENDIAN = new IntDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec LONG_LITTLE_ENDIAN = new LongDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec FLOAT_LITTLE_ENDIAN = new FloatDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec DOUBLE_LITTLE_ENDIAN = new DoubleDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec ZARR_STRING_LITTLE_STRING = new ZarrStringDataCodec(ByteOrder.LITTLE_ENDIAN); + + public static final DataCodec STRING = new N5StringDataCodec(); public static final DataCodec OBJECT = new ObjectDataCodec(); public static DataCodec SHORT(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? SHORT_BIG_ENDIAN : SHORT_LITTLE_ENDIAN; @@ -229,15 +232,14 @@ public double[] deserialize(final ReadData readData, int numElements) throws IOE } } - private static final class StringDataCodec extends DataCodec { - - StringDataCodec() { - super( -1, String[]::new); - } + private static final class N5StringDataCodec extends DataCodec { private static final Charset ENCODING = StandardCharsets.UTF_8; private static final String NULLCHAR = "\0"; + N5StringDataCodec() { + super( -1, String[]::new); + } @Override public ReadData serialize(String[] data) { final String flattenedArray = String.join(NULLCHAR, data) + NULLCHAR; @@ -251,6 +253,56 @@ private static final class StringDataCodec extends DataCodec { } } + private static final class ZarrStringDataCodec extends DataCodec { + + private final ByteOrder order; + private static final Charset ENCODING = StandardCharsets.UTF_8; + + ZarrStringDataCodec(ByteOrder order) { + super( -1, String[]::new); + this.order = order; + } + + @Override public ReadData serialize(String[] data) { + + final int N = data.length; + final byte[][] encodedStrings = Arrays.stream(data).map(str -> str.getBytes(ENCODING)).toArray(byte[][]::new); + final int[] lengths = Arrays.stream(encodedStrings).mapToInt(a -> a.length).toArray(); + final int totalLength = Arrays.stream(lengths).sum(); + final ByteBuffer buf = ByteBuffer.wrap(new byte[totalLength + 4 * N + 4]); + buf.order(order); + buf.putInt(N); + for (int i = 0; i < N; ++i) { + buf.putInt(lengths[i]); + buf.put(encodedStrings[i]); + } + return ReadData.from(buf.array()); + } + + @Override public String[] deserialize(ReadData readData, int numElements) throws IOException { + + final ByteBuffer serialized = readData.toByteBuffer(); + serialized.order(order); + + // sanity check to avoid out of memory errors + if (serialized.limit() < 4) + throw new RuntimeException("Corrupt buffer, data seems truncated."); + + final int n = serialized.getInt(); + if (serialized.limit() < n) + throw new RuntimeException("Corrupt buffer, data seems truncated."); + + final String[] actualData = new String[n]; + for (int i = 0; i < n; ++i) { + final int length = serialized.getInt(); + final byte[] encodedString = new byte[length]; + serialized.get(encodedString); + actualData[i] = new String(encodedString, ENCODING); + } + return actualData; + } + } + private static final class ObjectDataCodec extends DataCodec { From b4ca260c4df3c405620ca994cdee161122a9bc99 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 12 May 2025 13:56:23 -0400 Subject: [PATCH 205/423] tmp --- .../saalfeldlab/n5/AbstractDataBlock.java | 21 +- .../saalfeldlab/n5/ByteArrayDataBlock.java | 43 +-- .../saalfeldlab/n5/Bzip2Compression.java | 40 +- .../janelia/saalfeldlab/n5/Compression.java | 25 -- .../saalfeldlab/n5/CompressionAdapter.java | 2 +- .../org/janelia/saalfeldlab/n5/DataBlock.java | 72 ++-- .../org/janelia/saalfeldlab/n5/DataType.java | 144 ++----- .../saalfeldlab/n5/DatasetAttributes.java | 33 +- .../saalfeldlab/n5/DefaultBlockReader.java | 75 +--- .../saalfeldlab/n5/DefaultBlockWriter.java | 71 +--- .../saalfeldlab/n5/DoubleArrayDataBlock.java | 52 +-- .../saalfeldlab/n5/FloatArrayDataBlock.java | 52 +-- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 30 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 15 +- .../saalfeldlab/n5/GzipCompression.java | 41 +- .../saalfeldlab/n5/IntArrayDataBlock.java | 54 +-- .../saalfeldlab/n5/LongArrayDataBlock.java | 53 +-- .../saalfeldlab/n5/Lz4Compression.java | 60 ++- .../org/janelia/saalfeldlab/n5/N5Reader.java | 4 +- .../org/janelia/saalfeldlab/n5/N5Writer.java | 26 +- .../saalfeldlab/n5/NameConfigAdapter.java | 4 +- .../saalfeldlab/n5/RawCompression.java | 44 +-- .../saalfeldlab/n5/ShortArrayDataBlock.java | 52 +-- .../saalfeldlab/n5/StringDataBlock.java | 20 +- .../janelia/saalfeldlab/n5/XzCompression.java | 39 +- .../saalfeldlab/n5/codec/AsTypeCodec.java | 29 +- .../janelia/saalfeldlab/n5/codec/Codec.java | 145 ++----- .../n5/codec/ConcatenatedBytesCodec.java | 41 ++ .../saalfeldlab/n5/codec/DataBlockCodec.java | 18 + .../saalfeldlab/n5/codec/DataCodec.java | 274 +++++++++++++ .../FixedLengthConvertedInputStream.java | 11 +- .../n5/codec/FixedScaleOffsetCodec.java | 25 +- .../saalfeldlab/n5/codec/IdentityCodec.java | 16 +- .../saalfeldlab/n5/codec/N5BlockCodec.java | 188 +-------- .../saalfeldlab/n5/codec/N5Codecs.java | 365 ++++++++++++++++++ .../saalfeldlab/n5/codec/RawBlockCodecs.java | 115 ++++++ .../saalfeldlab/n5/codec/RawBytes.java | 77 ++-- .../n5/codec/checksum/ChecksumCodec.java | 26 +- .../readdata/AbstractInputStreamReadData.java | 32 ++ .../readdata/ByteArraySplittableReadData.java | 47 +++ .../n5/readdata/InputStreamReadData.java | 32 ++ .../n5/readdata/KeyValueAccessReadData.java | 43 +++ .../saalfeldlab/n5/readdata/LazyReadData.java | 81 ++++ .../saalfeldlab/n5/readdata/ReadData.java | 269 +++++++++++++ .../janelia/saalfeldlab/n5/shard/Shard.java | 7 +- .../saalfeldlab/n5/shard/ShardingCodec.java | 37 +- .../saalfeldlab/n5/shard/VirtualShard.java | 31 +- .../saalfeldlab/n5/AbstractN5Test.java | 58 +-- .../janelia/saalfeldlab/n5/N5Benchmark.java | 4 +- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 4 +- .../saalfeldlab/n5/codec/AsTypeTests.java | 14 +- .../saalfeldlab/n5/codec/BytesTests.java | 6 +- .../saalfeldlab/n5/demo/BlockIterators.java | 5 +- .../saalfeldlab/n5/shard/ShardIndexTest.java | 9 +- .../saalfeldlab/n5/shard/ShardTest.java | 80 ++-- 55 files changed, 1827 insertions(+), 1334 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index 59208fcfe..e101fe81b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -29,6 +29,7 @@ import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.function.ToIntFunction; /** * Abstract base class for {@link DataBlock} implementations. @@ -43,12 +44,19 @@ public abstract class AbstractDataBlock implements DataBlock { protected final int[] size; protected final long[] gridPosition; protected final T data; + private final ToIntFunction numElements; public AbstractDataBlock(final int[] size, final long[] gridPosition, final T data) { + this(size, gridPosition, data, (t) -> -1); + } + + public AbstractDataBlock(final int[] size, final long[] gridPosition, final T data, final ToIntFunction numElements) { + this.size = size; this.gridPosition = gridPosition; this.data = data; + this.numElements = numElements; } @Override @@ -70,18 +78,9 @@ public T getData() { } @Override - public void readData(final DataInput input) throws IOException { - - final ByteBuffer buffer = toByteBuffer(); - input.readFully(buffer.array()); - readData(buffer); - } - - @Override - public void writeData(final DataOutput output) throws IOException { + public int getNumElements() { - final ByteBuffer buffer = toByteBuffer(); - output.write(buffer.array()); + return numElements.applyAsInt(data); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index 4610811d7..73ec42647 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,39 +25,10 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; -import java.io.IOException; -import java.nio.ByteBuffer; - public class ByteArrayDataBlock extends AbstractDataBlock { public ByteArrayDataBlock(final int[] size, final long[] gridPosition, final byte[] data) { - super(size, gridPosition, data); - } - - @Override - public ByteBuffer toByteBuffer() { - - return ByteBuffer.wrap(getData()); - } - - @Override - public void readData(final ByteBuffer buffer) { - - if (buffer.array() != getData()) - buffer.get(getData()); - } - - @Override - public void readData(final DataInput inputStream) throws IOException { - - inputStream.readFully(data); - } - - @Override - public int getNumElements() { - - return data.length; + super(size, gridPosition, data, a -> a.length); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 49a333f3e..a30a03c48 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -32,6 +32,7 @@ import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("bzip2") @@ -53,49 +54,32 @@ public Bzip2Compression() { this(BZip2CompressorOutputStream.MAX_BLOCKSIZE); } - @Override - public InputStream decode(final InputStream in) throws IOException { - + private InputStream getInputStream(final InputStream in) throws IOException { return new BZip2CompressorInputStream(in); } - @Override - public InputStream getInputStream(final InputStream in) throws IOException { - - return decode(in); - } - - @Override - public OutputStream encode(final OutputStream out) throws IOException { + private OutputStream getOutputStream(final OutputStream out) throws IOException { return new BZip2CompressorOutputStream(out, blockSize); } @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { - - return encode(out); - } - - @Override - public Bzip2Compression getReader() { + public boolean equals(final Object other) { - return this; + if (other == null || other.getClass() != Bzip2Compression.class) + return false; + else + return blockSize == ((Bzip2Compression)other).blockSize; } @Override - public Bzip2Compression getWriter() { + public ReadData decode(final ReadData readData) throws IOException { - return this; + return ReadData.from(new BZip2CompressorInputStream(readData.inputStream())); } @Override - public boolean equals(final Object other) { - - if (other == null || other.getClass() != Bzip2Compression.class) - return false; - else - return blockSize == ((Bzip2Compression)other).blockSize; + public ReadData encode(final ReadData readData) { + return readData.encode(this::getOutputStream); } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index 2e8b9cdfd..7f64c8ed5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -81,29 +81,4 @@ default String getType() { return compressionType.value(); } - - BlockReader getReader(); - - BlockWriter getWriter(); - - /** - * Decode an {@link InputStream}. - * - * @param in - * input stream - * @return the decoded input stream - */ - @Override - InputStream decode(InputStream in) throws IOException; - - /** - * Encode an {@link OutputStream}. - * - * @param out - * the output stream - * @return the encoded output stream - */ - @Override - OutputStream encode(OutputStream out) throws IOException; - } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java index 7f5a82e07..d0d56c4d0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java @@ -96,7 +96,7 @@ public static synchronized void update(final boolean override) { newInstance.compressionConstructors.put(type, constructor); newInstance.compressionParameters.put(type, parameters); - } catch (final ClassNotFoundException | NoSuchMethodException | ClassCastException + } catch (final NoClassDefFoundError | ClassNotFoundException | NoSuchMethodException | ClassCastException | UnsatisfiedLinkError e) { System.err.println("Compression '" + item.className() + "' could not be registered"); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 5ccdbbaf9..855d088d5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,9 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; import java.nio.ByteBuffer; /** @@ -43,63 +40,31 @@ public interface DataBlock { /** * Returns the size of this data block. - * + *

    * The size of a data block is expected to be smaller than or equal to the * spacing of the block grid. The dimensionality of size is expected to be * equal to the dimensionality of the dataset. Consistency is not enforced. * * @return size of the data block */ - public int[] getSize(); + int[] getSize(); /** * Returns the position of this data block on the block grid. - * + *

    * The dimensionality of the grid position is expected to be equal to the * dimensionality of the dataset. Consistency is not enforced. * * @return position on the block grid */ - public long[] getGridPosition(); + long[] getGridPosition(); /** * Returns the data object held by this data block. * * @return data object */ - public T getData(); - - /** - * Creates a {@link ByteBuffer} that contains the data object of this data - * block. - * - * The {@link ByteBuffer} may or may not map directly to the data - * object of this data block. I.e. modifying the {@link ByteBuffer} after - * calling this method may or may not change the data of this data block. - * modifying the data object of this data block after calling this method - * may or may not change the content of the {@link ByteBuffer}. - * - * @return {@link ByteBuffer} containing data - */ - public ByteBuffer toByteBuffer(); - - /** - * Reads the data object of this data block from a {@link ByteBuffer}. - * - * The {@link ByteBuffer} may or may not map directly to the data - * object of this data block. I.e. modifying the {@link ByteBuffer} after - * calling this method may or may not change the data of this data block. - * modifying the data object of this data block after calling this method - * may or may not change the content of the {@link ByteBuffer}. - * - * @param buffer - * the byte buffer - */ - public void readData(final ByteBuffer buffer); - - public void readData(final DataInput inputStream) throws IOException; - - public void writeData(final DataOutput output) throws IOException; + T getData(); /** * Returns the number of elements in this {@link DataBlock}. This number is @@ -108,7 +73,7 @@ public interface DataBlock { * * @return the number of elements */ - public int getNumElements(); + int getNumElements(); /** * Returns the number of elements in a box of given size. @@ -117,11 +82,22 @@ public interface DataBlock { * the size * @return the number of elements */ - public static int getNumElements(final int[] size) { + static int getNumElements(final int[] size) { int n = size[0]; for (int i = 1; i < size.length; ++i) n *= size[i]; return n; } + + /** + * Factory for creating {@code DataBlock}. + * + * @param + * type of the data contained in the DataBlock + */ + interface DataBlockFactory { + + DataBlock createDataBlock(int[] blockSize, long[] gridPosition, T data); + } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index a6d60cb81..ace9095d5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,8 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.lang.reflect.Type; - import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; @@ -35,6 +33,10 @@ import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import javax.annotation.Nullable; +import java.lang.reflect.Type; +import java.nio.ByteOrder; + /** * Enumerates available data types. * @@ -42,87 +44,24 @@ */ public enum DataType { - UINT8( - "uint8", - (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( - blockSize, - gridPosition, - new byte[numElements])), - UINT16( - "uint16", - (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( - blockSize, - gridPosition, - new short[numElements])), - UINT32( - "uint32", - (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( - blockSize, - gridPosition, - new int[numElements])), - UINT64( - "uint64", - (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( - blockSize, - gridPosition, - new long[numElements])), - INT8( - "int8", - (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( - blockSize, - gridPosition, - new byte[numElements])), - INT16( - "int16", - (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( - blockSize, - gridPosition, - new short[numElements])), - INT32( - "int32", - (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( - blockSize, - gridPosition, - new int[numElements])), - INT64( - "int64", - (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( - blockSize, - gridPosition, - new long[numElements])), - FLOAT32( - "float32", - (blockSize, gridPosition, numElements) -> new FloatArrayDataBlock( - blockSize, - gridPosition, - new float[numElements])), - FLOAT64( - "float64", - (blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock( - blockSize, - gridPosition, - new double[numElements])), - STRING( - "string", - (blockSize, gridPosition, numElements) -> new StringDataBlock( - blockSize, - gridPosition, - new byte[numElements])), - OBJECT( - "object", - (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( - blockSize, - gridPosition, - new byte[numElements])); + UINT8("uint8"), + UINT16("uint16"), + UINT32("uint32"), + UINT64("uint64"), + INT8("int8"), + INT16("int16"), + INT32("int32"), + INT64("int64"), + FLOAT32("float32"), + FLOAT64("float64"), + STRING("string"), + OBJECT("object"); private final String label; - private final DataBlockFactory dataBlockFactory; - - DataType(final String label, final DataBlockFactory dataBlockFactory) { + DataType(final String label) { this.label = label; - this.dataBlockFactory = dataBlockFactory; } @Override @@ -139,43 +78,6 @@ public static DataType fromString(final String string) { return null; } - /** - * Factory for {@link DataBlock DataBlocks}. - * - * @param blockSize - * the block size - * @param gridPosition - * the grid position - * @param numElements - * the number of elements (not necessarily one element per block - * element) - * @return the data block - */ - public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements) { - - return dataBlockFactory.createDataBlock(blockSize, gridPosition, numElements); - } - - /** - * Factory for {@link DataBlock DataBlocks} with one data element for each - * block element (e.g. pixel image). - * - * @param blockSize - * the block size - * @param gridPosition - * the grid position - * @return the data block - */ - public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition) { - - return dataBlockFactory.createDataBlock(blockSize, gridPosition, DataBlock.getNumElements(blockSize)); - } - - private interface DataBlockFactory { - - DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements); - } - static public class JsonAdapter implements JsonDeserializer, JsonSerializer { @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index d2cfa4a51..8494a4203 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -57,7 +57,7 @@ public class DatasetAttributes implements BlockParameters, ShardParameters, Seri private final long[] dimensions; private final int[] blockSize; private final DataType dataType; - private final ArrayCodec arrayCodec; + private final ArrayCodec arrayCodec; private final BytesCodec[] byteCodecs; @Nullable private final int[] shardSize; @@ -81,29 +81,30 @@ public DatasetAttributes( this.shardSize = shardSize; this.blockSize = blockSize; this.dataType = dataType; - if (codecs == null || codecs.length == 0) { + final Codec[] filteredCodecs = Arrays.stream(codecs).filter(it -> !(it instanceof RawCompression)).toArray(Codec[]::new); + if (filteredCodecs.length == 0) { byteCodecs = new BytesCodec[]{}; arrayCodec = new N5BlockCodec(); - } else if (codecs.length == 1 && codecs[0] instanceof Compression) { - final BytesCodec compression = (BytesCodec)codecs[0]; + } else if (filteredCodecs.length == 1 && filteredCodecs[0] instanceof Compression) { + final BytesCodec compression = (BytesCodec)filteredCodecs[0]; byteCodecs = compression instanceof RawCompression ? new BytesCodec[]{} : new BytesCodec[]{compression}; arrayCodec = new N5BlockCodec(); } else { - if (!(codecs[0] instanceof ArrayCodec)) - throw new N5Exception("Expected first element of codecs to be ArrayCodec, but was: " + codecs[0].getClass()); + if (!(filteredCodecs[0] instanceof ArrayCodec)) + throw new N5Exception("Expected first element of filteredCodecs to be ArrayCodec, but was: " + filteredCodecs[0].getClass()); - if (Arrays.stream(codecs).filter(c -> c instanceof ArrayCodec).count() > 1) + if (Arrays.stream(filteredCodecs).filter(c -> c instanceof ArrayCodec).count() > 1) throw new N5Exception("Multiple ArrayCodecs found. Only one is allowed."); - arrayCodec = (ArrayCodec)codecs[0]; - byteCodecs = Stream.of(codecs) + arrayCodec = (ArrayCodec)filteredCodecs[0]; + byteCodecs = Stream.of(filteredCodecs) .skip(1) - .filter(c -> !(c instanceof RawCompression)) .filter(c -> c instanceof BytesCodec) .toArray(BytesCodec[]::new); - } - + } + //TODO Caleb: factory style for setDatasetAttributes + arrayCodec.setDatasetAttributes(this); } /** @@ -190,9 +191,9 @@ public DataType getDataType() { return dataType; } - public ArrayCodec getArrayCodec() { + public ArrayCodec getArrayCodec() { - return arrayCodec; + return (ArrayCodec) arrayCodec; } public BytesCodec[] getCodecs() { @@ -276,11 +277,11 @@ public static class DatasetAttributesAdapter implements JsonSerializer n5BlockCodec = new N5BlockCodec<>(); codecs = new Codec[]{compression, n5BlockCodec}; } else if (obj.has(compressionTypeKey)) { final Compression compression = getCompressionVersion0(obj.get(compressionTypeKey).getAsString()); - final N5BlockCodec n5BlockCodec = dataType == DataType.UINT8 || dataType == DataType.INT8 ? new N5BlockCodec(null) : new N5BlockCodec(); + final N5BlockCodec n5BlockCodec = new N5BlockCodec<>(); codecs = new Codec[]{compression, n5BlockCodec}; } else { return null; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 0a49fc19b..b26ce5b89 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,75 +25,34 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInputStream; +import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + import java.io.IOException; import java.io.InputStream; -import java.nio.ByteBuffer; - -import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; -import org.janelia.saalfeldlab.n5.codec.Codec.DataBlockInputStream; /** - * Default implementation of {@link BlockReader}. - * * @author Stephan Saalfeld * @author Igor Pisarev */ -public interface DefaultBlockReader extends BlockReader { - - public InputStream getInputStream(final InputStream in) throws IOException; - - @Override - public default > void read( - final B dataBlock, - final InputStream in) throws IOException { - - // do not try with this input stream because subsequent block reads may happen if the stream points to a shard - final InputStream inflater = getInputStream(in); - readFromStream(dataBlock, inflater); - } +public interface DefaultBlockReader { /** * Reads a {@link DataBlock} from an {@link InputStream}. * - * @param in - * the input stream - * @param datasetAttributes - * the dataset attributes - * @param gridPosition - * the grid position + * @param in the input stream + * @param datasetAttributes the dataset attributes + * @param gridPosition the grid position * @return the block - * @throws IOException - * the exception + * @throws IOException the exception */ - public static DataBlock readBlock( + static DataBlock readBlock( final InputStream in, final DatasetAttributes datasetAttributes, final long[] gridPosition) throws IOException { - final BytesCodec[] codecs = datasetAttributes.getCodecs(); - final ArrayCodec arrayCodec = datasetAttributes.getArrayCodec(); - final DataBlockInputStream dataBlockStream = arrayCodec.decode(datasetAttributes, gridPosition, in); - - InputStream stream = dataBlockStream; - for (final BytesCodec codec : codecs) { - stream = codec.decode(stream); - } - - final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); - dataBlock.readData(dataBlockStream.getDataInput(stream)); - stream.close(); - - return dataBlock; - } - - public static > void readFromStream(final B dataBlock, final InputStream in) throws IOException { - - final ByteBuffer buffer = dataBlock.toByteBuffer(); - final DataInputStream dis = new DataInputStream(in); - dis.readFully(buffer.array()); - dataBlock.readData(buffer); + final ArrayCodec codec = datasetAttributes.getArrayCodec(); + return codec.decode(ReadData.from(in), gridPosition); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index 5f27c930a..c8c243bb8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,69 +25,32 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; - import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; -import org.janelia.saalfeldlab.n5.codec.Codec.DataBlockOutputStream; -import static org.janelia.saalfeldlab.n5.codec.Codec.encode; +import java.io.IOException; +import java.io.OutputStream; /** - * Default implementation of {@link BlockWriter}. - * * @author Stephan Saalfeld * @author Igor Pisarev */ -public interface DefaultBlockWriter extends BlockWriter { - - public OutputStream getOutputStream(final OutputStream out) throws IOException; - - @Override - public default void write( - final DataBlock dataBlock, - final OutputStream out) throws IOException { - - final ByteBuffer buffer = dataBlock.toByteBuffer(); - try (final OutputStream deflater = getOutputStream(out)) { - deflater.write(buffer.array()); - deflater.flush(); - } - } +public interface DefaultBlockWriter { /** * Writes a {@link DataBlock} into an {@link OutputStream}. * - * @param the type of data - * @param out - * the output stream - * @param datasetAttributes - * the dataset attributes - * @param dataBlock - * the data block the block data type - * @throws IOException - * the exception + * @param the type of data + * @param out the output stream + * @param datasetAttributes the dataset attributes + * @param dataBlock the data block the block data type + * @throws IOException the exception */ - public static void writeBlock( + static void writeBlock( final OutputStream out, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - final BytesCodec[] codecs = datasetAttributes.getCodecs(); - final ArrayCodec arrayCodec = datasetAttributes.getArrayCodec(); - final DataBlockOutputStream dataBlockOutput = arrayCodec.encode(datasetAttributes, dataBlock, out); - - OutputStream stream = encode(dataBlockOutput, codecs); - - dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); - stream.close(); - } - - public static void writeFromStream(final DataBlock dataBlock, final OutputStream out) throws IOException { - - final ByteBuffer buffer = dataBlock.toByteBuffer(); - out.write(buffer.array()); + final ArrayCodec arrayCodec = datasetAttributes.getArrayCodec(); + arrayCodec.encode(dataBlock).writeTo(out); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 0240e6fae..3e7632758 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,49 +25,11 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.nio.ByteBuffer; - public class DoubleArrayDataBlock extends AbstractDataBlock { public DoubleArrayDataBlock(final int[] size, final long[] gridPosition, final double[] data) { - super(size, gridPosition, data); - } - - @Override - public ByteBuffer toByteBuffer() { - - final ByteBuffer buffer = ByteBuffer.allocate(data.length * 8); - buffer.asDoubleBuffer().put(data); - return buffer; + super(size, gridPosition, data, a -> a.length); } - @Override - public void readData(final ByteBuffer buffer) { - - buffer.asDoubleBuffer().get(data); - } - - @Override - public void readData(final DataInput inputStream) throws IOException { - - for (int i = 0; i < data.length; i++) - data[i] = inputStream.readDouble(); - } - - @Override - public void writeData(final DataOutput output) throws IOException { - - for (int i = 0; i < data.length; i++) - output.writeDouble(data[i]); - } - - @Override - public int getNumElements() { - - return data.length; - } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index a2bc2c696..d924582f4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,49 +25,11 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.nio.ByteBuffer; - public class FloatArrayDataBlock extends AbstractDataBlock { public FloatArrayDataBlock(final int[] size, final long[] gridPosition, final float[] data) { - super(size, gridPosition, data); - } - - @Override - public ByteBuffer toByteBuffer() { - - final ByteBuffer buffer = ByteBuffer.allocate(data.length * 4); - buffer.asFloatBuffer().put(data); - return buffer; + super(size, gridPosition, data, a -> a.length); } - @Override - public void readData(final ByteBuffer buffer) { - - buffer.asFloatBuffer().get(data); - } - - @Override - public void readData(final DataInput inputStream) throws IOException { - - for (int i = 0; i < data.length; i++) - data[i] = inputStream.readFloat(); - } - - @Override - public void writeData(final DataOutput output) throws IOException { - - for (int i = 0; i < data.length; i++) - output.writeFloat(data[i]); - } - - @Override - public int getNumElements() { - - return data.length; - } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 2f9658f75..775ea1449 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -25,22 +25,22 @@ */ package org.janelia.saalfeldlab.n5; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.VirtualShard; +import org.janelia.saalfeldlab.n5.util.Position; + import java.io.IOException; +import java.io.InputStream; import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.shard.Shard; -import org.janelia.saalfeldlab.n5.shard.ShardParameters; -import org.janelia.saalfeldlab.n5.shard.VirtualShard; -import org.janelia.saalfeldlab.n5.util.Position; - -import com.google.gson.Gson; -import com.google.gson.JsonElement; - /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON * attributes parsed with {@link Gson}. @@ -125,14 +125,16 @@ default DataBlock readBlock( final SplitKeyValueAccessData splitData; try { splitData = new SplitKeyValueAccessData(getKeyValueAccess(), keyPath); + try (final InputStream inputStream = splitData.newInputStream()) { + final ReadData decodeData = ReadData.from(inputStream); + return datasetAttributes.getArrayCodec().decode(decodeData, gridPosition); + } + } catch (N5Exception.N5NoSuchKeyException e) { + return null; } catch (IOException e) { throw new N5IOException(e); } - return datasetAttributes.getArrayCodec().readBlock( - splitData, - datasetAttributes, - gridPosition - ); + } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 927881e35..3fb13feb9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -36,6 +36,7 @@ import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shard.InMemoryShard; import org.janelia.saalfeldlab.n5.shard.Shard; import org.janelia.saalfeldlab.n5.shard.ShardParameters; @@ -254,16 +255,18 @@ default void writeBlock( final long[] keyPos = datasetAttributes.getArrayCodec().getPositionForBlock(datasetAttributes, dataBlock); final String keyPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), keyPos); - final SplitKeyValueAccessData splitData; + final SplitableData splitData; try { - splitData = new SplitKeyValueAccessData(getKeyValueAccess(), keyPath); + //TODO Caleb: What behavior do we want when writing a new block at an existing one? + // If the encoded size is smaller, should it truncate the remaining, or ignore it? + // currently we truncate for the default writeBlock method (shards dont' truncate) + splitData = new SplitKeyValueAccessData(getKeyValueAccess(), keyPath).split(0, Long.MAX_VALUE); + try (final OutputStream out = splitData.newOutputStream()) { + datasetAttributes.getArrayCodec().encode(dataBlock).writeTo(out); + } } catch (IOException e) { throw new N5IOException(e); } - datasetAttributes.getArrayCodec().writeBlock( - splitData, - datasetAttributes, - dataBlock); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index b03a4d930..eb649914d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -37,6 +37,7 @@ import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; import org.apache.commons.compress.compressors.gzip.GzipParameters; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("gzip") @@ -75,8 +76,7 @@ public GzipCompression(final int level, final boolean useZlib) { this.useZlib = useZlib; } - @Override - public InputStream decode(InputStream in) throws IOException { + private InputStream getInputStream(final InputStream in) throws IOException { if (useZlib) { return new InflaterInputStream(in); @@ -85,14 +85,7 @@ public InputStream decode(InputStream in) throws IOException { } } - @Override - public InputStream getInputStream(final InputStream in) throws IOException { - - return decode(in); - } - - @Override - public OutputStream encode(OutputStream out) throws IOException { + private OutputStream getOutputStream(final OutputStream out) throws IOException { if (useZlib) { return new DeflaterOutputStream(out, new Deflater(level)); @@ -102,24 +95,6 @@ public OutputStream encode(OutputStream out) throws IOException { } } - @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { - - return encode(out); - } - - @Override - public GzipCompression getReader() { - - return this; - } - - @Override - public GzipCompression getWriter() { - - return this; - } - private void readObject(final ObjectInputStream in) throws Exception { in.defaultReadObject(); @@ -137,4 +112,14 @@ public boolean equals(final Object other) { } } + @Override + public ReadData decode(final ReadData readData) throws IOException { + return ReadData.from(getInputStream(readData.inputStream())); + } + + @Override + public ReadData encode(final ReadData readData) { + return readData.encode(this::getOutputStream); + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 4d3383328..5e1f901e0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,49 +25,9 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.nio.ByteBuffer; - - public class IntArrayDataBlock extends AbstractDataBlock { public IntArrayDataBlock(final int[] size, final long[] gridPosition, final int[] data) { - super(size, gridPosition, data); - } - - @Override - public ByteBuffer toByteBuffer() { - - final ByteBuffer buffer = ByteBuffer.allocate(data.length * 4); - buffer.asIntBuffer().put(data); - return buffer; - } - - @Override - public void readData(final ByteBuffer buffer) { - - buffer.asIntBuffer().get(data); - } - - @Override - public void readData(final DataInput input) throws IOException { - - for (int i = 0; i < data.length; i++) - data[i] = input.readInt(); - } - - @Override - public void writeData(final DataOutput output) throws IOException { - - for (int i = 0; i < data.length; i++) - output.writeInt(data[i]); - } - - @Override - public int getNumElements() { - - return data.length; + super(size, gridPosition, data, a -> a.length); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index be435c4f9..ef2379e5d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,49 +25,10 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.nio.ByteBuffer; - public class LongArrayDataBlock extends AbstractDataBlock { public LongArrayDataBlock(final int[] size, final long[] gridPosition, final long[] data) { - super(size, gridPosition, data); - } - - @Override - public ByteBuffer toByteBuffer() { - - final ByteBuffer buffer = ByteBuffer.allocate(data.length * 8); - buffer.asLongBuffer().put(data); - return buffer; - } - - @Override - public void readData(final ByteBuffer buffer) { - - buffer.asLongBuffer().get(data); - } - - @Override - public void readData(final DataInput inputStream) throws IOException { - - for (int i = 0; i < data.length; i++) - data[i] = inputStream.readLong(); - } - - @Override - public void writeData(final DataOutput output) throws IOException { - - for (int i = 0; i < data.length; i++) - output.writeLong(data[i]); - } - - @Override - public int getNumElements() { - - return data.length; + super(size, gridPosition, data, a -> a.length); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index 654ca4b56..a87b98aff 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,16 +25,16 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -import org.janelia.saalfeldlab.n5.Compression.CompressionType; - import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; +import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + @CompressionType("lz4") @NameConfig.Name("lz4") public class Lz4Compression implements DefaultBlockReader, DefaultBlockWriter, Compression { @@ -54,48 +54,34 @@ public Lz4Compression() { this(1 << 16); } - @Override - public InputStream decode(final InputStream in) throws IOException { + private InputStream getInputStream(final InputStream in) throws IOException { return new LZ4BlockInputStream(in); } - @Override - public InputStream getInputStream(final InputStream in) throws IOException { - - return decode(in); - } - - @Override - public OutputStream encode(final OutputStream out) throws IOException { + private OutputStream getOutputStream(final OutputStream out) throws IOException { return new LZ4BlockOutputStream(out, blockSize); } @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { - - return encode(out); - } - - @Override - public Lz4Compression getReader() { + public boolean equals(final Object other) { - return this; + if (other == null || other.getClass() != Lz4Compression.class) + return false; + else + return blockSize == ((Lz4Compression)other).blockSize; } @Override - public Lz4Compression getWriter() { + public ReadData decode(final ReadData readData) throws IOException { - return this; + return ReadData.from(getInputStream(readData.inputStream())); } @Override - public boolean equals(final Object other) { + public ReadData encode(final ReadData readData) { - if (other == null || other.getClass() != Lz4Compression.class) - return false; - else - return blockSize == ((Lz4Compression)other).blockSize; + return readData.encode(this::getOutputStream); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 20a04b330..21f1088a7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -359,11 +359,11 @@ default T readSerializedBlock( final DatasetAttributes attributes, final long... gridPosition) throws N5Exception, ClassNotFoundException { - final DataBlock block = readBlock(dataset, attributes, gridPosition); + final DataBlock block = readBlock(dataset, attributes, gridPosition); if (block == null) return null; - final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.toByteBuffer().array()); + final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.getData()); try (ObjectInputStream in = new ObjectInputStream(byteArrayInputStream)) { return (T)in.readObject(); } catch (final IOException | UncheckedIOException e) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 016062043..50067f160 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,22 +25,19 @@ */ package org.janelia.saalfeldlab.n5; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; +import org.janelia.saalfeldlab.n5.shard.Shard; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; import java.io.Serializable; import java.io.UncheckedIOException; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -import org.checkerframework.checker.units.qual.A; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; -import org.janelia.saalfeldlab.n5.shard.Shard; -import org.janelia.saalfeldlab.n5.shard.ShardParameters; - /** * A simple structured container API for hierarchies of chunked * n-dimensional datasets and attributes. @@ -213,7 +210,6 @@ default void createDataset( setDatasetAttributes(normalPath, datasetAttributes); } - /** * Creates a dataset. This does not create any data but the path and * mandatory attributes only. @@ -257,7 +253,7 @@ default void createDataset( final DataType dataType, final Compression compression) throws N5Exception { - createDataset(datasetPath, dimensions, blockSize, dataType, new N5BlockCodec(), compression); + createDataset(datasetPath, dimensions, blockSize, dataType, new N5BlockCodec<>(), compression); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java index 30e45d80d..804a2cde8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java @@ -113,7 +113,9 @@ public static synchronized void update(final NameConfigAdapter adapter) { adapter.parameterNames.put(type, parameterNames); } catch (final ClassNotFoundException | NoSuchMethodException | ClassCastException | UnsatisfiedLinkError e) { + System.err.println("T '" + item.className() + "' could not be registered"); + e.printStackTrace(System.err); } } } @@ -161,7 +163,7 @@ public JsonElement serialize( if (!configuration.isEmpty()) json.add("configuration", configuration); } catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) { - e.printStackTrace(System.err); + new RuntimeException("Could not serialize " + clazz.getName(), e).printStackTrace(System.err); return null; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index ebd58b38b..53344e714 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -30,6 +30,7 @@ import java.io.OutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("raw") @@ -38,48 +39,17 @@ public class RawCompression implements DefaultBlockReader, DefaultBlockWriter, C private static final long serialVersionUID = 7526445806847086477L; @Override - public InputStream decode(final InputStream in) throws IOException { - - return in; - } - - @Override - public InputStream getInputStream(final InputStream in) throws IOException { - - return decode(in); - } - - - @Override - public OutputStream encode(final OutputStream out) throws IOException { - - return out; - } - - @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { - - return encode(out); - } - - @Override - public RawCompression getReader() { + public boolean equals(final Object other) { - return this; + return other != null && other.getClass() == RawCompression.class; } - @Override - public RawCompression getWriter() { - - return this; + public ReadData encode(final ReadData readData) { + return readData; } @Override - public boolean equals(final Object other) { - - if (other == null || other.getClass() != RawCompression.class) - return false; - else - return true; + public ReadData decode(final ReadData readData) { + return readData; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index c7d141f39..d05277935 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,49 +25,11 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.nio.ByteBuffer; - public class ShortArrayDataBlock extends AbstractDataBlock { public ShortArrayDataBlock(final int[] size, final long[] gridPosition, final short[] data) { - super(size, gridPosition, data); - } - - @Override - public ByteBuffer toByteBuffer() { - - final ByteBuffer buffer = ByteBuffer.allocate(data.length * 2); - buffer.asShortBuffer().put(data); - return buffer; + super(size, gridPosition, data, a -> a.length); } - @Override - public void readData(final ByteBuffer buffer) { - - buffer.asShortBuffer().get(data); - } - - @Override - public void readData(final DataInput dataInput) throws IOException { - - for (int i = 0; i < data.length; i++) - data[i] = dataInput.readShort(); - } - - @Override - public void writeData(final DataOutput output) throws IOException { - - for (int i = 0; i < data.length; i++) - output.writeShort(data[i]); - } - - @Override - public int getNumElements() { - - return data.length; - } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index a64f3b9ca..f3639a258 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -37,7 +37,7 @@ public class StringDataBlock extends AbstractDataBlock { protected String[] actualData = null; public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { - super(size, gridPosition, new String[0]); + super(size, gridPosition, new String[0], a -> a.length); actualData = data; } @@ -46,24 +46,6 @@ public StringDataBlock(final int[] size, final long[] gridPosition, final byte[] serializedData = data; } - @Override - public ByteBuffer toByteBuffer() { - if (serializedData == null) - serializedData = serialize(actualData); - return ByteBuffer.wrap(serializedData); - } - - @Override - public void readData(final ByteBuffer buffer) { - - if (buffer.hasArray()) { - if (buffer.array() != serializedData) - buffer.get(serializedData); - actualData = deserialize(buffer.array()); - } else - actualData = ENCODING.decode(buffer).toString().split(NULLCHAR); - } - protected byte[] serialize(String[] strings) { final String flattenedArray = String.join(NULLCHAR, strings) + NULLCHAR; return flattenedArray.getBytes(ENCODING); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index d2c1ce3dc..dc5542f95 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -32,6 +32,7 @@ import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("xz") @@ -53,49 +54,33 @@ public XzCompression() { this(6); } - @Override - public InputStream decode(final InputStream in) throws IOException { + private InputStream getInputStream(final InputStream in) throws IOException { return new XZCompressorInputStream(in); } - @Override - public InputStream getInputStream(final InputStream in) throws IOException { - - return decode(in); - } - - @Override - public OutputStream encode(final OutputStream out) throws IOException { + private OutputStream getOutputStream(final OutputStream out) throws IOException { return new XZCompressorOutputStream(out, preset); } @Override - public OutputStream getOutputStream(final OutputStream out) throws IOException { - - return encode(out); - } - - @Override - public XzCompression getReader() { + public boolean equals(final Object other) { - return this; + if (other == null || other.getClass() != XzCompression.class) + return false; + else + return preset == ((XzCompression)other).preset; } @Override - public XzCompression getWriter() { + public ReadData decode(final ReadData readData) throws IOException { - return this; + return ReadData.from(getInputStream(readData.inputStream())); } @Override - public boolean equals(final Object other) { - - if (other == null || other.getClass() != XzCompression.class) - return false; - else - return preset == ((XzCompression)other).preset; + public ReadData encode(final ReadData readData) { + return readData.encode(this::getOutputStream); } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java index e8883c752..0cc2bdd19 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java @@ -7,6 +7,7 @@ import java.util.function.BiConsumer; import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @NameConfig.Name(AsTypeCodec.TYPE) @@ -55,28 +56,36 @@ public DataType getEncodedDataType() { return encodedType; } - @Override - public InputStream decode(InputStream in) throws IOException { + @Override public ReadData encode(ReadData readData) throws IOException { + numBytes = bytes(dataType); numEncodedBytes = bytes(encodedType); encoder = converter(dataType, encodedType); decoder = converter(encodedType, dataType); + return readData.encode(out -> { + numBytes = bytes(dataType); + numEncodedBytes = bytes(encodedType); - return new FixedLengthConvertedInputStream(numEncodedBytes, numBytes, decoder, in); + encoder = converter(dataType, encodedType); + decoder = converter(encodedType, dataType); + return new FixedLengthConvertedOutputStream(numBytes, numEncodedBytes, encoder, out); + }); } - @Override - public OutputStream encode(OutputStream out) throws IOException { + @Override public ReadData decode(ReadData readData) throws IOException { - numBytes = bytes(dataType); - numEncodedBytes = bytes(encodedType); + return ReadData.from(out -> { + numBytes = bytes(dataType); + numEncodedBytes = bytes(encodedType); - encoder = converter(dataType, encodedType); - decoder = converter(encodedType, dataType); + encoder = converter(dataType, encodedType); + decoder = converter(encodedType, dataType); - return new FixedLengthConvertedOutputStream(numBytes, numEncodedBytes, encoder, out); + final FixedLengthConvertedInputStream convertedIn = new FixedLengthConvertedInputStream(numEncodedBytes, numBytes, decoder, readData.inputStream()); + ReadData.from(convertedIn).writeTo(out); + }); } public static int bytes(DataType type) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index 6b64420ed..02dbbd2f0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -1,21 +1,15 @@ package org.janelia.saalfeldlab.n5.codec; -import org.apache.commons.io.input.ProxyInputStream; -import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.SplitableData; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import java.io.DataInput; -import java.io.DataOutput; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Serializable; -import java.io.UncheckedIOException; -import java.util.Arrays; + /** * Interface representing a filter can encode a {@link OutputStream}s when writing data, and decode * the {@link InputStream}s when reading data. @@ -26,44 +20,39 @@ @NameConfig.Prefix("codec") public interface Codec extends Serializable { - static OutputStream encode(OutputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { - - OutputStream stream = out; - for (final BytesCodec codec : bytesCodecs) - stream = codec.encode(stream); - - return stream; - } - - static InputStream decode(InputStream out, Codec.BytesCodec... bytesCodecs) throws IOException { - - InputStream stream = out; - for (final BytesCodec codec : bytesCodecs) - stream = codec.decode(stream); - - return stream; - } - interface BytesCodec extends Codec { + // -------------------------------------------------- + // + /** - * Decode an {@link InputStream}. + * Decode the given {@code readData}. + *

    + * The returned decoded {@code ReadData} reports {@link ReadData#length() + * length()}{@code == decodedLength}. Decoding may be lazy or eager, + * depending on the {@code BytesCodec} implementation. * - * @param in input stream - * @return the decoded input stream + * @param readData data to decode + * @return decoded ReadData + * @throws IOException if any I/O error occurs */ - InputStream decode(final InputStream in) throws IOException; + ReadData decode(ReadData readData) throws IOException; /** - * Encode an {@link OutputStream}. + * Encode the given {@code readData}. + *

    + * Encoding may be lazy or eager, depending on the {@code BytesCodec} + * implementation. * - * @param out the output stream - * @return the encoded output stream + * @param readData data to encode + * @return encoded ReadData + * @throws IOException if any I/O error occurs */ - OutputStream encode(final OutputStream out) throws IOException; + ReadData encode(ReadData readData) throws IOException; + } - interface ArrayCodec extends DeterministicSizeCodec { + interface ArrayCodec extends DeterministicSizeCodec, DataBlockCodec { default long[] getPositionForBlock(final DatasetAttributes attributes, final DataBlock datablock) { @@ -75,26 +64,7 @@ default long[] getPositionForBlock(final DatasetAttributes attributes, final lon return blockPosition; } - /** - * Decode an {@link InputStream}. - * - * @param in input stream - * @return the DataBlock corresponding to the input stream - */ - DataBlockInputStream decode( - final DatasetAttributes attributes, - final long[] gridPosition, - final InputStream in) throws IOException; - - /** - * Encode a {@link DataBlock}. - * - * @param datablock the datablock to encode - */ - DataBlockOutputStream encode( - final DatasetAttributes attributes, - final DataBlock datablock, - final OutputStream out) throws IOException; + void setDatasetAttributes(final DatasetAttributes attributes, final BytesCodec... codecs); @Override default long encodedSize(long size) { @@ -105,71 +75,6 @@ DataBlockOutputStream encode( return size; } - - default void writeBlock( - final SplitableData splitData, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) { - - final SplitableData truncateExisting = splitData.split(0, Long.MAX_VALUE); - try (final OutputStream out = truncateExisting.newOutputStream()) { - final DataBlockOutputStream dataBlockOutput = encode(datasetAttributes, dataBlock, out); - try (final OutputStream stream = Codec.encode(dataBlockOutput, datasetAttributes.getCodecs())) { - dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); - } - } catch (final IOException | UncheckedIOException e) { - final String msg = "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()); - throw new N5Exception.N5IOException(msg, e); - } - } - - default DataBlock readBlock( - final SplitableData splitData, - final DatasetAttributes datasetAttributes, - final long[] gridPosition) { - - try (final InputStream in = splitData.newInputStream()) { - final BytesCodec[] codecs = datasetAttributes.getCodecs(); - final ArrayCodec arrayCodec = datasetAttributes.getArrayCodec(); - final DataBlockInputStream dataBlockStream = arrayCodec.decode(datasetAttributes, gridPosition, in); - InputStream stream = Codec.decode(dataBlockStream, codecs); - - final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); - dataBlock.readData(dataBlockStream.getDataInput(stream)); - stream.close(); - - return dataBlock; - } catch (final N5Exception.N5NoSuchKeyException e) { - //TODO Caleb: Not sure these catch(s) belong here - return null; - } catch (final IOException | UncheckedIOException e) { - //TODO Caleb: Not sure these catch(s) belong here - final String msg = "Failed to read block " + Arrays.toString(gridPosition); - throw new N5Exception.N5IOException(msg, e); - } - } - } - - abstract class DataBlockInputStream extends ProxyInputStream { - - protected DataBlockInputStream(InputStream in) { - - super(in); - } - - public abstract DataBlock allocateDataBlock() throws IOException; - - public abstract DataInput getDataInput(final InputStream inputStream); - } - - abstract class DataBlockOutputStream extends ProxyOutputStream { - - protected DataBlockOutputStream(final OutputStream out) { - - super(out); - } - - public abstract DataOutput getDataOutput(final OutputStream outputStream); } String getType(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java new file mode 100644 index 000000000..1507db6ea --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java @@ -0,0 +1,41 @@ +package org.janelia.saalfeldlab.n5.codec; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +import java.io.IOException; + +class ConcatenatedBytesCodec implements Codec.BytesCodec { + + private final BytesCodec[] codecs; + + ConcatenatedBytesCodec(final BytesCodec... codecs) { + this.codecs = codecs; + } + + @Override public ReadData encode(ReadData readData) throws IOException { + + ReadData encodeData = readData; + if (codecs != null) { + for (Codec.BytesCodec codec : codecs) { + encodeData = codec.encode(encodeData); + } + } + return encodeData; + } + + @Override public ReadData decode(ReadData readData) throws IOException { + ReadData decodeData = readData; + if (codecs != null) { + for (int i = codecs.length - 1; i >= 0; i--) { + final BytesCodec codec = codecs[i]; + decodeData = codec.decode(decodeData); + } + } + return decodeData; + } + + @Override public String getType() { + + return "internal-concatenated-codecs"; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java new file mode 100644 index 000000000..87d31bbbb --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java @@ -0,0 +1,18 @@ +package org.janelia.saalfeldlab.n5.codec; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +import java.io.IOException; + +/** + * De/serialize {@link DataBlock} from/to {@link ReadData}. + * + * @param type of the data contained in the DataBlock + */ +public interface DataBlockCodec { + + ReadData encode(DataBlock dataBlock) throws IOException; + + DataBlock decode(ReadData readData, long[] gridPosition) throws IOException; +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java new file mode 100644 index 000000000..336dabf40 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -0,0 +1,274 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.IntBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.function.IntFunction; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +/** + * De/serialize the {@link DataBlock#getData() data} contained in a {@code + * DataBlock} from/to a sequence of bytes. + *

    + * Static fields {@code BYTE}, {@code SHORT_BIG_ENDIAN}, {@code + * SHORT_LITTLE_ENDIAN}, etc. contain {@code DataCodec}s for all primitive array + * types and big-endian / little-endian byte order. + * + * @param + * type of the data contained in the DataBlock + */ +public abstract class DataCodec { + + public abstract ReadData serialize(T data) throws IOException; + + public abstract T deserialize(ReadData readData, int numElements) throws IOException; + + public int bytesPerElement() { + return bytesPerElement; + } + + public T createData(final int numElements) { + return dataFactory.apply(numElements); + } + + // ------------------- instances -------------------- + // + + public static final DataCodec BYTE = new ByteDataCodec(); + public static final DataCodec SHORT_BIG_ENDIAN = new ShortDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec INT_BIG_ENDIAN = new IntDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec LONG_BIG_ENDIAN = new LongDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec FLOAT_BIG_ENDIAN = new FloatDataCodec(ByteOrder.BIG_ENDIAN); + public static final DataCodec DOUBLE_BIG_ENDIAN = new DoubleDataCodec(ByteOrder.BIG_ENDIAN); + + public static final DataCodec SHORT_LITTLE_ENDIAN = new ShortDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec INT_LITTLE_ENDIAN = new IntDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec LONG_LITTLE_ENDIAN = new LongDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec FLOAT_LITTLE_ENDIAN = new FloatDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final DataCodec DOUBLE_LITTLE_ENDIAN = new DoubleDataCodec(ByteOrder.LITTLE_ENDIAN); + + public static final DataCodec STRING = new StringDataCodec(); + public static final DataCodec OBJECT = new ObjectDataCodec(); + + public static DataCodec SHORT(ByteOrder order) { + return order == ByteOrder.BIG_ENDIAN ? SHORT_BIG_ENDIAN : SHORT_LITTLE_ENDIAN; + } + + public static DataCodec INT(ByteOrder order) { + return order == ByteOrder.BIG_ENDIAN ? INT_BIG_ENDIAN : INT_LITTLE_ENDIAN; + } + + public static DataCodec LONG(ByteOrder order) { + return order == ByteOrder.BIG_ENDIAN ? LONG_BIG_ENDIAN : LONG_LITTLE_ENDIAN; + } + + public static DataCodec FLOAT(ByteOrder order) { + return order == ByteOrder.BIG_ENDIAN ? FLOAT_BIG_ENDIAN : FLOAT_LITTLE_ENDIAN; + } + + public static DataCodec DOUBLE(ByteOrder order) { + return order == ByteOrder.BIG_ENDIAN ? DOUBLE_BIG_ENDIAN : DOUBLE_LITTLE_ENDIAN; + } + + // ---------------- implementations ----------------- + // + + private final int bytesPerElement; + private final IntFunction dataFactory; + + private DataCodec(int bytesPerElement, IntFunction dataFactory) { + this.bytesPerElement = bytesPerElement; + this.dataFactory = dataFactory; + } + + private static final class ByteDataCodec extends DataCodec { + + private ByteDataCodec() { + super(Byte.BYTES, byte[]::new); + } + + @Override + public ReadData serialize(final byte[] data) { + return ReadData.from(data); + } + + @Override + public byte[] deserialize(final ReadData readData, int numElements) throws IOException { + final byte[] data = createData((int)readData.length()); + new DataInputStream(readData.inputStream()).readFully(data); + return data; + } + } + + private static final class ShortDataCodec extends DataCodec { + + private final ByteOrder order; + + ShortDataCodec(ByteOrder order) { + super(Short.BYTES, short[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final short[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * data.length); + serialized.order(order).asShortBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public short[] deserialize(final ReadData readData, int numElements) throws IOException { + + final short[] data = createData(numElements); + readData.toByteBuffer().order(order).asShortBuffer().get(data); + return data; + } + } + + private static final class IntDataCodec extends DataCodec { + + private final ByteOrder order; + + IntDataCodec(ByteOrder order) { + super(Integer.BYTES, int[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final int[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Integer.BYTES * data.length); + serialized.order(order).asIntBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public int[] deserialize(final ReadData readData, int numElements) throws IOException { + + final int[] data = createData(numElements); + final ByteBuffer byteBuffer = readData.toByteBuffer(); + final IntBuffer intBuffer = byteBuffer.order(order).asIntBuffer(); + intBuffer.get(data); + return data; + } + } + + private static final class LongDataCodec extends DataCodec { + + private final ByteOrder order; + + LongDataCodec(ByteOrder order) { + super(Long.BYTES, long[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final long[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Long.BYTES * data.length); + serialized.order(order).asLongBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public long[] deserialize(final ReadData readData, int numElements) throws IOException { + final long[] data = createData(numElements); + readData.toByteBuffer().order(order).asLongBuffer().get(data); + return data; + } + } + + private static final class FloatDataCodec extends DataCodec { + + private final ByteOrder order; + + FloatDataCodec(ByteOrder order) { + super(Float.BYTES, float[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final float[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Float.BYTES * data.length); + serialized.order(order).asFloatBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public float[] deserialize(final ReadData readData, int numElements) throws IOException { + final float[] data = createData(numElements); + readData.toByteBuffer().order(order).asFloatBuffer().get(data); + return data; + } + } + + private static final class DoubleDataCodec extends DataCodec { + + private final ByteOrder order; + + DoubleDataCodec(ByteOrder order) { + super(Double.BYTES, double[]::new); + this.order = order; + } + + @Override + public ReadData serialize(final double[] data) { + final ByteBuffer serialized = ByteBuffer.allocate(Double.BYTES * data.length); + serialized.order(order).asDoubleBuffer().put(data); + return ReadData.from(serialized); + } + + @Override + public double[] deserialize(final ReadData readData, int numElements) throws IOException { + + final double[] data = createData(numElements); + readData.toByteBuffer().order(order).asDoubleBuffer().get(data); + return data; + } + } + + private static final class StringDataCodec extends DataCodec { + + StringDataCodec() { + super( -1, String[]::new); + } + + private static final Charset ENCODING = StandardCharsets.UTF_8; + private static final String NULLCHAR = "\0"; + + + @Override public ReadData serialize(String[] data) { + final String flattenedArray = String.join(NULLCHAR, data) + NULLCHAR; + return ReadData.from(flattenedArray.getBytes(ENCODING)); + } + + @Override public String[] deserialize(ReadData readData, int numElements) throws IOException { + final byte[] serializedData = readData.allBytes(); + final String rawChars = new String(serializedData, ENCODING); + return rawChars.split(NULLCHAR); + } + } + + private static final class ObjectDataCodec extends DataCodec { + + + ObjectDataCodec() { + super(-1, byte[]::new); + } + + @Override public ReadData serialize(byte[] data) { + + return ReadData.from(data); + } + + @Override public byte[] deserialize(ReadData readData, int numElements) throws IOException { + final byte[] data = createData((int)readData.length()); + new DataInputStream(readData.inputStream()).readFully(data); + return data; + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java index 78d6313af..0b786d041 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java @@ -55,8 +55,15 @@ public int read() throws IOException { rawBuffer.rewind(); decodedBuffer.rewind(); - for (int i = 0; i < numBytes; i++) - raw[i] = (byte)src.read(); + for (int i = 0; i < numBytes; i++) { + final int retval = src.read(); + if (retval == -1 && i == 0) + return retval; + else if (retval == -1) + throw new IOException("Unexpected end of stream"); + + raw[i] = (byte)retval; + } converter.accept(rawBuffer, decodedBuffer); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java index e6c831635..f5cde7ec0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java @@ -1,14 +1,14 @@ package org.janelia.saalfeldlab.n5.codec; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.function.BiConsumer; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.serialization.NameConfig; - @NameConfig.Name(FixedScaleOffsetCodec.TYPE) public class FixedScaleOffsetCodec extends AsTypeCodec { @@ -95,20 +95,21 @@ public String getType() { return TYPE; } - @Override - public InputStream decode(InputStream in) throws IOException { + @Override public ReadData decode(ReadData readData) throws IOException { numBytes = bytes(dataType); numEncodedBytes = bytes(encodedType); - return new FixedLengthConvertedInputStream(numEncodedBytes, numBytes, this.decoder, in); + return ReadData.from(new FixedLengthConvertedInputStream(numEncodedBytes, numBytes, this.decoder, readData.inputStream())); } - @Override - public OutputStream encode(OutputStream out) throws IOException { + @Override public ReadData encode(ReadData readData) throws IOException { - numBytes = bytes(dataType); - numEncodedBytes = bytes(encodedType); - return new FixedLengthConvertedOutputStream(numBytes, numEncodedBytes, this.encoder, out); + return readData.encode(out -> { + + numBytes = bytes(dataType); + numEncodedBytes = bytes(encodedType); + return new FixedLengthConvertedOutputStream(numBytes, numEncodedBytes, this.encoder, out); + }); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java index 93a384ddf..2050f593f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.io.OutputStream; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @NameConfig.Name(IdentityCodec.TYPE) @@ -14,21 +15,18 @@ public class IdentityCodec implements Codec.BytesCodec { public static final String TYPE = "id"; @Override - public InputStream decode(InputStream in) throws IOException { + public String getType() { - return in; + return TYPE; } - @Override - public OutputStream encode(OutputStream out) throws IOException { + @Override public ReadData decode(ReadData readData) throws IOException { - return out; + return readData; } - @Override - public String getType() { + @Override public ReadData encode(ReadData readData) throws IOException { - return TYPE; + return readData; } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java index 6b83f4666..f3342dac6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java @@ -1,72 +1,39 @@ package org.janelia.saalfeldlab.n5.codec; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteOrder; - import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import com.google.common.io.LittleEndianDataInputStream; -import com.google.common.io.LittleEndianDataOutputStream; - -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; +import java.io.IOException; @NameConfig.Name(value = N5BlockCodec.TYPE) -public class N5BlockCodec implements Codec.ArrayCodec { +public class N5BlockCodec implements Codec.ArrayCodec { private static final long serialVersionUID = 3523505403978222360L; public static final String TYPE = "n5bytes"; - public static final int MODE_DEFAULT = 0; - public static final int MODE_VARLENGTH = 1; - public static final int MODE_OBJECT = 2; - @Nullable - @NameConfig.Parameter(value = "endian", optional = true) - protected final ByteOrder byteOrder; + private DataBlockCodec dataBlockCodec; + private DatasetAttributes attributes; - public N5BlockCodec() { - this(ByteOrder.BIG_ENDIAN); + //TODO Caleb: Extract to factory that returns N5BlockCodec wrapper for datablockCodec + @Override public void setDatasetAttributes(final DatasetAttributes attributes, final Codec.BytesCodec... codecs) { + /*TODO: Consider an attributes.createDataBlockCodec() without parameters? */ + this.attributes = attributes; + final BytesCodec[] byteCodecs = codecs == null ? attributes.getCodecs() : codecs; + this.dataBlockCodec = N5Codecs.createDataBlockCodec(attributes.getDataType(), byteCodecs); } - public N5BlockCodec(@Nullable final ByteOrder byteOrder) { - - this.byteOrder = byteOrder; - } + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { - /** - * ByteOrder used to encode/decode this block of data.
    - * Will be `null` when {@link DatasetAttributes#getDataType()} refers to a single-byte type, - * - * @return the byte order for this codec - */ - @CheckForNull - public ByteOrder getByteOrder() { - return byteOrder; + return dataBlockCodec.decode(readData, gridPosition); } - @Override public DataBlockInputStream decode(final DatasetAttributes attributes, final long[] gridPosition, InputStream in) throws IOException { + @Override public ReadData encode(DataBlock dataBlock) throws IOException { - return new N5DataBlockInputStream(in, attributes, gridPosition, byteOrder); - } - - - @Override - public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, - final OutputStream out) - throws IOException { - - return new N5DataBlockOutputStream(out, attributes, dataBlock, byteOrder); + return dataBlockCodec.encode(dataBlock); } @Override @@ -74,129 +41,4 @@ public String getType() { return TYPE; } - - private static class N5DataBlockOutputStream extends DataBlockOutputStream { - - private final DatasetAttributes attributes; - private final DataBlock dataBlock; - private final ByteOrder byteOrder; - boolean start = true; - - - public N5DataBlockOutputStream(final OutputStream out, final DatasetAttributes attributes, final DataBlock dataBlock, ByteOrder byteOrder) { - super(out); - this.attributes = attributes; - this.dataBlock = dataBlock; - this.byteOrder = byteOrder; - } - - @Override - protected void beforeWrite(int n) throws IOException { - - if (start) { - writeHeader(); - start = false; - } - } - - private void writeHeader() throws IOException { - final DataOutput dos = getDataOutput(out); - - final int mode; - if (attributes.getDataType() == DataType.OBJECT || dataBlock.getSize() == null) - mode = MODE_OBJECT; - else if (dataBlock.getNumElements() == DataBlock.getNumElements(dataBlock.getSize())) - mode = MODE_DEFAULT; - else - mode = MODE_VARLENGTH; - - dos.writeShort(mode); - - if (mode != MODE_OBJECT) { - dos.writeShort(attributes.getNumDimensions()); - for (final int size : dataBlock.getSize()) - dos.writeInt(size); - } - - if (mode != MODE_DEFAULT) - dos.writeInt(dataBlock.getNumElements()); - } - - @Override - public DataOutput getDataOutput(final OutputStream outputStream) { - - if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) - return new DataOutputStream(outputStream); - else - return new LittleEndianDataOutputStream(outputStream); - } - } - - private static class N5DataBlockInputStream extends DataBlockInputStream { - private final DatasetAttributes attributes; - private final long[] gridPosition; - private final ByteOrder byteOrder; - - private short mode = -1; - private int[] blockSize = null; - private int numElements = -1; - - private boolean start = true; - - N5DataBlockInputStream(final InputStream in, final DatasetAttributes attributes, final long[] gridPosition, ByteOrder byteOrder) { - super(in); - this.attributes = attributes; - this.gridPosition = gridPosition; - this.byteOrder = byteOrder; - } - @Override protected void beforeRead(int n) throws IOException { - - if (start) { - readHeader(); - start = false; - } - } - - @Override - public DataBlock allocateDataBlock() throws IOException { - if (start) { - readHeader(); - start = false; - } - if (mode == MODE_OBJECT) { - return (DataBlock) attributes.getDataType().createDataBlock(null, gridPosition, numElements); - } - return (DataBlock) attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); - } - - private void readHeader() throws IOException { - final DataInput dis = getDataInput(in); - mode = dis.readShort(); - if (mode == MODE_OBJECT) { - numElements = dis.readInt(); - return; - } - - final int nDim = dis.readShort(); - blockSize = new int[nDim]; - for (int d = 0; d < nDim; ++d) - blockSize[d] = dis.readInt(); - if (mode == MODE_DEFAULT) { - numElements = DataBlock.getNumElements(blockSize); - } else { - numElements = dis.readInt(); - } - } - - @Override - public DataInput getDataInput(final InputStream inputStream) { - - if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) - return new DataInputStream(inputStream); - else - return new LittleEndianDataInputStream(inputStream); - } - - } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java new file mode 100644 index 000000000..3825f7c2d --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -0,0 +1,365 @@ +package org.janelia.saalfeldlab.n5.codec; + +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataBlock.DataBlockFactory; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock; +import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; +import org.janelia.saalfeldlab.n5.IntArrayDataBlock; +import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; +import org.janelia.saalfeldlab.n5.StringDataBlock; +import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +import javax.annotation.Nullable; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import static org.janelia.saalfeldlab.n5.codec.N5Codecs.BlockHeader.MODE_DEFAULT; +import static org.janelia.saalfeldlab.n5.codec.N5Codecs.BlockHeader.MODE_OBJECT; +import static org.janelia.saalfeldlab.n5.codec.N5Codecs.BlockHeader.MODE_VARLENGTH; + +public class N5Codecs { + + public static final N5DataBlockCodecFactory BYTE = (codecs) -> new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new, codecs); + public static final N5DataBlockCodecFactory SHORT = (codecs) -> new DefaultDataBlockCodec<>(DataCodec.SHORT_BIG_ENDIAN, ShortArrayDataBlock::new, codecs); + public static final N5DataBlockCodecFactory INT = (codecs) -> new DefaultDataBlockCodec<>(DataCodec.INT_BIG_ENDIAN, IntArrayDataBlock::new, codecs); + public static final N5DataBlockCodecFactory LONG = (codecs) -> new DefaultDataBlockCodec<>(DataCodec.LONG_BIG_ENDIAN, LongArrayDataBlock::new, codecs); + public static final N5DataBlockCodecFactory FLOAT = (codecs) -> new DefaultDataBlockCodec<>(DataCodec.FLOAT_BIG_ENDIAN, FloatArrayDataBlock::new, codecs); + public static final N5DataBlockCodecFactory DOUBLE = (codecs) -> new DefaultDataBlockCodec<>(DataCodec.DOUBLE_BIG_ENDIAN, DoubleArrayDataBlock::new, codecs); + public static final N5DataBlockCodecFactory STRING = (codecs) -> new StringDataBlockCodec(DataCodec.STRING, StringDataBlock::new, codecs); + public static final N5DataBlockCodecFactory OBJECT = (codecs) -> new ObjectDataBlockCodec(DataCodec.OBJECT, ByteArrayDataBlock::new, codecs); + + private N5Codecs() { + } + + public static DataBlockCodec createDataBlockCodec( + final DataType dataType, + @Nullable final BytesCodec... codecs) { + + final N5DataBlockCodecFactory factory; + switch (dataType) { + case UINT8: + case INT8: + factory = N5Codecs.BYTE; + break; + case UINT16: + case INT16: + factory = N5Codecs.SHORT; + break; + case UINT32: + case INT32: + factory = N5Codecs.INT; + break; + case UINT64: + case INT64: + factory = N5Codecs.LONG; + break; + case FLOAT32: + factory = N5Codecs.FLOAT; + break; + case FLOAT64: + factory = N5Codecs.DOUBLE; + break; + case STRING: + factory = N5Codecs.STRING; + break; + case OBJECT: + factory = N5Codecs.OBJECT; + break; + default: + throw new IllegalArgumentException("Unsupported data type: " + dataType); + } + final N5DataBlockCodecFactory tFactory = (N5DataBlockCodecFactory)factory; + return tFactory.createDataBlockCodec(codecs); + } + + public interface N5DataBlockCodecFactory { + + DataBlockCodec createDataBlockCodec(@Nullable BytesCodec... codecs); + } + + public abstract static class AbstractDataBlockCodec implements DataBlockCodec { + + private final DataCodec dataCodec; + private final DataBlockFactory dataBlockFactory; + private final BytesCodec[] codecs; + + public AbstractDataBlockCodec( + final DataCodec dataCodec, + final DataBlockFactory dataBlockFactory, + final BytesCodec... codecs + ) { + + this.dataCodec = dataCodec; + this.dataBlockFactory = dataBlockFactory; + this.codecs = codecs; + } + + public DataBlockFactory getDataBlockFactory() { + + return dataBlockFactory; + } + + public DataCodec getDataCodec() { + + return dataCodec; + } + + public BytesCodec[] getCodecs() { + + return codecs; + } + } + + static abstract class AbstractN5DataBlockCodec extends AbstractDataBlockCodec { + + private static final int VAR_OBJ_BYTES_PER_ELEMENT = 1; + + AbstractN5DataBlockCodec( + final DataCodec dataCodec, + final DataBlockFactory dataBlockFactory, + final BytesCodec... codecs) { + + super(dataCodec, dataBlockFactory, codecs); + } + + protected abstract BlockHeader encodeBlockHeader(final DataBlock dataBlock, ReadData blockData) throws IOException; + + @Override public ReadData encode(DataBlock dataBlock) throws IOException { + + return ReadData.from(out -> { + + final ReadData dataReadData = getDataCodec().serialize(dataBlock.getData()); + final ReadData encodedData = new ConcatenatedBytesCodec(getCodecs()).encode(dataReadData); + final BlockHeader header = encodeBlockHeader(dataBlock, dataReadData); + + header.writeTo(out); + encodedData.writeTo(out); + }); + } + + protected abstract BlockHeader decodeBlockHeader(final InputStream in) throws IOException; + + @Override + public DataBlock decode(final ReadData readData, final long[] gridPosition) throws IOException { + + try (final InputStream in = readData.inputStream()) { + final BlockHeader header = decodeBlockHeader(in); + + final int bytesPerElement + = getDataCodec().bytesPerElement() == -1 + ? VAR_OBJ_BYTES_PER_ELEMENT + : getDataCodec().bytesPerElement(); + + final int numElements = header.numElements(); + final ReadData blockData = ReadData.from(in, numElements * bytesPerElement); + final ReadData decodeData = new ConcatenatedBytesCodec(getCodecs()).decode(blockData); + final T data = getDataCodec().deserialize(decodeData, numElements); + return getDataBlockFactory().createDataBlock(header.blockSize(), gridPosition, data); + } + } + } + + /** + * DataBlockCodec for all N5 data types, except STRING and OBJECT + */ + static class DefaultDataBlockCodec extends AbstractN5DataBlockCodec { + + DefaultDataBlockCodec( + final DataCodec dataCodec, + final DataBlockFactory dataBlockFactory, + final BytesCodec... codecs) { + + super(dataCodec, dataBlockFactory, codecs); + } + + @Override + protected BlockHeader encodeBlockHeader(final DataBlock dataBlock, ReadData blockData) throws IOException { + + return new BlockHeader(dataBlock.getSize(), dataBlock.getNumElements()); + } + + @Override + protected BlockHeader decodeBlockHeader(final InputStream in) throws IOException { + + return BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); + } + } + + /** + * DataBlockCodec for N5 data type STRING + */ + static class StringDataBlockCodec extends AbstractN5DataBlockCodec { + + public StringDataBlockCodec( + final DataCodec dataCodec, + final DataBlockFactory dataBlockFactory, + final BytesCodec... codecs) { + + super(dataCodec, dataBlockFactory, codecs); + } + + @Override + protected BlockHeader encodeBlockHeader(final DataBlock dataBlock, ReadData blockData) throws IOException { + + return new BlockHeader(dataBlock.getSize(), (int)blockData.length()); + } + + @Override + protected BlockHeader decodeBlockHeader(final InputStream in) throws IOException { + + return BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); + } + } + + /** + * DataBlockCodec for N5 data type OBJECT + */ + static class ObjectDataBlockCodec extends AbstractN5DataBlockCodec { + + public ObjectDataBlockCodec( + final DataCodec dataCodec, + final DataBlockFactory dataBlockFactory, + final BytesCodec... codecs) { + + super(dataCodec, dataBlockFactory, codecs); + } + + @Override protected BlockHeader encodeBlockHeader(DataBlock dataBlock, ReadData blockData) throws IOException { + + return new BlockHeader(null, dataBlock.getNumElements()); + } + + @Override + protected BlockHeader decodeBlockHeader(final InputStream in) throws IOException { + + return BlockHeader.readFrom(in, MODE_OBJECT); + } + } + + static class BlockHeader { + + public static final short MODE_DEFAULT = 0; + public static final short MODE_VARLENGTH = 1; + public static final short MODE_OBJECT = 2; + + private final short mode; + private final int[] blockSize; + private final int numElements; + + BlockHeader(final short mode, final int[] blockSize, final int numElements) { + + this.mode = mode; + this.blockSize = blockSize; + this.numElements = numElements; + } + + BlockHeader(final int[] blockSize, final int numElements) { + + if (blockSize == null) { + this.mode = MODE_OBJECT; + } else if (DataBlock.getNumElements(blockSize) == numElements) { + this.mode = MODE_DEFAULT; + } else { + this.mode = MODE_VARLENGTH; + } + this.blockSize = blockSize; + this.numElements = numElements; + } + + public short mode() { + + return mode; + } + + public int[] blockSize() { + + return blockSize; + } + + public int numElements() { + + return numElements; + } + + private static int[] readBlockSize(final DataInputStream dis) throws IOException { + + final int nDim = dis.readShort(); + final int[] blockSize = new int[nDim]; + for (int d = 0; d < nDim; ++d) + blockSize[d] = dis.readInt(); + return blockSize; + } + + private static void writeBlockSize(final int[] blockSize, final DataOutputStream dos) throws IOException { + + dos.writeShort(blockSize.length); + for (final int size : blockSize) + dos.writeInt(size); + } + + void writeTo(final OutputStream out) throws IOException { + + final DataOutputStream dos = new DataOutputStream(out); + dos.writeShort(mode); + switch (mode) { + case MODE_DEFAULT:// default + writeBlockSize(blockSize, dos); + break; + case MODE_VARLENGTH:// varlength + writeBlockSize(blockSize, dos); + dos.writeInt(numElements); + break; + case MODE_OBJECT: // object + dos.writeInt(numElements); + break; + default: + throw new IOException("unexpected mode: " + mode); + } + dos.flush(); + } + + static BlockHeader readFrom(final InputStream in, short... allowedModes) throws IOException { + + final DataInputStream dis = new DataInputStream(in); + final short mode = dis.readShort(); + final int[] blockSize; + final int numElements; + switch (mode) { + case MODE_DEFAULT:// default + blockSize = readBlockSize(dis); + numElements = DataBlock.getNumElements(blockSize); + break; + case MODE_VARLENGTH:// varlength + blockSize = readBlockSize(dis); + numElements = dis.readInt(); + break; + case MODE_OBJECT: // object + blockSize = null; + numElements = dis.readInt(); + break; + default: + throw new IOException("Unexpected mode: " + mode); + } + + boolean modeIsOk = allowedModes == null || allowedModes.length == 0; + for (int i = 0; !modeIsOk && i < allowedModes.length; ++i) { + if (mode == allowedModes[i]) { + modeIsOk = true; + break; + } + } + if (!modeIsOk) { + throw new IOException("Unexpected mode: " + mode); + } + + return new BlockHeader(mode, blockSize, numElements); + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java new file mode 100644 index 000000000..c42e6a2ec --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java @@ -0,0 +1,115 @@ +package org.janelia.saalfeldlab.n5.codec; + +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock; +import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; +import org.janelia.saalfeldlab.n5.IntArrayDataBlock; +import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteOrder; + +public class RawBlockCodecs { + + public static final RawDataBlockCodecFactory BYTE = (byteOrder, blockSize, codecs) -> new RawDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new, blockSize, codecs); + public static final RawDataBlockCodecFactory SHORT = (byteOrder, blockSize, codecs) -> new RawDataBlockCodec<>(DataCodec.SHORT(byteOrder), ShortArrayDataBlock::new, blockSize, codecs); + public static final RawDataBlockCodecFactory INT = (byteOrder, blockSize, codecs) -> new RawDataBlockCodec<>(DataCodec.INT(byteOrder), IntArrayDataBlock::new, blockSize, codecs); + public static final RawDataBlockCodecFactory LONG = (byteOrder, blockSize, codecs) -> new RawDataBlockCodec<>(DataCodec.LONG(byteOrder), LongArrayDataBlock::new, blockSize, codecs); + public static final RawDataBlockCodecFactory FLOAT = (byteOrder, blockSize, codecs) -> new RawDataBlockCodec<>(DataCodec.FLOAT(byteOrder), FloatArrayDataBlock::new, blockSize, codecs); + public static final RawDataBlockCodecFactory DOUBLE = (byteOrder, blockSize, codecs) -> new RawDataBlockCodec<>(DataCodec.DOUBLE(byteOrder), DoubleArrayDataBlock::new, blockSize, codecs); + + private RawBlockCodecs() {} + + public static DataBlockCodec createDataBlockCodec( + final DataType dataType, + @Nullable final ByteOrder byteOrder, + final int[] blockSize, + @Nullable final Codec.BytesCodec... codecs) { + final RawDataBlockCodecFactory factory; + switch (dataType) { + case UINT8: + case INT8: + factory = RawBlockCodecs.BYTE; + break; + case UINT16: + case INT16: + factory = RawBlockCodecs.SHORT; + break; + case UINT32: + case INT32: + factory = RawBlockCodecs.INT; + break; + case UINT64: + case INT64: + factory = RawBlockCodecs.LONG; + break; + case FLOAT32: + factory = RawBlockCodecs.FLOAT; + break; + case FLOAT64: + factory = RawBlockCodecs.DOUBLE; + break; + default: + throw new IllegalArgumentException("Unsupported data type: " + dataType); + } + final RawDataBlockCodecFactory tFactory = (RawDataBlockCodecFactory)factory; + return tFactory.createDataBlockCodec(byteOrder, blockSize, codecs); + } + + public interface RawDataBlockCodecFactory { + + DataBlockCodec createDataBlockCodec(@Nullable ByteOrder byteOrder, int[] blockSize, @Nullable Codec.BytesCodec... codecs); + } + + private static class RawDataBlockCodec extends N5Codecs.AbstractDataBlockCodec { + + private final int[] blockSize; + + RawDataBlockCodec( + final DataCodec dataCodec, + final DataBlock.DataBlockFactory dataBlockFactory, + final int[] blockSize, + final Codec.BytesCodec... codecs) { + + super(dataCodec, dataBlockFactory, codecs); + this.blockSize = blockSize; + } + + private int numElements() { + if (blockSize.length == 0) + return 0; + + int elements = 1; + for (int dimLen : blockSize) { + elements *= dimLen; + } + return elements; + } + + @Override public ReadData encode(DataBlock dataBlock) { + + return ReadData.from(out -> { + final ReadData blockData = getDataCodec().serialize(dataBlock.getData()); + new ConcatenatedBytesCodec(getCodecs()).encode(blockData).writeTo(out); + }); + } + + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { + + try (final InputStream in = readData.inputStream()) { + final int bytesPerElement = getDataCodec().bytesPerElement(); + final ReadData blockData = ReadData.from(in, numElements() * bytesPerElement); + final ReadData decodeData = new ConcatenatedBytesCodec(getCodecs()).decode(blockData); + + final T data = getDataCodec().deserialize(decodeData, numElements()); + return getDataBlockFactory().createDataBlock(blockSize, gridPosition, data); + } + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java index bb3232e43..90f9a6604 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java @@ -11,7 +11,9 @@ import java.util.Arrays; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; import com.google.common.io.LittleEndianDataInputStream; @@ -27,7 +29,7 @@ import javax.annotation.Nullable; @NameConfig.Name(value = RawBytes.TYPE) -public class RawBytes implements Codec.ArrayCodec { +public class RawBytes implements Codec.ArrayCodec { private static final long serialVersionUID = 3282569607795127005L; @@ -36,9 +38,12 @@ public class RawBytes implements Codec.ArrayCodec { @NameConfig.Parameter(value = "endian", optional = true) protected final ByteOrder byteOrder; + private DataBlockCodec dataBlockCodec; + private DatasetAttributes attributes; + public RawBytes() { - this(ByteOrder.LITTLE_ENDIAN); + this(ByteOrder.BIG_ENDIAN); } public RawBytes(final ByteOrder byteOrder) { @@ -51,55 +56,21 @@ public ByteOrder getByteOrder() { return byteOrder; } - @Override - public DataBlockInputStream decode(final DatasetAttributes attributes, final long[] gridPosition, InputStream in) - throws IOException { - - return new DataBlockInputStream(in) { - - private int[] blockSize = attributes.getBlockSize(); - private int numElements = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y); - - @Override - protected void beforeRead(int n) {} - - @Override - public DataBlock allocateDataBlock() { - - return attributes.getDataType().createDataBlock(blockSize, gridPosition, numElements); - } - - @Override - public DataInput getDataInput(final InputStream inputStream) { - - if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) - return new DataInputStream(inputStream); - - return new LittleEndianDataInputStream(inputStream); - } - - }; + @Override public void setDatasetAttributes(DatasetAttributes attributes, BytesCodec... codecs) { + ensureValidByteOrder(attributes.getDataType(), getByteOrder()); + this.attributes = attributes; + final BytesCodec[] byteCodecs = codecs == null ? attributes.getCodecs() : codecs; + this.dataBlockCodec = RawBlockCodecs.createDataBlockCodec(attributes.getDataType(), getByteOrder(), attributes.getBlockSize(), byteCodecs); } - @Override - public DataBlockOutputStream encode(final DatasetAttributes attributes, final DataBlock dataBlock, - final OutputStream out) - throws IOException { - - return new DataBlockOutputStream(out) { + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { - @Override - protected void beforeWrite(int n) throws IOException {} + return dataBlockCodec.decode(readData, gridPosition); + } - @Override - public DataOutput getDataOutput(OutputStream outputStream) { + @Override public ReadData encode(DataBlock dataBlock) throws IOException { - if (byteOrder.equals(ByteOrder.BIG_ENDIAN)) - return new DataOutputStream(outputStream); - else - return new LittleEndianDataOutputStream(outputStream); - } - }; + return dataBlockCodec.encode(dataBlock); } @Override @@ -110,6 +81,20 @@ public String getType() { public static final ByteOrderAdapter byteOrderAdapter = new ByteOrderAdapter(); + public static void ensureValidByteOrder(final DataType dataType, @Nullable final ByteOrder byteOrder) { + + switch (dataType) { + case INT8: + case UINT8: + case STRING: + case OBJECT: + return; + } + + if (byteOrder == null) + throw new IllegalArgumentException("DataType (" + dataType + ") requires ByteOrder, but was null"); + } + public static class ByteOrderAdapter implements JsonDeserializer, JsonSerializer { @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java index 7d7a58fbc..88a6d8534 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java @@ -11,6 +11,7 @@ import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.readdata.ReadData; /** * A {@link Codec} that appends a checksum to data when encoding and can validate against that checksum when decoding. @@ -39,11 +40,8 @@ public int numChecksumBytes() { return numChecksumBytes; } - @Override - public CheckedOutputStream encode(final OutputStream out) throws IOException { - - // when do we validate? - return new CheckedOutputStream(out, getChecksum()) { + private CheckedOutputStream createStream(OutputStream out) throws IOException { + return new CheckedOutputStream(out, getChecksum()) { private boolean closed = false; @Override public void close() throws IOException { @@ -57,26 +55,16 @@ public CheckedOutputStream encode(final OutputStream out) throws IOException { }; } - @Override - public CheckedInputStream decode(final InputStream in) throws IOException { + @Override public ReadData encode(ReadData readData) throws IOException { - // TODO get the correct expected checksum - // TODO write a test with nested checksum codecs + return readData.encode(this::createStream); - // has to know the number of it needs to read? - return new CheckedInputStream(in, getChecksum()); } - public ByteBuffer decodeAndValidate(final InputStream in, int numBytes) throws IOException, ChecksumException { - - final CheckedInputStream cin = decode(in); - final byte[] data = new byte[numBytes]; - cin.read(data); + @Override public ReadData decode(ReadData readData) throws IOException { - if (!valid(in)) - throw new ChecksumException("Invalid checksum"); - return ByteBuffer.wrap(data); + return ReadData.from(new CheckedInputStream(readData.inputStream(), getChecksum())); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java new file mode 100644 index 000000000..6f636ad75 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java @@ -0,0 +1,32 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.DataInputStream; +import java.io.IOException; +import org.apache.commons.io.IOUtils; + +// not thread-safe +abstract class AbstractInputStreamReadData implements ReadData { + + private ByteArraySplittableReadData bytes; + + @Override + public ReadData materialize() throws IOException { + if (bytes == null) { + final byte[] data; + final int length = (int) length(); + if (length >= 0) { + data = new byte[length]; + new DataInputStream(inputStream()).readFully(data); + } else { + data = IOUtils.toByteArray(inputStream()); + } + bytes = new ByteArraySplittableReadData(data); + } + return bytes; + } + + @Override + public byte[] allBytes() throws IOException, IllegalStateException { + return materialize().allBytes(); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java new file mode 100644 index 000000000..a46ef5f64 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java @@ -0,0 +1,47 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +class ByteArraySplittableReadData implements ReadData { + + private final byte[] data; + private final int offset; + private final int length; + + ByteArraySplittableReadData(final byte[] data) { + this(data, 0, data.length); + } + + ByteArraySplittableReadData(final byte[] data, final int offset, final int length) { + this.data = data; + this.offset = offset; + this.length = length; + } + + @Override + public long length() { + return length; + } + + @Override + public InputStream inputStream() throws IOException { + return new ByteArrayInputStream(data, offset, length); + } + + @Override + public byte[] allBytes() { + if (offset == 0 && data.length == length) { + return data; + } else { + return Arrays.copyOfRange(data, offset, offset + length); + } + } + + @Override + public ReadData materialize() throws IOException { + return this; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java new file mode 100644 index 000000000..8d15bde13 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java @@ -0,0 +1,32 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.InputStream; + +// not thread-safe +class InputStreamReadData extends AbstractInputStreamReadData { + + private final InputStream inputStream; + private final int length; + + InputStreamReadData(final InputStream inputStream, final int length) { + this.inputStream = inputStream; + this.length = length; + } + + @Override + public long length() { + return length; + } + + private boolean inputStreamCalled = false; + + @Override + public InputStream inputStream() throws IllegalStateException { + if (inputStreamCalled) { + throw new IllegalStateException("InputStream() already called"); + } else { + inputStreamCalled = true; + return inputStream; + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java new file mode 100644 index 000000000..ea114c061 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java @@ -0,0 +1,43 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.IOException; +import java.io.InputStream; +import org.apache.commons.io.input.ProxyInputStream; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; + +class KeyValueAccessReadData extends AbstractInputStreamReadData { + + private final KeyValueAccess keyValueAccess; + private final String normalPath; + + KeyValueAccessReadData(final KeyValueAccess keyValueAccess, final String normalPath) { + this.keyValueAccess = keyValueAccess; + this.normalPath = normalPath; + } + + /** + * Open a {@code InputStream} on this data. + *

    + * This will open a {@code LockedChannel} on the underlying {@code + * KeyValueAccess}. Make sure to {@code close()} the returned {@code + * InputStream} to release the underlying {@code LockedChannel}. + * + * @return an InputStream on this data + * + * @throws IOException + * if any I/O error occurs + */ + @Override + public InputStream inputStream() throws IOException { + final LockedChannel channel = keyValueAccess.lockForReading(normalPath); + return new ProxyInputStream(channel.newInputStream()) { + + @Override + public void close() throws IOException { + in.close(); + channel.close(); + } + }; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java new file mode 100644 index 000000000..b67d99e13 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -0,0 +1,81 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import org.apache.commons.io.output.ProxyOutputStream; + +class LazyReadData implements ReadData { + + LazyReadData(final OutputStreamWriter writer) { + this.writer = writer; + } + + /** + * Construct a {@code LazyReadData} that uses the given {@code OutputStreamEncoder} to + * encode the given {@code ReadData}. + * + * @param data + * the ReadData to encode + * @param encoder + * OutputStreamEncoder to use for encoding + */ + LazyReadData(final ReadData data, final OutputStreamOperator encoder) { + this(outputStream -> { + try (final OutputStream deflater = encoder.apply(interceptClose.apply(outputStream))) { + data.writeTo(deflater); + } + }); + } + + private final OutputStreamWriter writer; + + private ByteArraySplittableReadData bytes; + + @Override + public ReadData materialize() throws IOException { + if (bytes == null) { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); + writeTo(baos); + bytes = new ByteArraySplittableReadData(baos.toByteArray()); + } + return bytes; + } + + @Override + public long length() throws IOException { + return materialize().length(); + } + + @Override + public InputStream inputStream() throws IOException, IllegalStateException { + return materialize().inputStream(); + } + + @Override + public byte[] allBytes() throws IOException, IllegalStateException { + return materialize().allBytes(); + } + + @Override + public void writeTo(final OutputStream outputStream) throws IOException, IllegalStateException { + if (bytes != null) { + outputStream.write(bytes.allBytes()); + } else { + writer.writeTo(outputStream); + } + } + + /** + * {@code UnaryOperator} that wraps {@code OutputStream} to intercept {@code + * close()} and call {@code flush()} instead + */ + private static OutputStreamOperator interceptClose = o -> new ProxyOutputStream(o) { + + @Override + public void close() throws IOException { + out.flush(); + } + }; +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java new file mode 100644 index 000000000..21f2ab3e3 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -0,0 +1,269 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import org.janelia.saalfeldlab.n5.KeyValueAccess; + +/** + * An abstraction over {@code byte[]} data. + *

    + * The data may come from a {@code byte[]} array, a {@code ByteBuffer}, an + * {@code InputStream}, a {@code KeyValueAccess}. + *

    + * {@code ReadData} instances can be created via one of the static {@link #from} + * methods. For example, use {@link #from(InputStream, int)} to wrap an {@code + * InputStream}. + *

    + * {@code ReadData} may be lazy-loaded. For example, for {@code InputStream} and + * {@code KeyValueAccess} sources, loading is deferred until the data is + * accessed (e.g., {@link #allBytes()}, {@link #writeTo(OutputStream)}). + *

    + * {@code ReadData} can be {@code encoded} and {@code decoded} with a {@code + * Codec}, which will also be lazy if possible. + */ +public interface ReadData { + + /** + * Returns number of bytes in this {@link ReadData}, if known. Otherwise + * {@code -1}. + * + * @return number of bytes, if known, or -1 + * + * @throws IOException + * if an I/O error occurs while trying to get the length + */ + default long length() throws IOException { + return -1; + } + + /** + * Open a {@code InputStream} on this data. + *

    + * Repeatedly calling this method may or may not work, depending on how + * the underlying data is stored. For example, if the underlying data is + * stored as a {@code byte[]} array, multiple streams can be opened. If + * the underlying data is just an {@code InputStream} then this will be + * returned on the first call. + * + * @return an InputStream on this data + * + * @throws IOException + * if any I/O error occurs + * @throws IllegalStateException + * if this method was already called once and cannot be called again. + */ + InputStream inputStream() throws IOException, IllegalStateException; + + /** + * Return the contained data as a {@code byte[]} array. + *

    + * This may use {@link #inputStream()} to read the data. + * Because repeatedly calling {@link #inputStream()} may not work, + *

      + *
    1. this method may fail with {@code IllegalStateException} if {@code inputStream()} was already called
      2. + *
    2. subsequent {@code inputStream()} calls may fail with {@code IllegalStateException}
      3. + *
    + * + * @return all contained data as a byte[] array + * + * @throws IOException + * if any I/O error occurs + * @throws IllegalStateException + * if {@link #inputStream()} was already called once and cannot be called again. + */ + byte[] allBytes() throws IOException, IllegalStateException; + + /** + * Return the contained data as a {@code ByteBuffer}. + *

    + * This may use {@link #inputStream()} to read the data. + * Because repeatedly calling {@link #inputStream()} may not work, + *

      + *
    1. this method may fail with {@code IllegalStateException} if {@code inputStream()} was already called
      2. + *
    2. subsequent {@code inputStream()} calls may fail with {@code IllegalStateException}
      3. + *
    + * The byte order of the returned {@code ByteBuffer} is {@code BIG_ENDIAN}. + * + * @return all contained data as a ByteBuffer + * + * @throws IOException + * if any I/O error occurs + * @throws IllegalStateException + * if {@link #inputStream()} was already called once and cannot be called again. + */ + default ByteBuffer toByteBuffer() throws IOException, IllegalStateException { + return ByteBuffer.wrap(allBytes()); + } + + /** + * Read the underlying data into a {@code byte[]} array, and return it as a {@code ReadData}. + * (If this {@code ReadData} is already in a {@code byte[]} array or {@code + * ByteBuffer}, just return {@code this}.) + *

    + * The returned {@code ReadData} has a known {@link #length} and multiple + * {@link #inputStream InputStreams} can be opened on it. + */ + ReadData materialize() throws IOException; + + /** + * Write the contained data into an {@code OutputStream}. + *

    + * This may use {@link #inputStream()} to read the data. + * Because repeatedly calling {@link #inputStream()} may not work, + *

      + *
    1. this method may fail with {@code IllegalStateException} if {@code inputStream()} was already called
      2. + *
    2. subsequent {@code inputStream()} calls may fail with {@code IllegalStateException}
      3. + *
    + * + * @param outputStream + * destination to write to + * + * @throws IOException + * if any I/O error occurs + * @throws IllegalStateException + * if {@link #inputStream()} was already called once and cannot be called again. + */ + default void writeTo(OutputStream outputStream) throws IOException, IllegalStateException { + outputStream.write(allBytes()); + } + + // ------------- Encoding / Decoding ---------------- + // + + /** + * Returns a new ReadData that uses the given {@code OutputStreamEncoder} to + * encode this ReadData. + * + * @param encoder + * OutputStreamEncoder to use for encoding + * + * @return encoded ReadData + */ + default ReadData encode(OutputStreamOperator encoder) { + return new LazyReadData(this, encoder); + } + + /** + * Like {@code UnaryOperator}, but {@code apply} throws {@code IOException}. + */ + @FunctionalInterface + interface OutputStreamOperator { + + OutputStream apply(OutputStream o) throws IOException; + + } + + // --------------- Factory Methods ------------------ + // + + /** + * Create a new {@code ReadData} that loads lazily from {@code inputStream} + * and {@link #length() reports} the given {@code length}. + *

    + * No effort is made to ensure that the {@code inputStream} in fact contains + * exactly {@code length} bytes. + * + * @param inputStream + * InputStream to read from + * @param length + * reported length of the ReadData + * + * @return a new ReadData + */ + static ReadData from(final InputStream inputStream, final int length) { + return new InputStreamReadData(inputStream, length); + } + + /** + * Create a new {@code ReadData} that loads lazily from {@code inputStream} + * and reports {@link #length() length() == -1} (i.e., unknown length). + * + * @param inputStream + * InputStream to read from + * + * @return a new ReadData + */ + static ReadData from(final InputStream inputStream) { + return from(inputStream, -1); + } + + /** + * Create a new {@code ReadData} that loads lazily from {@code normalPath} + * in {@code keyValueAccess}. The returned ReadData reports {@link #length() + * length() == -1} (i.e., unknown length). + * + * @param keyValueAccess + * KeyValueAccess to read from + * @param normalPath + * path in the {@code keyValueAccess} to read from + * + * @return a new ReadData + */ + static ReadData from(final KeyValueAccess keyValueAccess, final String normalPath) { + return new KeyValueAccessReadData(keyValueAccess, normalPath); + } + + /** + * Create a new {@code ReadData} that wraps the specified portion of a + * {@code byte[]} array. + * + * @param data + * array containing the data + * @param offset + * start offset of the ReadData in the data array + * @param length + * length of the ReadData (in bytes) + * + * @return a new ReadData + */ + static ReadData from(final byte[] data, final int offset, final int length) { + return new ByteArraySplittableReadData(data, offset, length); + } + + /** + * Create a new {@code ReadData} that wraps the given {@code byte[]} array. + * + * @param data + * array containing the data + * + * @return a new ReadData + */ + static ReadData from(final byte[] data) { + return from(data, 0, data.length); + } + + /** + * Create a new {@code ReadData} that wraps the given {@code ByteBuffer}. + * + * @param data + * buffer containing the data + * + * @return a new ReadData + */ + static ReadData from(final ByteBuffer data) { + if (data.hasArray()) { + return from(data.array(), 0, data.limit()); + } else { + throw new UnsupportedOperationException("TODO. Direct ByteBuffer not supported yet."); + } + } + + @FunctionalInterface + interface OutputStreamWriter { + void writeTo(OutputStream outputStream) throws IOException, IllegalStateException; + } + + /** + * Create a new {@code ReadData} that is lazily generated by the given {@link OutputStreamWriter}. + * + * @param generator + * generates the data + * + * @return a new ReadData + */ + static ReadData from(OutputStreamWriter generator) { + return new LazyReadData(generator); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 0ee9fa98f..59c828ba3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -158,7 +158,7 @@ default InputStream getAsStream() throws IOException { final SplitableData splitData = new SplitByteBufferedData(); ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); - final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); + final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); final Codec.BytesCodec[] codecs = shardingCodec.getCodecs(); final ShardIndex index = ShardIndex.createIndex(datasetAttributes); @@ -168,10 +168,7 @@ default InputStream getAsStream() throws IOException { try (final OutputStream blocksOut = blocksSplitData.newOutputStream()) { for (DataBlock block : getBlocks()) { try (final CountingOutputStream blockOut = new CountingOutputStream(blocksOut)) { - final Codec.DataBlockOutputStream dataBlockOutput = arrayCodec.encode(datasetAttributes, block, blockOut); - try (final OutputStream stream = Codec.encode(dataBlockOutput, codecs)) { - block.writeData(dataBlockOutput.getDataOutput(stream)); - } + arrayCodec.encode(block).writeTo(blockOut); index.set(blocksOffset, blockOut.getByteCount(), getBlockPosition(block.getGridPosition())); blocksOffset += blockOut.getByteCount(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 69aa5e1b8..8fdf63f23 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -12,16 +12,19 @@ import org.janelia.saalfeldlab.n5.SplitableData; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.N5Annotations; import org.janelia.saalfeldlab.n5.serialization.NameConfig; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.lang.reflect.Array; import java.lang.reflect.Type; +import java.util.Objects; @NameConfig.Name(ShardingCodec.TYPE) -public class ShardingCodec implements Codec.ArrayCodec { +public class ShardingCodec implements Codec.ArrayCodec { private static final long serialVersionUID = -5879797314954717810L; @@ -31,6 +34,7 @@ public class ShardingCodec implements Codec.ArrayCodec { public static final String INDEX_LOCATION_KEY = "index_location"; public static final String CODECS_KEY = "codecs"; public static final String INDEX_CODECS_KEY = "index_codecs"; + private DatasetAttributes attributes = null; public enum IndexLocation { START, END; @@ -80,18 +84,21 @@ public IndexLocation getIndexLocation() { return indexLocation; } - public ArrayCodec getArrayCodec() { + public ArrayCodec getArrayCodec() { - return (Codec.ArrayCodec)codecs[0]; + Objects.requireNonNull(codecs); + if (codecs.length == 0) + throw new IllegalArgumentException("Sharding Codec requires a single ArrayCodec. None found."); + + return (ArrayCodec)codecs[0]; } public BytesCodec[] getCodecs() { - if (codecs.length == 1) - return new BytesCodec[]{}; - + Objects.requireNonNull(codecs); final BytesCodec[] bytesCodecs = new BytesCodec[codecs.length - 1]; - System.arraycopy(codecs, 1, bytesCodecs, 0, bytesCodecs.length); + for (int i = 1; i < codecs.length; i++) + bytesCodecs[i] = (BytesCodec)codecs[i]; return bytesCodecs; } @@ -111,17 +118,22 @@ public DeterministicSizeCodec[] getIndexCodecs() { return attributes.getShardPositionForBlock(blockPosition); } - @Override public DataBlockInputStream decode(DatasetAttributes attributes, long[] gridPosition, InputStream in) throws IOException { + @Override public void setDatasetAttributes(DatasetAttributes attributes, final BytesCodec... codecs) { + this.attributes = attributes; + getArrayCodec().setDatasetAttributes(attributes, getCodecs()); + } - return getArrayCodec().decode(attributes, gridPosition, in); + @Override public ReadData encode(DataBlock dataBlock) throws IOException { + + return getArrayCodec().encode(dataBlock); } - @Override public DataBlockOutputStream encode(DatasetAttributes attributes, DataBlock dataBlock, OutputStream out) throws IOException { + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { - return getArrayCodec().encode(attributes, dataBlock, out); + return getArrayCodec().decode(readData, gridPosition); } - @Override public void writeBlock( + public void writeBlock( final SplitableData splitData, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) { @@ -130,7 +142,6 @@ public DeterministicSizeCodec[] getIndexCodecs() { new VirtualShard(datasetAttributes, shardPos, splitData).writeBlock(dataBlock); } - @Override public DataBlock readBlock( final SplitableData splitData, final DatasetAttributes datasetAttributes, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 6ff449669..a7e61fdbf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -6,6 +6,7 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.SplitableData; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.util.GridIterator; import java.io.IOException; @@ -33,20 +34,10 @@ public VirtualShard( } @SuppressWarnings("unchecked") - public DataBlock getBlock(InputStream in, long... blockGridPosition) throws IOException { + public DataBlock getBlock(ReadData blockData, long... blockGridPosition) throws IOException { - ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); - final Codec.BytesCodec[] codecs = shardingCodec.getCodecs(); - final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); - - final Codec.DataBlockInputStream dataBlockStream = arrayCodec.decode(datasetAttributes, blockGridPosition, in); - - final DataBlock dataBlock = dataBlockStream.allocateDataBlock(); - final InputStream stream = Codec.decode(in, codecs); - dataBlock.readData(dataBlockStream.getDataInput(stream)); - stream.close(); - - return dataBlock; + ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); + return shardingCodec.getArrayCodec().decode(blockData, blockGridPosition); } @Override @@ -92,7 +83,7 @@ public List> getBlocks(final int[] blockIndexes) { final long numBytes = index.getNumBytesByBlockIndex((int)idx); //TODO Caleb: Do this with a single access (start at first offset, read through the last) try (final InputStream in = splitableData.split(offset, numBytes).newInputStream()) { - final DataBlock block = getBlock(in, position.clone()); + final DataBlock block = getBlock(ReadData.from(in), position.clone()); blocks.add(block); } catch (final N5Exception.N5NoSuchKeyException e) { return blocks; @@ -119,7 +110,7 @@ public DataBlock getBlock(long... blockGridPosition) { final long[] blockPosInImg = getDatasetAttributes().getBlockPositionFromShardPosition(getGridPosition(), blockGridPosition); try (final InputStream in = splitableData.split(blockOffset, blockSize).newInputStream()) { - return getBlock(in, blockPosInImg); + return getBlock(ReadData.from(in), blockPosInImg); } catch (final N5Exception.N5NoSuchKeyException e) { return null; } catch (final IOException | UncheckedIOException e) { @@ -178,14 +169,8 @@ void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); - final Codec.BytesCodec[] codecs = shardingCodec.getCodecs(); - final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); - final Codec.DataBlockOutputStream dataBlockOutput = arrayCodec.encode(datasetAttributes, dataBlock, out); - final OutputStream stream = Codec.encode(dataBlockOutput, codecs); - - dataBlock.writeData(dataBlockOutput.getDataOutput(stream)); - stream.close(); + ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); + shardingCodec.getArrayCodec().encode(dataBlock); } public ShardIndex createIndex() { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index b8083e06c..deff73ea9 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.nio.ByteOrder; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -53,6 +54,7 @@ import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; +import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.junit.After; import org.junit.Before; @@ -102,7 +104,7 @@ public N5Writer createTempN5Writer() { } } - protected final N5Writer createTempN5Writer(String location) { + public final N5Writer createTempN5Writer(String location) { return createTempN5Writer(location, new GsonBuilder()); } @@ -159,11 +161,11 @@ protected Compression[] getCompressions() { return new Compression[]{ new RawCompression(), - new Bzip2Compression(), - new GzipCompression(), - new GzipCompression(5, true), - new Lz4Compression(), - new XzCompression() +// new Bzip2Compression(), +// new GzipCompression(), +// new GzipCompression(5, true), +// new Lz4Compression(), +// new XzCompression() }; } @@ -256,30 +258,31 @@ public void testWriteReadByteBlockMultipleCodecs() { /*TODO: this tests "passes" in the sense that we get the correct output, but it * maybe is not the behavior we actually want*/ + try (final N5Writer n5 = createTempN5Writer()) { + final String dataset = "8_64_32"; + n5.remove(dataset); final Codec[] codecs = { - new N5BlockCodec(), - new AsTypeCodec(DataType.INT32, DataType.INT8), - new AsTypeCodec(DataType.INT64, DataType.INT32), + new N5BlockCodec<>(), + new AsTypeCodec(DataType.INT8, DataType.INT32), + new AsTypeCodec(DataType.INT32, DataType.INT64) }; - final long[] longBlock1 = new long[]{1,2,3,4,5,6,7,8}; + final byte[] byteBlock1 = new byte[]{1,2,3,4,5,6,7,8}; final long[] dimensions1 = new long[]{2,2,2}; final int[] blockSize1 = new int[]{2,2,2}; - n5.createDataset(datasetName, dimensions1, blockSize1, DataType.INT8, codecs); - final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); - final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize1, new long[]{0, 0, 0}, longBlock1); - n5.writeBlock(datasetName, attributes, dataBlock); - - final DatasetAttributes fakeAttributes = new DatasetAttributes(dimensions1, blockSize1, DataType.INT64, codecs); - final DataBlock loadedDataBlock = n5.readBlock(datasetName, fakeAttributes, 0, 0, 0); - assertArrayEquals(longBlock1, (long[])loadedDataBlock.getData()); - assertTrue(n5.remove(datasetName)); - + final DatasetAttributes attrs = new DatasetAttributes(dimensions1, blockSize1, DataType.INT8, codecs); + n5.createDataset(dataset, attrs); + final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); + final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize1, new long[]{0, 0, 0}, byteBlock1); + n5.writeBlock(dataset, attributes, dataBlock); + + final DataBlock loadedDataBlock = n5.readBlock(dataset, attrs, 0, 0, 0); + assertArrayEquals(byteBlock1, (byte[])loadedDataBlock.getData()); } } @Test - public void testWriteReadStringBlock() { + public void testWriteReadStringBlock() throws IOException, URISyntaxException { // test dataset; all characters are valid UTF8 but may have different numbers of bytes! final DataType dataType = DataType.STRING; @@ -287,7 +290,7 @@ public void testWriteReadStringBlock() { final String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-þ"}; for (final Compression compression : getCompressions()) { - try (final N5Writer n5 = createTempN5Writer()) { + try (final N5Writer n5 = createN5Writer("test.n5")) { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final StringDataBlock dataBlock = new StringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); @@ -297,7 +300,7 @@ public void testWriteReadStringBlock() { assertArrayEquals(stringBlock, (String[])loadedDataBlock.getData()); - assertTrue(n5.remove(datasetName)); +// assertTrue(n5.remove(datasetName)); } } @@ -476,7 +479,7 @@ public void testWriteReadSerializableBlock() throws ClassNotFoundException { @Test public void testOverwriteBlock() { - try (final N5Writer n5 = createTempN5Writer()) { + try (final N5Writer n5 = createTempN5Writer("test.n5")) { n5.createDataset(datasetName, dimensions, blockSize, DataType.INT32, new GzipCompression()); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); @@ -485,11 +488,12 @@ public void testOverwriteBlock() { final DataBlock loadedRandomDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(intBlock, (int[])loadedRandomDataBlock.getData()); - // test the case where the resulting file becomes shorter - final IntArrayDataBlock emptyDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, new int[DataBlock.getNumElements(blockSize)]); + // test the case where the resulting file becomes shorter (because the data compresses better) + final int[] emptyBlock = new int[DataBlock.getNumElements(blockSize)]; + final IntArrayDataBlock emptyDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, emptyBlock); n5.writeBlock(datasetName, attributes, emptyDataBlock); final DataBlock loadedEmptyDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); - assertArrayEquals(new int[DataBlock.getNumElements(blockSize)], (int[])loadedEmptyDataBlock.getData()); + assertArrayEquals(emptyBlock, (int[])loadedEmptyDataBlock.getData()); assertTrue(n5.remove(datasetName)); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java index 8f72edaf9..647a3500d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java @@ -100,7 +100,7 @@ public static void setUpBeforeClass() throws Exception { throw new IOException("Could not create benchmark directory for HDF5Utils benchmark."); data = new short[64 * 64 * 64]; - final ImagePlus imp = new Opener().openURL("https://imagej.nih.gov/ij/images/t1-head-raw.zip"); + final ImagePlus imp = new Opener().openURL("https://imagej.net/ij/images/t1-head-raw.zip"); final ImagePlusImg img = (ImagePlusImg)(Object)ImagePlusImgs.from(imp); final Cursor cursor = Views.flatIterable(Views.interval(img, new long[]{100, 100, 30}, new long[]{163, 163, 93})).cursor(); for (int i = 0; i < data.length; ++i) @@ -221,7 +221,7 @@ public void benchmarkWritingSpeed() { } } - @Test +// @Test public void benchmarkParallelWritingSpeed() { final int nBlocks = 5; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index da0a38ec0..a1d659f7c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -132,7 +132,7 @@ public void customObjectTest() { } } - @Test +// @Test public void testReadLock() throws IOException { final Path path = Paths.get(tempN5PathName(), "lock"); @@ -165,7 +165,7 @@ public void testReadLock() throws IOException { exec.shutdownNow(); } - @Test +// @Test public void testWriteLock() throws IOException { final Path path = Paths.get(tempN5PathName(), "lock"); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java index 59aa32984..c1ff93a14 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java @@ -11,6 +11,7 @@ import java.nio.ByteBuffer; import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.junit.Test; public class AsTypeTests { @@ -55,19 +56,14 @@ public static void testEncodingAndDecoding(Codec.BytesCodec codec, byte[] encode public static void testDecoding(final Codec.BytesCodec codec, final byte[] expected, final byte[] input) throws IOException { - final InputStream result = codec.decode(new ByteArrayInputStream(input)); - for (int i = 0; i < expected.length; i++) - assertEquals(expected[i], (byte)result.read()); + final ReadData result = codec.decode(ReadData.from(input)); + assertArrayEquals(expected, result.allBytes()); } public static void testEncoding(final Codec.BytesCodec codec, final byte[] expected, final byte[] data) throws IOException { - final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(expected.length); - final OutputStream encodedStream = codec.encode(outputStream); - encodedStream.write(data); - encodedStream.flush(); - assertArrayEquals(expected, outputStream.toByteArray()); - encodedStream.close(); + final byte[] encodedData = codec.encode(ReadData.from(data)).allBytes(); + assertArrayEquals(expected, encodedData); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java index ea24a7d90..8a0d8abb7 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java @@ -37,8 +37,8 @@ public void testSerialization() { new long[]{8, 8}, new int[]{4, 4}, DataType.UINT8, - new N5BlockCodec(ByteOrder.LITTLE_ENDIAN), - new IdentityCodec() + new N5BlockCodec<>(), + new IdentityCodec() ); writer.createGroup("shard"); //Should already exist, but this will ensure. writer.setAttribute("shard", "/", datasetAttributes); @@ -47,7 +47,5 @@ public void testSerialization() { assertEquals("1 codecs", 1, deserialized.getCodecs().length); assertTrue("Identity", deserialized.getCodecs()[0] instanceof IdentityCodec); assertTrue("Bytes", deserialized.getArrayCodec() instanceof N5BlockCodec); - assertEquals("LittleEndian", ByteOrder.LITTLE_ENDIAN, - ((N5BlockCodec)deserialized.getArrayCodec()).byteOrder); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java index c96edc079..4bcca0bc3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java @@ -12,6 +12,7 @@ import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; @@ -33,8 +34,8 @@ public static void shardBlockIterator() { new int[] {6, 4}, // shard size new int[] {2, 2}, // block size DataType.UINT8, - new Codec[] { new RawBytes() }, - new DeterministicSizeCodec[] { new RawBytes() }, + new Codec[] { new N5BlockCodec<>() }, + new DeterministicSizeCodec[] { new N5BlockCodec<>() }, IndexLocation.END); shardPositions(attrs) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index 0fbca0a6e..c1cb8b09f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -6,6 +6,7 @@ import org.janelia.saalfeldlab.n5.SplitKeyValueAccessData; import org.janelia.saalfeldlab.n5.SplitableData; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; @@ -34,7 +35,7 @@ public void testOffsetIndex() { int[] shardBlockGridSize = new int[]{5, 4, 3}; ShardIndex index = new ShardIndex( shardBlockGridSize, - IndexLocation.END, new RawBytes()); + IndexLocation.END, new RawBytes<>()); GridIterator it = new GridIterator(shardBlockGridSize); int i = 0; @@ -47,7 +48,7 @@ public void testOffsetIndex() { shardBlockGridSize = new int[]{5, 4, 3, 13}; index = new ShardIndex( shardBlockGridSize, - IndexLocation.END, new RawBytes()); + IndexLocation.END, new RawBytes<>()); it = new GridIterator(shardBlockGridSize); i = 0; @@ -67,7 +68,7 @@ public void testReadVirtual() throws IOException { final int[] shardBlockGridSize = new int[]{6, 5}; final IndexLocation indexLocation = IndexLocation.END; - final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { new RawBytes(), + final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { new RawBytes<>(), new Crc32cChecksumCodec()}; final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "0").toString(); @@ -97,7 +98,7 @@ public void testReadInMemory() throws IOException { final int[] shardBlockGridSize = new int[]{6, 5}; final IndexLocation indexLocation = IndexLocation.END; final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[]{ - new RawBytes(), + new RawBytes<>(), new Crc32cChecksumCodec()}; final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "indexTest").toString(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index cfb9d0199..a4681b8ab 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -36,7 +36,7 @@ @RunWith(Parameterized.class) public class ShardTest { - private static final boolean LOCAL_DEBUG = false; + private static final boolean LOCAL_DEBUG = true; private static final N5FSTest tempN5Factory = new N5FSTest() { @@ -90,10 +90,10 @@ private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, shardSize, blockSize, DataType.UINT8, - new ShardingCodec( + new ShardingCodec<>( blockSize, - new Codec[]{new N5BlockCodec(dataByteOrder)}, //, new GzipCompression(4)}, - new DeterministicSizeCodec[]{new RawBytes(indexByteOrder), new Crc32cChecksumCodec()}, + new Codec[]{new N5BlockCodec<>()}, //, new GzipCompression(4)}, + new DeterministicSizeCodec[]{new N5BlockCodec<>(), new Crc32cChecksumCodec()}, indexLocation ) ); @@ -221,29 +221,29 @@ public void writeReadBlockTest() { final HashMap writtenBlocks = new HashMap<>(); - for (int idx1 = 1; idx1 >= 0; idx1--) { - for (int idx2 = 1; idx2 >= 0; idx2--) { - final long[] gridPosition = {idx1, idx2}; - final DataBlock dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); - byte[] data = (byte[])dataBlock.getData(); - for (int i = 0; i < data.length; i++) { - data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); - } - writer.writeBlock(dataset, datasetAttributes, dataBlock); - - final DataBlock block = writer.readBlock(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); - Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); - - for (Map.Entry entry : writtenBlocks.entrySet()) { - final long[] otherGridPosition = entry.getKey(); - final byte[] otherData = entry.getValue(); - final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); - Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); - } - - writtenBlocks.put(gridPosition, data); - } - } +// for (int idx1 = 1; idx1 >= 0; idx1--) { +// for (int idx2 = 1; idx2 >= 0; idx2--) { +// final long[] gridPosition = {idx1, idx2}; +// final DataBlock dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); +// byte[] data = (byte[])dataBlock.getData(); +// for (int i = 0; i < data.length; i++) { +// data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); +// } +// writer.writeBlock(dataset, datasetAttributes, dataBlock); +// +// final DataBlock block = writer.readBlock(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); +// Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); +// +// for (Map.Entry entry : writtenBlocks.entrySet()) { +// final long[] otherGridPosition = entry.getKey(); +// final byte[] otherData = entry.getValue(); +// final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); +// Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); +// } +// +// writtenBlocks.put(gridPosition, data); +// } +// } } @Test @@ -265,18 +265,18 @@ public void writeReadShardTest() { final InMemoryShard shard = new InMemoryShard<>(datasetAttributes, new long[]{0, 0}); - for (int idx1 = 1; idx1 >= 0; idx1--) { - for (int idx2 = 1; idx2 >= 0; idx2--) { - final long[] gridPosition = {idx1, idx2}; - final DataBlock dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); - byte[] data = (byte[])dataBlock.getData(); - for (int i = 0; i < data.length; i++) { - data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); - } - shard.addBlock((DataBlock)dataBlock); - writtenBlocks.put(gridPosition, data); - } - } +// for (int idx1 = 1; idx1 >= 0; idx1--) { +// for (int idx2 = 1; idx2 >= 0; idx2--) { +// final long[] gridPosition = {idx1, idx2}; +// final DataBlock dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); +// byte[] data = (byte[])dataBlock.getData(); +// for (int i = 0; i < data.length; i++) { +// data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); +// } +// shard.addBlock((DataBlock)dataBlock); +// writtenBlocks.put(gridPosition, data); +// } +// } writer.writeShard(dataset, datasetAttributes, shard); @@ -324,7 +324,7 @@ private DatasetAttributes getNestedShardCodecsAttributes(int[] blockSize) { // probably better to forget about this class - only use DatasetAttributes // and detect shading in another way final ShardingCodec innerShard = new ShardingCodec(innerShardSize, - new Codec[]{new RawBytes()}, + new Codec[]{new N5BlockCodec<>()}, new DeterministicSizeCodec[]{new RawBytes(indexByteOrder), new Crc32cChecksumCodec()}, IndexLocation.START); From 1ba662d72b29c688c4c57cb45d10946c11ecd6a6 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 20 May 2025 14:34:50 -0400 Subject: [PATCH 206/423] chore: remove duplicate license headers --- .../n5/CachedGsonKeyValueN5Reader.java | 25 ------------------- .../n5/CachedGsonKeyValueN5Writer.java | 25 ------------------- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 25 ------------------- .../janelia/saalfeldlab/n5/GsonN5Reader.java | 25 ------------------- .../janelia/saalfeldlab/n5/GsonN5Writer.java | 25 ------------------- .../saalfeldlab/n5/N5KeyValueReader.java | 25 ------------------- .../saalfeldlab/n5/N5KeyValueWriter.java | 25 ------------------- .../saalfeldlab/n5/http/N5HttpTest.java | 25 ------------------- 8 files changed, 200 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index bc0e5583d..8be7376cb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.lang.reflect.Type; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index 2858ef725..aad0ec7a9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 8c4e1e04d..7cac000f8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import com.google.gson.Gson; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index 7d273158a..c4ae61495 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.lang.reflect.Type; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java index 4b782ae30..f5f7ea6c4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import com.google.gson.Gson; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 27fc99bd7..f814ab9da 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.net.URI; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 37a9ecdb4..1b49b013c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import com.google.gson.GsonBuilder; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/N5HttpTest.java b/src/test/java/org/janelia/saalfeldlab/n5/http/N5HttpTest.java index 98c866976..672e3e0b6 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/N5HttpTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/N5HttpTest.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5.http; import com.google.gson.GsonBuilder; From 31e0a9a3c92448731a5ed14a16864ea39f6d0707 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 20 May 2025 16:30:27 -0400 Subject: [PATCH 207/423] test: add ReadData benchmark --- .../n5/benchmarks/ReadDataBenchmarks.java | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java new file mode 100644 index 000000000..2fb90ea87 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java @@ -0,0 +1,143 @@ +package org.janelia.saalfeldlab.n5.benchmarks; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@State(Scope.Benchmark) +@Warmup(iterations = 10, time = 100, timeUnit = TimeUnit.MICROSECONDS) +@Measurement(iterations = 100, time = 100, timeUnit = TimeUnit.MICROSECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(1) +public class ReadDataBenchmarks { + + @Param(value = { "10000000" }) + protected int objectSizeBytes; + + protected Path basePath; + protected ArrayList tmpPaths; + protected KeyValueAccess kva; + protected Random random; + + public ReadDataBenchmarks() {} + + public static void main(String... args) throws RunnerException { + + final Options options = new OptionsBuilder().include(ReadDataBenchmarks.class.getSimpleName() + "\\.") + .build(); + + new Runner(options).run(); + } + + @Benchmark + public void run(Blackhole hole) throws IOException { + + hole.consume(read().materialize()); + } + + public ReadData read() throws IOException { + + final Path path = basePath.resolve("tmp-" + objectSizeBytes); + return ReadData.from(kva, path.toString()); + } + + @Setup(Level.Trial) + public void setup() throws IOException { + + random = new Random(); + kva = new FileSystemKeyValueAccess(FileSystems.getDefault()); + + basePath = Files.createTempDirectory("ReadDataBenchmark-"); + tmpPaths = new ArrayList<>(); + for (final int sz : sizes()) { + Path p = basePath.resolve("tmp-"+sz); + write(p, sz); + tmpPaths.add(p); + } + } + + protected void write(Path path, int numBytes) { + + final byte[] data = new byte[numBytes]; + random.nextBytes(data); + + System.out.println(path.toAbsolutePath().toString()); + System.out.println(numBytes); + try (final LockedChannel ch = kva.lockForWriting(path.toAbsolutePath().toString())) { + final OutputStream os = ch.newOutputStream(); + os.write(data); + os.flush(); + os.close(); + } catch (final IOException e) { + e.printStackTrace(); + } + } + + private void read(String path, int startByte, int numBytes) { + + try ( final LockedChannel ch = kva.lockForReading(path, startByte, numBytes); + final InputStream is = ch.newInputStream(); ) { + final byte[] data = new byte[numBytes]; + is.read(data); + } catch (final IOException e) { + e.printStackTrace(); + } + } + + @TearDown(Level.Trial) + public void teardown() { + + for ( Path p : tmpPaths ) { + p.toFile().delete(); + } + basePath.toFile().delete(); + } + + public int[] sizes() { + + try { + final Param ann = ReadDataBenchmarks.class.getDeclaredField("objectSizeBytes").getAnnotation(Param.class); + System.out.println(Arrays.toString(ann.value())); + return Arrays.stream(ann.value()).mapToInt(Integer::parseInt).toArray(); + + } catch (final NoSuchFieldException e) { + e.printStackTrace(); + } catch (final SecurityException e) { + e.printStackTrace(); + } + + return null; + } + +} From 38017fff277c9d11abfb9ec2ceb4773562df34d4 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 20 May 2025 16:34:23 -0400 Subject: [PATCH 208/423] fix: AbstractInputStreamReadData close stream --- .../n5/readdata/AbstractInputStreamReadData.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java index c13bd1bf0..5b84a9692 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java @@ -30,6 +30,8 @@ import java.io.DataInputStream; import java.io.IOException; +import java.io.InputStream; + import org.apache.commons.io.IOUtils; // not thread-safe @@ -44,9 +46,13 @@ public ReadData materialize() throws IOException { final int length = (int) length(); if (length >= 0) { data = new byte[length]; - new DataInputStream(inputStream()).readFully(data); + try( InputStream is = inputStream()) { + new DataInputStream(is).readFully(data); + } } else { - data = IOUtils.toByteArray(inputStream()); + try( InputStream is = inputStream()) { + data = IOUtils.toByteArray(is); + } } bytes = new ByteArraySplittableReadData(data); } From 20b5c003e92df59aab50cfdbe49f1777e67714bd Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 20 May 2025 16:55:55 -0400 Subject: [PATCH 209/423] feat: add splittable read data * add a test --- .../readdata/AbstractInputStreamReadData.java | 2 +- .../readdata/ByteArraySplittableReadData.java | 96 ++++++++++++------- .../saalfeldlab/n5/readdata/LazyReadData.java | 2 +- .../saalfeldlab/n5/readdata/ReadData.java | 4 +- .../n5/readdata/SplittableReadData.java | 17 ++++ .../n5/readdata/ReadDataTests.java | 96 +++++++++++++++++++ 6 files changed, 178 insertions(+), 39 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java index 5b84a9692..0a1b8173d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java @@ -40,7 +40,7 @@ abstract class AbstractInputStreamReadData implements ReadData { private ByteArraySplittableReadData bytes; @Override - public ReadData materialize() throws IOException { + public SplittableReadData materialize() throws IOException { if (bytes == null) { final byte[] data; final int length = (int) length(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java index c41b0d195..9bf86ef9c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java @@ -26,50 +26,76 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ - package org.janelia.saalfeldlab.n5.readdata; +package org.janelia.saalfeldlab.n5.readdata; - import java.io.ByteArrayInputStream; - import java.io.IOException; - import java.io.InputStream; - import java.util.Arrays; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; - class ByteArraySplittableReadData implements ReadData { +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; - private final byte[] data; - private final int offset; - private final int length; +class ByteArraySplittableReadData implements SplittableReadData { - ByteArraySplittableReadData(final byte[] data) { - this(data, 0, data.length); - } + private final byte[] data; + private final int offset; + private final int length; - ByteArraySplittableReadData(final byte[] data, final int offset, final int length) { - this.data = data; - this.offset = offset; - this.length = length; - } + ByteArraySplittableReadData(final byte[] data) { - @Override - public long length() { - return length; - } + this(data, 0, data.length); + } - @Override - public InputStream inputStream() throws IOException { - return new ByteArrayInputStream(data, offset, length); - } + ByteArraySplittableReadData(final byte[] data, final int offset, final int length) { + + this.data = data; + this.offset = offset; + this.length = length; + } + + @Override + public long length() { + + return length; + } + + @Override + public InputStream inputStream() throws IOException { - @Override - public byte[] allBytes() { - if (offset == 0 && data.length == length) { - return data; - } else { - return Arrays.copyOfRange(data, offset, offset + length); - } + return new ByteArrayInputStream(data, offset, length); + } + + @Override + public byte[] allBytes() { + + if (offset == 0 && data.length == length) { + return data; + } else { + return Arrays.copyOfRange(data, offset, offset + length); } + } + + @Override + public SplittableReadData materialize() throws IOException { - @Override - public ReadData materialize() throws IOException { - return this; + return this; + } + + @Override + public SplittableReadData slice(final long offset, final long length) throws IOException { + + if (offset < 0 || offset >= this.length || length < 0) { + throw new IndexOutOfBoundsException(); } + final int o = this.offset + (int)offset; + final int l = Math.min((int)length, this.length - o); + return new ByteArraySplittableReadData(data, o, l); + } + + @Override + public Pair split(final long pivot) throws IOException { + + return ImmutablePair.of(slice(0, pivot), slice(offset + pivot, length - pivot)); } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java index 68bb5cf72..7d2f61be1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -62,7 +62,7 @@ class LazyReadData implements ReadData { private ByteArraySplittableReadData bytes; @Override - public ReadData materialize() throws IOException { + public SplittableReadData materialize() throws IOException { if (bytes == null) { final ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); writeTo(baos); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 2dfed0bf5..7e91827f0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -126,14 +126,14 @@ default ByteBuffer toByteBuffer() throws IOException, IllegalStateException { } /** - * Read the underlying data into a {@code byte[]} array, and return it as a {@code ReadData}. + * Read the underlying data into a {@code byte[]} array, and return it as a {@code SplittableReadData}. * (If this {@code ReadData} is already in a {@code byte[]} array or {@code * ByteBuffer}, just return {@code this}.) *

    * The returned {@code ReadData} has a known {@link #length} and multiple * {@link #inputStream InputStreams} can be opened on it. */ - ReadData materialize() throws IOException; + SplittableReadData materialize() throws IOException; /** * Write the contained data into an {@code OutputStream}. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java new file mode 100644 index 000000000..f5195bef5 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java @@ -0,0 +1,17 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.IOException; + +import org.apache.commons.lang3.tuple.Pair; + +public interface SplittableReadData extends ReadData { + + default ReadData limit(final long length) throws IOException { + return slice(0, length); + } + + ReadData slice(final long offset, final long length) throws IOException; + + // TODO do we want this? how should it work? + Pair split(final long pivot) throws IOException; +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java new file mode 100644 index 000000000..b85647416 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -0,0 +1,96 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.lang3.tuple.Pair; +import org.junit.Test; + +public class ReadDataTests { + + @Test + public void testByteArrayReadData() throws IOException { + + final int N = 128; + byte[] data = new byte[N]; + for( int i = 0; i < N; i++ ) + data[i] = (byte)i; + + SplittableReadData readData = ReadData.from(data).materialize(); + assertTrue(readData instanceof ByteArraySplittableReadData); + + readDataTestHelper(readData, N); + splittableReadDataTestHelper(readData, N, 5); + + /* + * Using Long.MAX_VALUE breaks ByteArraySplittableReadData.split + * because the line: + * final int l = Math.min((int) length, this.length - o); + * returns -1 + * + * do we care to address this? + */ +// ReadData unboundedLength = readData.slice(1, Long.MAX_VALUE); + } + + private void readDataTestHelper( ReadData readData, int N ) throws IOException { + + assertEquals("full length", N, readData.length()); + } + + private void splittableReadDataTestHelper( SplittableReadData readData, int N, int pivot ) throws IOException { + + assertEquals("length one", 1, readData.slice(9, 1).length()); + + assertEquals("split length zero", 0, readData.slice(9, 0).length()); + assertEquals("split length zero allBytes", 0, readData.slice(9, 0).allBytes().length); + + ReadData splitOutOfRange = readData.slice(N-1, 3); + assertEquals("Out-of-range split truncates", 1, splitOutOfRange.length()); + assertEquals("Out-of-range split truncates allBytes", 1, splitOutOfRange.allBytes().length); + + ReadData unboundedLength = readData.slice(1, Integer.MAX_VALUE); + assertEquals("unbounded length", N - 1, unboundedLength.length()); + assertEquals("unbounded length allBytes", N - 1, unboundedLength.allBytes().length); + + assertThrows("negative offset", IndexOutOfBoundsException.class, () -> readData.slice(-1, 1)); + assertThrows("negative length", IndexOutOfBoundsException.class, () -> readData.slice(0, -1)); + assertThrows("too large offset", IndexOutOfBoundsException.class, () -> readData.slice(N, 1)); + + final Pair split = readData.split(pivot); + final ReadData first = split.getLeft(); + final ReadData last = split.getRight(); + + assertEquals(pivot, first.length()); + assertEquals(0, first.allBytes()[0]); + + assertEquals(N-pivot, last.length()); + assertEquals(pivot, last.allBytes()[0]); + } + + @Test + public void testInputStreamReadData() throws IOException { + + final int N = 128; + byte[] data = new byte[N]; + for( int i = 0; i < N; i++ ) + data[i] = (byte)i; + + final InputStream is = new InputStream() { + int val = 0; + + @Override + public int read() throws IOException { + return val++; + } + }; + + final ReadData readData = ReadData.from(is, N); + readDataTestHelper(readData, N); + splittableReadDataTestHelper(readData.materialize(), N, 5); + } +} From c7a119ec632360db9a82954d120aa297a8a3fefb Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 21 May 2025 13:42:39 -0400 Subject: [PATCH 210/423] test: fix ShardProperties tests --- .../saalfeldlab/n5/shard/ShardPropertiesTests.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java index eb0d6de47..cf2f08d4b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java @@ -4,6 +4,7 @@ import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.Position; import org.junit.Test; @@ -33,13 +34,13 @@ public void testShardProperties() throws Exception { DataType.UINT8, new ShardingCodec( blkSize, - new Codec[]{}, + new Codec[]{ new RawBytes() }, new DeterministicSizeCodec[]{}, IndexLocation.END ) ); - @SuppressWarnings({"rawtypes", "unchecked"}) final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); + @SuppressWarnings({"rawtypes"}) final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); assertArrayEquals(new int[]{4, 4}, shard.getBlockGridSize()); @@ -72,7 +73,7 @@ public void testShardBlockPositionIterator() throws Exception { DataType.UINT8, new ShardingCodec( blkSize, - new Codec[]{}, + new Codec[]{new RawBytes()}, new DeterministicSizeCodec[]{}, IndexLocation.END ) @@ -109,7 +110,7 @@ public void testShardGrouping() { DataType.UINT8, new ShardingCodec( blkSize, - new Codec[]{}, + new Codec[]{new RawBytes()}, new DeterministicSizeCodec[]{}, IndexLocation.END ) From 1564aa572ba4a45721aab1205c63b970a2aabd34 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 21 May 2025 16:25:55 -0400 Subject: [PATCH 211/423] feat: add FileSplittableReadData --- .../n5/FileSystemKeyValueAccess.java | 22 +++++ .../n5/readdata/FileSplittableReadData.java | 99 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 602a02984..c2677bcfb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -54,6 +54,8 @@ package org.janelia.saalfeldlab.n5; import org.apache.commons.io.input.BoundedInputStream; +import org.janelia.saalfeldlab.n5.readdata.FileSplittableReadData; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import java.io.File; import java.io.IOException; @@ -63,6 +65,7 @@ import java.io.Writer; import java.net.URI; import java.net.URISyntaxException; +import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.OverlappingFileLockException; @@ -244,6 +247,25 @@ public LockedFileChannel lockForReading(final String normalPath, final long star } } + public FileSplittableReadData createReadData(final String normalPath) { + return new FileSplittableReadData(Paths.get(normalPath), 0, -1); + } + + public ReadData read(final String normalPath, final long startByte, final long length) throws IOException { + final FileChannel channel = FileChannel.open(fileSystem.getPath(normalPath), new OpenOption[]{StandardOpenOption.READ}); + channel.position(startByte); + + final int sz = (int)(length < 0 ? channel.size() : (int)length); + final byte[] data = new byte[sz]; + final ByteBuffer buf = ByteBuffer.wrap(data); + channel.read(buf); + return ReadData.from(data); + } + + public ReadData readFully(final String normalPath) throws IOException { + return read(normalPath, 0, -1); + } + @Override public LockedFileChannel lockForWriting(final String normalPath) throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java new file mode 100644 index 000000000..4b76a6799 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java @@ -0,0 +1,99 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; + +public class FileSplittableReadData implements SplittableReadData { + + private ByteArraySplittableReadData materialized; + + private final Path path; + private final long offset; + private final long length; + + public FileSplittableReadData(Path path, long offset, long length) { + this.path = path; + this.offset = offset; + this.length = length; + } + + public FileSplittableReadData(Path path, long offset) { + this(path, offset, -1); + } + + @Override + public long length() { + if (materialized != null) + return materialized.length(); + + return length; + } + + @Override + public InputStream inputStream() throws IOException, IllegalStateException { + return materialize().inputStream(); + } + + @Override + public byte[] allBytes() throws IOException, IllegalStateException { + return materialize().allBytes(); + } + + @Override + public SplittableReadData materialize() throws IOException { + if (materialized == null) + read(); + + return materialized; + } + + private void read() throws IOException { + + final FileChannel channel = FileChannel.open(path, new OpenOption[]{StandardOpenOption.READ}); + channel.position(offset); + + if (length > Integer.MAX_VALUE) + throw new IOException("Attempt to materialize too large data"); + + final int sz = (int)(length < 0 ? channel.size() : (int)length); + final byte[] data = new byte[sz]; + final ByteBuffer buf = ByteBuffer.wrap(data); + channel.read(buf); + materialized = new ByteArraySplittableReadData(data); + } + + @Override + public ReadData slice(long offset, long length) throws IOException { + + if (materialized != null) + return materialize().slice(offset, length); + + return new FileSplittableReadData(path, this.offset + offset, length); + } + + @Override + public Pair split(long pivot) throws IOException { + + if (materialized != null) + return materialize().split(pivot); + + final long offsetL = 0; + final long lenL = pivot; + + final long offsetR = offset + pivot; + final long lenR = this.length - pivot; + + return new ImmutablePair( + new FileSplittableReadData(path, offsetL, lenL), + new FileSplittableReadData(path, offsetR, lenR)); + } + +} From fecd95db597f84dadfc7ae9dbca326753a8fd18e Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 21 May 2025 16:26:26 -0400 Subject: [PATCH 212/423] test: more ReadDataTests --- .../n5/readdata/ReadDataTests.java | 100 +++++++++++++++--- 1 file changed, 88 insertions(+), 12 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index b85647416..68bf3a778 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -1,17 +1,44 @@ package org.janelia.saalfeldlab.n5.readdata; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.FileSystems; +import java.util.Arrays; +import java.util.function.IntUnaryOperator; +import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.lang3.tuple.Pair; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; +import org.janelia.saalfeldlab.n5.readdata.ReadData.OutputStreamOperator; import org.junit.Test; public class ReadDataTests { + @Test + public void testLazyReadData() throws IOException { + + final int N = 128; + byte[] data = new byte[N]; + for( int i = 0; i < N; i++ ) + data[i] = (byte)i; + + final ReadData readData = ReadData.from(out -> { + out.write(data); + }); + assertTrue(readData instanceof LazyReadData); + + readDataTestHelper(readData, N); + splittableReadDataTestHelper(readData.materialize(), N, 5); + } + @Test public void testByteArrayReadData() throws IOException { @@ -24,17 +51,8 @@ public void testByteArrayReadData() throws IOException { assertTrue(readData instanceof ByteArraySplittableReadData); readDataTestHelper(readData, N); + readDataTestEncodeHelper(readData, N); splittableReadDataTestHelper(readData, N, 5); - - /* - * Using Long.MAX_VALUE breaks ByteArraySplittableReadData.split - * because the line: - * final int l = Math.min((int) length, this.length - o); - * returns -1 - * - * do we care to address this? - */ -// ReadData unboundedLength = readData.slice(1, Long.MAX_VALUE); } private void readDataTestHelper( ReadData readData, int N ) throws IOException { @@ -42,6 +60,22 @@ private void readDataTestHelper( ReadData readData, int N ) throws IOException { assertEquals("full length", N, readData.length()); } + private void readDataTestEncodeHelper( ReadData readData, int N ) throws IOException { + + final byte[] origCopy = new byte[N]; + IOUtils.readFully(readData.inputStream(), origCopy); + + final byte[] expected = Arrays.copyOf(origCopy, N); + for( int i = 0; i < expected.length; i++) + expected[i]+=2; + + final ReadData encoded = readData.encode(new ByteFun(x -> x+2)); + assertArrayEquals(expected, encoded.allBytes()); + + final ReadData encodedTwice = encoded.encode(new ByteFun(x -> x-2)); + assertArrayEquals(origCopy, encodedTwice.allBytes()); + } + private void splittableReadDataTestHelper( SplittableReadData readData, int N, int pivot ) throws IOException { assertEquals("length one", 1, readData.slice(9, 1).length()); @@ -49,6 +83,9 @@ private void splittableReadDataTestHelper( SplittableReadData readData, int N, i assertEquals("split length zero", 0, readData.slice(9, 0).length()); assertEquals("split length zero allBytes", 0, readData.slice(9, 0).allBytes().length); + ReadData limited = readData.limit(2); + assertEquals(2, limited.length()); + ReadData splitOutOfRange = readData.slice(N-1, 3); assertEquals("Out-of-range split truncates", 1, splitOutOfRange.length()); assertEquals("Out-of-range split truncates allBytes", 1, splitOutOfRange.allBytes().length); @@ -79,10 +116,9 @@ public void testInputStreamReadData() throws IOException { byte[] data = new byte[N]; for( int i = 0; i < N; i++ ) data[i] = (byte)i; - + final InputStream is = new InputStream() { int val = 0; - @Override public int read() throws IOException { return val++; @@ -93,4 +129,44 @@ public int read() throws IOException { readDataTestHelper(readData, N); splittableReadDataTestHelper(readData.materialize(), N, 5); } + + @Test + public void testFileSplittableReadData() throws IOException { + + int N = 128; + byte[] data = new byte[N]; + for( int i = 0; i < N; i++ ) + data[i] = (byte)i; + + final File tmpF = File.createTempFile("test-file-splittable-data", ".bin"); + tmpF.deleteOnExit(); + try (FileOutputStream os = new FileOutputStream(tmpF)) { + os.write(data); + os.close(); + } + + final FileSplittableReadData readData = new FileSystemKeyValueAccess(FileSystems.getDefault()) + .createReadData(tmpF.getAbsolutePath()); + assertEquals("file read data length unknown", -1, readData.length()); + splittableReadDataTestHelper(readData.materialize(), N, 5); + } + + private class ByteFun implements OutputStreamOperator { + + IntUnaryOperator fun; + public ByteFun(IntUnaryOperator fun) { + this.fun = fun; + } + + @Override + public OutputStream apply(OutputStream o) throws IOException { + return new OutputStream() { + @Override + public void write(int b) throws IOException { + o.write(fun.applyAsInt(b)); + } + }; + } + } + } From 24817915bc14d4159628521bda1378037097c708 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 21 May 2025 16:27:04 -0400 Subject: [PATCH 213/423] test: more read data benchmarks --- .../n5/benchmarks/ReadDataBenchmarks.java | 8 +++- .../ReadDataBenchmarksKvaReadData.java | 43 +++++++++++++++++++ .../ReadDataBenchmarksKvaReadFully.java | 43 +++++++++++++++++++ 3 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadFully.java diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java index 2fb90ea87..989f531e6 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java @@ -68,8 +68,12 @@ public void run(Blackhole hole) throws IOException { public ReadData read() throws IOException { - final Path path = basePath.resolve("tmp-" + objectSizeBytes); - return ReadData.from(kva, path.toString()); + return ReadData.from(kva, getPath().toString()); + } + + protected Path getPath() { + + return basePath.resolve("tmp-" + objectSizeBytes); } @Setup(Level.Trial) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java new file mode 100644 index 000000000..da0bf9958 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java @@ -0,0 +1,43 @@ +package org.janelia.saalfeldlab.n5.benchmarks; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@State(Scope.Benchmark) +@Warmup(iterations = 10, time = 100, timeUnit = TimeUnit.MICROSECONDS) +@Measurement(iterations = 100, time = 100, timeUnit = TimeUnit.MICROSECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(1) +public class ReadDataBenchmarksKvaReadData extends ReadDataBenchmarks { + + public static void main(String... args) throws RunnerException { + + final Options options = new OptionsBuilder().include(ReadDataBenchmarksKvaReadData.class.getSimpleName() + "\\.") + .build(); + + new Runner(options).run(); + } + + public ReadData read() throws IOException { + + return ((FileSystemKeyValueAccess)kva).createReadData(getPath().toString()); + } + +} \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadFully.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadFully.java new file mode 100644 index 000000000..f526f83da --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadFully.java @@ -0,0 +1,43 @@ +package org.janelia.saalfeldlab.n5.benchmarks; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@State(Scope.Benchmark) +@Warmup(iterations = 10, time = 100, timeUnit = TimeUnit.MICROSECONDS) +@Measurement(iterations = 100, time = 100, timeUnit = TimeUnit.MICROSECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(1) +public class ReadDataBenchmarksKvaReadFully extends ReadDataBenchmarks { + + public static void main(String... args) throws RunnerException { + + final Options options = new OptionsBuilder().include(ReadDataBenchmarksKvaReadFully.class.getSimpleName() + "\\.") + .build(); + + new Runner(options).run(); + } + + public ReadData read() throws IOException { + + return ((FileSystemKeyValueAccess)kva).readFully(getPath().toString()); + } + +} \ No newline at end of file From 424c1018550f6b4ea3f5ef748fd97986f160f005 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 21 May 2025 21:04:21 -0400 Subject: [PATCH 214/423] perf: use range request for http KeyValueAccess --- .../janelia/saalfeldlab/n5/HttpKeyValueAccess.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index 4da1f5b09..4b31c9366 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -395,13 +395,15 @@ protected HttpObjectChannel(final URI uri, long startByte, long size) { @Override public InputStream newInputStream() throws IOException { - final InputStream inputStream = uri.toURL().openStream(); - final long skipped = inputStream.skip(startByte); - assert(startByte == skipped); + HttpURLConnection conn = (HttpURLConnection)uri.toURL().openConnection(); + conn.setRequestProperty("Range", rangeString()); + return conn.getInputStream(); + } + + private String rangeString() { - if (size >= 0) - return new BoundedInputStream(inputStream, size); - return inputStream; + final String lastByte = (size > 0) ? Long.toString(startByte + size - 1) : ""; + return String.format("bytes=%d-%s", startByte, lastByte); } @Override From 0023d4e5ade62cf7e64f63cec75b6628ae89a28d Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 22 May 2025 09:49:33 -0400 Subject: [PATCH 215/423] fix(test): add N5BlockCodec to shard test --- .../janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java index eb0d6de47..de6bea363 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java @@ -4,6 +4,7 @@ import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.Position; import org.junit.Test; @@ -33,7 +34,7 @@ public void testShardProperties() throws Exception { DataType.UINT8, new ShardingCodec( blkSize, - new Codec[]{}, + new Codec[]{new N5BlockCodec()}, new DeterministicSizeCodec[]{}, IndexLocation.END ) @@ -72,7 +73,7 @@ public void testShardBlockPositionIterator() throws Exception { DataType.UINT8, new ShardingCodec( blkSize, - new Codec[]{}, + new Codec[]{ new N5BlockCodec() }, new DeterministicSizeCodec[]{}, IndexLocation.END ) @@ -109,7 +110,7 @@ public void testShardGrouping() { DataType.UINT8, new ShardingCodec( blkSize, - new Codec[]{}, + new Codec[]{ new N5BlockCodec() }, new DeterministicSizeCodec[]{}, IndexLocation.END ) From d8aedbdf1277fd87232f1580a85dc707726d80cc Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 22 May 2025 10:22:28 -0400 Subject: [PATCH 216/423] perf/style: http kva, implement range request * remove duplicate license header * add constants --- .../saalfeldlab/n5/HttpKeyValueAccess.java | 70 ++++++++----------- 1 file changed, 31 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index 4da1f5b09..b9cc0c692 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -1,31 +1,3 @@ -/*- - * #%L - * Not HDF5 - * %% - * Copyright (C) 2017 - 2025 Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ /** * Copyright (c) 2017, Stephan Saalfeld All rights reserved. *

    @@ -49,9 +21,10 @@ package org.janelia.saalfeldlab.n5; import org.apache.commons.io.IOUtils; -import org.apache.commons.io.input.BoundedInputStream; + import org.apache.commons.lang3.function.TriFunction; import org.janelia.saalfeldlab.n5.http.ListResponseParser; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import java.io.Closeable; import java.io.IOException; @@ -77,6 +50,13 @@ */ public class HttpKeyValueAccess implements KeyValueAccess { + public static final String HEAD = "HEAD"; + public static final String GET = "GET"; + + public static final String RANGE = "Range"; + public static final String ACCEPT_RANGE = "Accept-Range"; + public static final String BYTES = "bytes"; + private int readTimeoutMilliseconds; private int connectionTimeoutMilliseconds; @@ -168,7 +148,7 @@ public boolean exists(final String normalPath) { public boolean isDirectory(final String normalPath) { try { - requireValidHttpResponse(getDirectoryPath(normalPath), "HEAD", (code, msg,http) -> { + requireValidHttpResponse(getDirectoryPath(normalPath), HEAD, (code, msg,http) -> { final N5Exception cause = validExistsResponse(code, "Error checking directory: " + normalPath, msg, true); if (code >= 300 && code < 400) { final String redirectLocation = http.getHeaderField("Location"); @@ -210,7 +190,7 @@ public boolean isFile(final String normalPath) { /* Files must not end in `/` And Don't accept a redirect to a location ending in `/` */ try { - requireValidHttpResponse(getFilePath(normalPath), "HEAD", (code, msg, http) -> { + requireValidHttpResponse(getFilePath(normalPath), HEAD, (code, msg, http) -> { final N5Exception cause = validExistsResponse(code, "Error accessing file: " + normalPath, msg, true); if (code >= 300 && code < 400) { final String redirectLocation = http.getHeaderField("Location"); @@ -307,7 +287,7 @@ public String[] list(final String normalPath) throws IOException { private String[] queryListEntries(String normalPath, ListResponseParser parser, boolean allowRedirect) { - final HttpURLConnection http = requireValidHttpResponse(normalPath, "GET", "Error listing directory at " + normalPath, allowRedirect); + final HttpURLConnection http = requireValidHttpResponse(normalPath, GET, "Error listing directory at " + normalPath, allowRedirect); try { final String listResponse = responseToString(http.getInputStream()); return parser.parseListResponse(listResponse); @@ -373,7 +353,7 @@ private class HttpObjectChannel implements LockedChannel { private final ArrayList resources = new ArrayList<>(); protected HttpObjectChannel(final URI uri) { - this(uri, -1); + this(uri, 0, -1); } protected HttpObjectChannel(final URI uri, int size) { @@ -392,16 +372,28 @@ protected HttpObjectChannel(final URI uri, long startByte, long size) { return size; } + private boolean isPartialRead() { + return startByte > 0 || (size < 0 && size != Long.MAX_VALUE); + } + @Override public InputStream newInputStream() throws IOException { - final InputStream inputStream = uri.toURL().openStream(); - final long skipped = inputStream.skip(startByte); - assert(startByte == skipped); + HttpURLConnection conn = (HttpURLConnection)uri.toURL().openConnection(); + if (isPartialRead()) { + + conn.setRequestProperty(RANGE, rangeString()); +// final String acceptRanges = conn.getHeaderField(ACCEPT_RANGE); +// if (acceptRanges == null || !acceptRanges.equals(BYTES)) { +// return ReadData.from(conn.getInputStream()).materialize().sli +// } + } + return conn.getInputStream(); + } - if (size >= 0) - return new BoundedInputStream(inputStream, size); - return inputStream; + private String rangeString() { + final String lastByte = (size > 0) ? Long.toString(startByte + size - 1) : ""; + return String.format("%s=%d-%s", BYTES, startByte, lastByte); } @Override From 3d27db9903b109508ff36e632429227494a58e8e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 May 2025 09:33:19 -0400 Subject: [PATCH 217/423] refactor: rename to ArrayCodec#initialize --- .../java/org/janelia/saalfeldlab/n5/DatasetAttributes.java | 4 ++-- src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java | 2 +- .../java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java | 3 +-- src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java | 3 +-- .../java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java | 4 ++-- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index e4cd420cc..7fa306ec0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -153,8 +153,8 @@ public DatasetAttributes( .toArray(BytesCodec[]::new); } - //TODO Caleb: factory style for setDatasetAttributes - arrayCodec.setDatasetAttributes(this); + //TODO Caleb: factory style for initialize + arrayCodec.initialize(this, byteCodecs); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index 02dbbd2f0..d9e6178be 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -64,7 +64,7 @@ default long[] getPositionForBlock(final DatasetAttributes attributes, final lon return blockPosition; } - void setDatasetAttributes(final DatasetAttributes attributes, final BytesCodec... codecs); + void initialize(final DatasetAttributes attributes, final BytesCodec... codecs); @Override default long encodedSize(long size) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java index 93b6bf326..457b6d23f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java @@ -18,9 +18,8 @@ public class N5BlockCodec implements Codec.ArrayCodec { //TODO Caleb: Extract to factory that returns N5BlockCodec wrapper for datablockCodec - @Override public void setDatasetAttributes(final DatasetAttributes attributes, final Codec.BytesCodec... codecs) { + @Override public void initialize(final DatasetAttributes attributes, final Codec.BytesCodec[] byteCodecs) { /*TODO: Consider an attributes.createDataBlockCodec() without parameters? */ - final BytesCodec[] byteCodecs = codecs == null ? attributes.getCodecs() : codecs; final ConcatenatedBytesCodec concatenatedBytesCodec = new ConcatenatedBytesCodec(byteCodecs); this.dataBlockCodec = N5Codecs.createDataBlockCodec(attributes.getDataType(), concatenatedBytesCodec); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java index e1774b543..22f844697 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java @@ -43,9 +43,8 @@ public ByteOrder getByteOrder() { return byteOrder; } - @Override public void setDatasetAttributes(DatasetAttributes attributes, BytesCodec... codecs) { + @Override public void initialize(DatasetAttributes attributes, BytesCodec[] byteCodecs) { ensureValidByteOrder(attributes.getDataType(), getByteOrder()); - final BytesCodec[] byteCodecs = codecs == null ? attributes.getCodecs() : codecs; final ConcatenatedBytesCodec concatenatedBytesCodec = new ConcatenatedBytesCodec(byteCodecs); this.dataBlockCodec = RawBlockCodecs.createDataBlockCodec(attributes.getDataType(), getByteOrder(), attributes.getBlockSize(), concatenatedBytesCodec); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 8fdf63f23..825f5e121 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -118,9 +118,9 @@ public DeterministicSizeCodec[] getIndexCodecs() { return attributes.getShardPositionForBlock(blockPosition); } - @Override public void setDatasetAttributes(DatasetAttributes attributes, final BytesCodec... codecs) { + @Override public void initialize(DatasetAttributes attributes, final BytesCodec[] codecs) { this.attributes = attributes; - getArrayCodec().setDatasetAttributes(attributes, getCodecs()); + getArrayCodec().initialize(attributes, getCodecs()); } @Override public ReadData encode(DataBlock dataBlock) throws IOException { From f3949376f6b4f4d7692978edbeb68600caaa8eb6 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 May 2025 09:34:26 -0400 Subject: [PATCH 218/423] feat: ReadDataSplittableReadData and KVA#createReadData; --- .../n5/FileSystemKeyValueAccess.java | 1 + .../saalfeldlab/n5/KeyValueAccess.java | 6 + .../n5/readdata/FileSplittableReadData.java | 9 +- .../readdata/ReadDataSplittableReadData.java | 112 ++++++++++++++++++ 4 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadDataSplittableReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index c2677bcfb..c8d29d611 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -247,6 +247,7 @@ public LockedFileChannel lockForReading(final String normalPath, final long star } } + @Override public FileSplittableReadData createReadData(final String normalPath) { return new FileSplittableReadData(Paths.get(normalPath), 0, -1); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 6fff4de58..4f058c3c6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -53,6 +53,8 @@ */ package org.janelia.saalfeldlab.n5; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -252,6 +254,10 @@ default URI uri(final String uriString) throws URISyntaxException { */ public boolean isFile(String normalPath); // TODO: Looks un-used. Remove? + default ReadData createReadData(final String normalPath) throws IOException { + return ReadData.from(this, normalPath); + } + /** * Create a lock on a path for reading. This isn't meant to be kept * around. Create, use, [auto]close, e.g. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java index 4b76a6799..8382b0b59 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java @@ -4,12 +4,14 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.NoSuchFileException; import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.janelia.saalfeldlab.n5.N5Exception; public class FileSplittableReadData implements SplittableReadData { @@ -57,7 +59,12 @@ public SplittableReadData materialize() throws IOException { private void read() throws IOException { - final FileChannel channel = FileChannel.open(path, new OpenOption[]{StandardOpenOption.READ}); + final FileChannel channel; + try { + channel = FileChannel.open(path, StandardOpenOption.READ); + } catch (final NoSuchFileException e) { + throw new N5Exception.N5NoSuchKeyException(e); + } channel.position(offset); if (length > Integer.MAX_VALUE) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadDataSplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadDataSplittableReadData.java new file mode 100644 index 000000000..58add721f --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadDataSplittableReadData.java @@ -0,0 +1,112 @@ +/*- + * #%L + * Not HDF5 + * %% + * Copyright (C) 2017 - 2025 Stephan Saalfeld + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.janelia.saalfeldlab.n5.readdata; + +import org.apache.commons.io.input.BoundedInputStream; +import org.apache.commons.io.input.ProxyInputStream; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +class ReadDataSplittableReadData implements SplittableReadData { + + private final SplittableReadData readData; + private final long offset; + private final long length; + + ReadDataSplittableReadData(final SplittableReadData readData, final long offset, final long length) { + + this.readData = readData; + this.offset = offset; + this.length = length; + } + + @Override + public long length() { + + return length; + } + + @Override + public InputStream inputStream() throws IOException { + + final InputStream offsetInputStream = new ProxyInputStream(readData.inputStream()) { + + private boolean firstRead = true; + + @Override protected void beforeRead(int n) throws IOException { + + if (firstRead) { + inputStream().skip(offset); + + firstRead = false; + } + + super.beforeRead(n); + } + }; + + return BoundedInputStream.builder() + .setInputStream(offsetInputStream) + .setBufferSize((int)length) + .get(); + } + + @Override + public byte[] allBytes() throws IOException, IllegalStateException { + + final byte[] bytes = readData.allBytes(); + if (offset == 0 && bytes.length == length) { + return bytes; + } else { + return Arrays.copyOfRange(bytes, (int)offset, (int)(offset + length)); + } + } + + @Override + public SplittableReadData materialize() throws IOException { + + return this; + } + + @Override + public SplittableReadData slice(final long offset, final long length) { + + return new ReadDataSplittableReadData(this, offset, length); + } + + @Override + public Pair split(final long pivot) throws IOException { + + return ImmutablePair.of(slice(0, pivot), slice(offset + pivot, length - pivot)); + } +} From b56a3945594ebb24e495bf551c521330ed6e41cd Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 May 2025 09:37:13 -0400 Subject: [PATCH 219/423] refactor: Shards should use ReadData, not previous SplitData implementation refactor: reduce scope of shard capability. Only support full Shard writes (InMemoryShard) and partial shard reads (VirtualShard) --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 32 ++-- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 60 ++++---- .../saalfeldlab/n5/shard/InMemoryShard.java | 15 +- .../janelia/saalfeldlab/n5/shard/Shard.java | 79 +++++----- .../saalfeldlab/n5/shard/ShardIndex.java | 12 +- .../saalfeldlab/n5/shard/ShardingCodec.java | 26 +--- .../saalfeldlab/n5/shard/VirtualShard.java | 145 ++++-------------- 7 files changed, 129 insertions(+), 240 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 7cac000f8..73637de22 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -32,7 +32,9 @@ import com.google.gson.JsonElement; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.SplittableReadData; import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.shard.VirtualShard; import org.janelia.saalfeldlab.n5.util.Position; @@ -103,16 +105,15 @@ default Shard readShard( long... shardGridPosition) { final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(keyPath), shardGridPosition); - final SplitKeyValueAccessData splitableData; + final SplittableReadData splitableData; try { - splitableData = new SplitKeyValueAccessData(getKeyValueAccess(), path); + splitableData = getKeyValueAccess().createReadData(path).materialize(); + } catch (N5Exception.N5NoSuchKeyException e) { + return null; } catch (IOException e) { throw new N5IOException(e); } - return new VirtualShard( - datasetAttributes, - shardGridPosition, - splitableData); + return new VirtualShard( datasetAttributes, shardGridPosition, splitableData); } @Override @@ -124,20 +125,14 @@ default DataBlock readBlock( final long[] keyPos = datasetAttributes.getArrayCodec().getPositionForBlock(datasetAttributes, gridPosition); final String keyPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), keyPos); - - final SplitKeyValueAccessData splitData; try { - splitData = new SplitKeyValueAccessData(getKeyValueAccess(), keyPath); - try (final InputStream inputStream = splitData.newInputStream()) { - final ReadData decodeData = ReadData.from(inputStream); - return datasetAttributes.getArrayCodec().decode(decodeData, gridPosition); - } + final ReadData decodeData = getKeyValueAccess().createReadData(keyPath); + return datasetAttributes.getArrayCodec().decode(decodeData, gridPosition); } catch (N5Exception.N5NoSuchKeyException e) { return null; } catch (IOException e) { throw new N5IOException(e); } - } @Override @@ -147,20 +142,21 @@ default List> readBlocks( final List blockPositions) throws N5Exception { // TODO which interface should have this implementation? - if (datasetAttributes.getShardSize() != null) { + if (datasetAttributes.getArrayCodec() instanceof ShardingCodec) { /* Group by shard position */ final Map> shardBlockMap = datasetAttributes.groupBlockPositions(blockPositions); final ArrayList> blocks = new ArrayList<>(); for( Entry> e : shardBlockMap.entrySet()) { - final Shard shard = readShard(pathName, datasetAttributes, e.getKey().get()); + Shard currentShard = readShard(pathName, datasetAttributes, e.getKey().get()); + if (currentShard == null) + continue; for (final long[] blkPosition : e.getValue()) { - blocks.add(shard.getBlock(blkPosition)); + blocks.add(currentShard.getBlock(blkPosition)); } } - return blocks; } return GsonN5Reader.super.readBlocks(pathName, datasetAttributes, blockPositions); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 0fd17dee2..414b974b7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -53,6 +53,16 @@ */ package org.janelia.saalfeldlab.n5; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonSyntaxException; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.shard.InMemoryShard; +import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.util.Position; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -63,17 +73,6 @@ import java.util.Map.Entry; import java.util.stream.Collectors; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.shard.InMemoryShard; -import org.janelia.saalfeldlab.n5.shard.Shard; - -import com.google.gson.Gson; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.JsonSyntaxException; -import org.janelia.saalfeldlab.n5.util.Position; - /** * Default implementation of {@link N5Writer} with JSON attributes parsed with * {@link Gson}. @@ -256,13 +255,18 @@ default boolean removeAttributes(final String pathName, final List attri final Map>> shardBlockMap = datasetAttributes.groupBlocks( Arrays.stream(dataBlocks).collect(Collectors.toList())); - for( final Entry>> e : shardBlockMap.entrySet()) { + for (final Entry>> e : shardBlockMap.entrySet()) { final long[] shardPosition = e.getKey().get(); final Shard currentShard = readShard(datasetPath, datasetAttributes, shardPosition); + final InMemoryShard newShard; + if (currentShard != null) { + newShard = InMemoryShard.fromShard(currentShard); + } else { + newShard = new InMemoryShard<>(datasetAttributes, shardPosition); + } - final InMemoryShard newShard = InMemoryShard.fromShard(currentShard); - for( DataBlock blk : e.getValue()) + for (DataBlock blk : e.getValue()) newShard.addBlock(blk); writeShard(datasetPath, datasetAttributes, newShard); @@ -279,17 +283,21 @@ default void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception { + if (datasetAttributes.getShardSize() != null) { + writeBlocks(path, datasetAttributes, dataBlock); + return; + } + final long[] keyPos = datasetAttributes.getArrayCodec().getPositionForBlock(datasetAttributes, dataBlock); final String keyPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), keyPos); - final SplitableData splitData; - try { + try ( + final LockedChannel channel = getKeyValueAccess().lockForWriting(keyPath); + final OutputStream outputStream = channel.newOutputStream() + ) { //TODO Caleb: What behavior do we want when writing a new block at an existing one? // If the encoded size is smaller, should it truncate the remaining, or ignore it? // currently we truncate for the default writeBlock method (shards don't truncate) - splitData = new SplitKeyValueAccessData(getKeyValueAccess(), keyPath).split(0, Long.MAX_VALUE); - try (final OutputStream out = splitData.newOutputStream()) { - datasetAttributes.getArrayCodec().encode(dataBlock).writeTo(out); - } + datasetAttributes.getArrayCodec().encode(dataBlock).writeTo(outputStream); } catch (IOException e) { throw new N5IOException(e); } @@ -302,13 +310,11 @@ default void writeShard( final Shard shard) throws N5Exception { final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shard.getGridPosition()); - final SplitKeyValueAccessData splitData = new SplitKeyValueAccessData(getKeyValueAccess(), shardPath, 0, Long.MAX_VALUE); - try (final OutputStream shardOut = splitData.newOutputStream()) { - try (final InputStream shardIn = shard.getAsStream()) { - while (shardIn.available() > 0) { - shardOut.write(shardIn.read()); - } - } + try ( + final LockedChannel channel = getKeyValueAccess().lockForWriting(shardPath); + final OutputStream shardOut = channel.newOutputStream() + ) { + shard.createReadData().writeTo(shardOut); } catch (final IOException | UncheckedIOException e) { throw new N5IOException( "Failed to write shard " + Arrays.toString(shard.getGridPosition()) + " into dataset " + path, e); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 6bfa889e2..38019beb0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -1,18 +1,11 @@ package org.janelia.saalfeldlab.n5.shard; -import org.apache.commons.io.output.ByteArrayOutputStream; -import org.apache.commons.io.output.CountingOutputStream; -import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.DefaultBlockWriter; -import org.janelia.saalfeldlab.n5.SplitByteBufferedData; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.janelia.saalfeldlab.n5.util.Position; -import java.io.IOException; -import java.io.OutputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -59,12 +52,6 @@ private void storeBlock(DataBlock block) { return blocks.get(Position.wrap(blockGridPosition)); } - @Override - public void writeBlock(DataBlock block) { - - addBlock(block); - } - public void addBlock(DataBlock block) { storeBlock(block); @@ -121,7 +108,7 @@ public static InMemoryShard fromShard(Shard shard) { shard.getDatasetAttributes(), shard.getGridPosition()); - shard.forEach(blk -> inMemoryShard.addBlock(blk)); + shard.forEach(inMemoryShard::addBlock); return inMemoryShard; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 59c828ba3..b4ddc0476 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -1,21 +1,17 @@ package org.janelia.saalfeldlab.n5.shard; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.io.output.CountingOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.SplitByteBufferedData; -import org.janelia.saalfeldlab.n5.SplitableData; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.util.GridIterator; public interface Shard extends Iterable> { @@ -106,10 +102,6 @@ default long[] getShardPosition(long... blockPosition) { public DataBlock getBlock(long... blockGridPosition); - public void writeBlock(DataBlock block); - - //TODO Caleb: add writeBlocks that does NOT always expect to overwrite the entire existing Shard - default Iterator> iterator() { return new DataBlockIterator<>(this); @@ -151,40 +143,53 @@ static Shard createEmpty(final DatasetAttributes attributes, long... shar return new InMemoryShard(attributes, shardPosition, shardIndex); } - default InputStream getAsStream() throws IOException { + default ReadData createReadData() throws IOException { final DatasetAttributes datasetAttributes = getDatasetAttributes(); + final ShardIndex index = ShardIndex.createIndex(datasetAttributes); - final SplitableData splitData = new SplitByteBufferedData(); - - ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); + @SuppressWarnings("unchecked") + ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); - final Codec.BytesCodec[] codecs = shardingCodec.getCodecs(); - - final ShardIndex index = ShardIndex.createIndex(datasetAttributes); - long blocksOffset = index.getLocation() == ShardingCodec.IndexLocation.START ? index.numBytes() : 0; - - final SplitableData blocksSplitData = splitData.split(blocksOffset, Long.MAX_VALUE); - try (final OutputStream blocksOut = blocksSplitData.newOutputStream()) { - for (DataBlock block : getBlocks()) { - try (final CountingOutputStream blockOut = new CountingOutputStream(blocksOut)) { - arrayCodec.encode(block).writeTo(blockOut); - index.set(blocksOffset, blockOut.getByteCount(), getBlockPosition(block.getGridPosition())); - blocksOffset += blockOut.getByteCount(); + long blocksStartBytes = index.getLocation() == ShardingCodec.IndexLocation.START ? index.numBytes() : 0; + final AtomicLong blockOffset = new AtomicLong(blocksStartBytes); + + if (index.getLocation() == ShardingCodec.IndexLocation.END) { + return ReadData.from(out -> { + try (final CountingOutputStream countOut = new CountingOutputStream(out)) { + long prevCount = 0; + for (DataBlock block : getBlocks()) { + arrayCodec.encode(block).writeTo(countOut); + final int[] blockPosition = getBlockPosition(block.getGridPosition()); + final long curCount = countOut.getByteCount(); + final long blockWrittenSize = curCount - prevCount; + prevCount = curCount; + synchronized (index) { + index.set(blockOffset.getAndAdd(blockWrittenSize), blockWrittenSize, blockPosition); + } + } + } + synchronized (index) { + ShardIndex.write(out, index); + } + }); + } else { + final ArrayList blocksData = new ArrayList<>(); + for (DataBlock dataBlock : getBlocks()) { + ReadData readDataBlock = ReadData.from(out -> arrayCodec.encode(dataBlock).writeTo(out)); + blocksData.add(readDataBlock); + final long length = readDataBlock.length(); + synchronized (index) { + index.set(blockOffset.getAndAdd(length), length, getBlockPosition(dataBlock.getGridPosition())); } } - } catch (final IOException | UncheckedIOException e) { - throw new N5Exception.N5IOException( - "Failed to write block to shard " + Arrays.toString(getGridPosition()), e); - } - final long indexOffset = index.getLocation() == ShardingCodec.IndexLocation.START ? 0 : blocksOffset; - try { - ShardIndex.write(splitData.split(indexOffset, index.numBytes()), index); - } catch (final IOException | UncheckedIOException e) { - throw new N5Exception.N5IOException( - "Failed to write index to shard " + Arrays.toString(getGridPosition()), e); + return ReadData.from(out -> { + ShardIndex.write(out, index); + for (ReadData blockData : blocksData) { + blockData.writeTo(out); + } + }); } - return splitData.newInputStream(); } class DataBlockIterator implements Iterator> { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 6d2e5933a..a72ad4508 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -9,9 +9,9 @@ import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.SplitableData; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import java.io.ByteArrayInputStream; @@ -161,11 +161,11 @@ public static void read(InputStream in, final ShardIndex index) throws IOExcepti } public static boolean read( - final SplitableData indexData, + final ReadData indexData, final ShardIndex index ) { - try (final InputStream in = indexData.newInputStream()) { + try (final InputStream in = indexData.inputStream()) { read(in, index); return true; } catch (final N5Exception.N5NoSuchKeyException e) { @@ -176,12 +176,12 @@ public static boolean read( } public static void write( - final SplitableData indexData, + final OutputStream outputStream, final ShardIndex index ) throws IOException { - try (final OutputStream os = indexData.newOutputStream()) { - write(index, os); + try { + write(index, outputStream); } catch (final IOException | UncheckedIOException e) { throw new N5IOException("Failed to write shard index", e); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 825f5e121..b45781c78 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -9,17 +9,14 @@ import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.SplitableData; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.SplittableReadData; import org.janelia.saalfeldlab.n5.serialization.N5Annotations; import org.janelia.saalfeldlab.n5.serialization.NameConfig; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.reflect.Array; import java.lang.reflect.Type; import java.util.Objects; @@ -130,25 +127,10 @@ public DeterministicSizeCodec[] getIndexCodecs() { @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { - return getArrayCodec().decode(readData, gridPosition); - } - - public void writeBlock( - final SplitableData splitData, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) { - - final long[] shardPos = datasetAttributes.getShardPositionForBlock(dataBlock.getGridPosition()); - new VirtualShard(datasetAttributes, shardPos, splitData).writeBlock(dataBlock); - } - - public DataBlock readBlock( - final SplitableData splitData, - final DatasetAttributes datasetAttributes, - final long... gridPosition) { + final SplittableReadData splitableReadData = readData.materialize(); - final long[] shardPosition = datasetAttributes.getShardPositionForBlock(gridPosition); - return new VirtualShard(datasetAttributes, shardPosition, splitData).getBlock(gridPosition); + final VirtualShard shard = new VirtualShard<>(attributes, gridPosition, splitableReadData); + return shard.getBlock(gridPosition); } public ShardIndex createIndex(final DatasetAttributes attributes) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index a7e61fdbf..c651e02e8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -4,13 +4,11 @@ import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.SplitableData; -import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.SplittableReadData; import org.janelia.saalfeldlab.n5.util.GridIterator; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.ArrayList; @@ -22,15 +20,15 @@ public class VirtualShard extends AbstractShard { - private final SplitableData splitableData; + private final SplittableReadData shardData; public VirtualShard( final DatasetAttributes datasetAttributes, long[] gridPosition, - final SplitableData splitableData) { + final SplittableReadData shardData) { super(datasetAttributes, gridPosition, null); - this.splitableData = splitableData; + this.shardData = shardData; } @SuppressWarnings("unchecked") @@ -82,8 +80,9 @@ public List> getBlocks(final int[] blockIndexes) { final long numBytes = index.getNumBytesByBlockIndex((int)idx); //TODO Caleb: Do this with a single access (start at first offset, read through the last) - try (final InputStream in = splitableData.split(offset, numBytes).newInputStream()) { - final DataBlock block = getBlock(ReadData.from(in), position.clone()); + try { + final ReadData blockData = shardData.slice(offset, numBytes); + final DataBlock block = getBlock(blockData, position.clone()); blocks.add(block); } catch (final N5Exception.N5NoSuchKeyException e) { return blocks; @@ -109,8 +108,9 @@ public DataBlock getBlock(long... blockGridPosition) { final long blockSize = idx.getNumBytes(relativePosition); final long[] blockPosInImg = getDatasetAttributes().getBlockPositionFromShardPosition(getGridPosition(), blockGridPosition); - try (final InputStream in = splitableData.split(blockOffset, blockSize).newInputStream()) { - return getBlock(ReadData.from(in), blockPosInImg); + try { + final ReadData blockData = shardData.slice(blockOffset, blockSize); + return getBlock(blockData, blockPosInImg); } catch (final N5Exception.N5NoSuchKeyException e) { return null; } catch (final IOException | UncheckedIOException e) { @@ -118,122 +118,35 @@ public DataBlock getBlock(long... blockGridPosition) { } } - @Override - public void writeBlock(final DataBlock block) { - - final int[] relativePosition = getBlockPosition(block.getGridPosition()); - if (relativePosition == null) - throw new N5IOException("Attempted to write block in the wrong shard."); - - final ShardIndex index = getIndex(); - final long blockOffset; - - //TODO Caleb: is it safe to assume we can ALWAYS write at an offset into a value that doesn't exist yet? - // Files seem to fill the beginning with 0, not sure how backends handle this. - // Otherwise, may need to explicitly write an empty index first. - if (index.getLocation() == ShardingCodec.IndexLocation.START) - blockOffset = splitableData.getSize() == 0 ? index.numBytes() : splitableData.getSize(); - else - blockOffset = splitableData.getSize(); - - final SplitableData blockData = splitableData.split(blockOffset, Long.MAX_VALUE - blockOffset); //TODO Caleb: Should ideally remove offset also, but would need it to be absolute - - final long sizeWritten; - try (final OutputStream blockOut = blockData.newOutputStream()) { - try (final CountingOutputStream out = new CountingOutputStream(blockOut)) { - writeBlock(out, datasetAttributes, block); - - /* Update and write the index to the shard*/ - sizeWritten = out.getNumBytes(); - index.set(blockOffset, sizeWritten, relativePosition); - } - } catch (IOException e) { - throw new N5IOException("Failed to write block to shard ", e); - } - - //TODO Caleb: Could do END in the blockOut block, to avoid an additional access - final long indexOffset = index.getLocation() == ShardingCodec.IndexLocation.START ? 0 : blockOffset + sizeWritten; - - - final SplitableData indexData = splitableData.split( indexOffset, index.numBytes()); - try { - ShardIndex.write(indexData, index); - } catch (IOException e) { - throw new N5IOException("Failed to write index to shard ", e); - } - - } - - void writeBlock( - final OutputStream out, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws IOException { - - ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); - shardingCodec.getArrayCodec().encode(dataBlock); - } - public ShardIndex createIndex() { // Empty index of the correct size - return ((ShardingCodec)getDatasetAttributes().getArrayCodec()).createIndex(getDatasetAttributes()); + return ((ShardingCodec)getDatasetAttributes().getArrayCodec()).createIndex(getDatasetAttributes()); } @Override public ShardIndex getIndex() { - //TODO Caleb: How to handle whne this shard doesn't exist (splitableData.getSize() <= 0) + //TODO Caleb: How to handle when this shard doesn't exist (splitableData.getSize() <= 0) index = createIndex(); - final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, splitableData.getSize()); - ShardIndex.read(splitableData.split(bounds.start, index.numBytes()), index); - return index; - } - - static class CountingOutputStream extends OutputStream { - private final OutputStream out; - private long numBytes; - - public CountingOutputStream(OutputStream out) { - - this.out = out; - this.numBytes = 0; - } - - @Override - public void write(int b) throws IOException { - - out.write(b); - numBytes++; - } - - @Override - public void write(byte[] b) throws IOException { - - out.write(b); - numBytes += b.length; - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - - out.write(b, off, len); - numBytes += len; - } - - @Override - public void flush() throws IOException { - - out.flush(); - } - - @Override - public void close() throws IOException { - - } - - public long getNumBytes() { + final ReadData indexData; + try { + /* we require a length, so materialize if we don't have one. */ + if (shardData.length() == -1) { + shardData.materialize(); + } + final long length = shardData.length(); + if (length == -1) + throw new N5IOException("ReadData for shard index must have a valid length, but was " + length); - return numBytes; + final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, length); + indexData = shardData.slice(bounds.start, index.numBytes()); + } catch (N5Exception.N5NoSuchKeyException e) { + return null; + } catch (IOException | UncheckedIOException e) { + throw new N5IOException(e); } + ShardIndex.read(indexData, index); + return index; } } From 2cc81aab194c64dd9455efc675b5e1bca8da0f14 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 May 2025 09:37:33 -0400 Subject: [PATCH 220/423] refactor(test): for sharding --- .../saalfeldlab/n5/demo/BlockIterators.java | 2 +- .../saalfeldlab/n5/shard/ShardIndexTest.java | 48 ++++++++++++------- .../n5/shard/ShardPropertiesTests.java | 7 +-- .../saalfeldlab/n5/shard/ShardTest.java | 8 ++-- 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java index 4bcca0bc3..091faf615 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java @@ -35,7 +35,7 @@ public static void shardBlockIterator() { new int[] {2, 2}, // block size DataType.UINT8, new Codec[] { new N5BlockCodec<>() }, - new DeterministicSizeCodec[] { new N5BlockCodec<>() }, + new DeterministicSizeCodec[] { new RawBytes<>() }, IndexLocation.END); shardPositions(attrs) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index c1cb8b09f..ee34a5771 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -1,12 +1,10 @@ package org.janelia.saalfeldlab.n5.shard; import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5FSTest; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; -import org.janelia.saalfeldlab.n5.SplitKeyValueAccessData; -import org.janelia.saalfeldlab.n5.SplitableData; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; @@ -15,6 +13,8 @@ import org.junit.Test; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Paths; import static org.junit.Assert.assertEquals; @@ -78,15 +78,23 @@ public void testReadVirtual() throws IOException { index.set(19, 32, new int[]{1, 0}); index.set(93, 111, new int[]{3, 0}); index.set(143, 1, new int[]{1, 2}); - final SplitKeyValueAccessData splitableData = new SplitKeyValueAccessData(kva, path); - final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, splitableData.getSize()); - final SplitableData indexData = splitableData.split(bounds.start, index.numBytes()); - ShardIndex.write(indexData, index); - final ShardIndex other = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); - ShardIndex.read(indexData, other); + final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, kva.size(path)); + try ( + final LockedChannel channel = kva.lockForWriting(path, bounds.start, index.numBytes()); + final OutputStream out = channel.newOutputStream() + ) { + ShardIndex.write(index, out); + } - assertEquals(index, other); + final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); + try ( + final LockedChannel channel = kva.lockForReading(path, bounds.start, index.numBytes()); + final InputStream in = channel.newInputStream() + ) { + ShardIndex.read(in, indexRead); + } + assertEquals(index, indexRead); } @Test @@ -107,14 +115,22 @@ public void testReadInMemory() throws IOException { index.set(19, 32, new int[]{1, 0}); index.set(93, 111, new int[]{3, 0}); index.set(143, 1, new int[]{1, 2}); - SplitKeyValueAccessData splitableData = new SplitKeyValueAccessData(kva, path); - final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, splitableData.getSize()); - final SplitableData indexData = splitableData.split(bounds.start, index.numBytes()); - ShardIndex.write(indexData, index); - final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); - ShardIndex.read(indexData, indexRead); + final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, kva.size(path)); + try ( + final LockedChannel channel = kva.lockForWriting(path, bounds.start, index.numBytes()); + final OutputStream out = channel.newOutputStream() + ) { + ShardIndex.write(index, out); + } + final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); + try ( + final LockedChannel channel = kva.lockForReading(path, bounds.start, index.numBytes()); + final InputStream in = channel.newInputStream() + ) { + ShardIndex.read(in, indexRead); + } assertEquals(index, indexRead); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java index 886736c24..aaeed0347 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java @@ -5,6 +5,7 @@ import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; +import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.Position; import org.junit.Test; @@ -35,7 +36,7 @@ public void testShardProperties() throws Exception { new ShardingCodec( blkSize, new Codec[]{new N5BlockCodec()}, - new DeterministicSizeCodec[]{}, + new DeterministicSizeCodec[]{new RawBytes()}, IndexLocation.END ) ); @@ -74,7 +75,7 @@ public void testShardBlockPositionIterator() throws Exception { new ShardingCodec( blkSize, new Codec[]{ new N5BlockCodec() }, - new DeterministicSizeCodec[]{}, + new DeterministicSizeCodec[]{new RawBytes()}, IndexLocation.END ) ); @@ -111,7 +112,7 @@ public void testShardGrouping() { new ShardingCodec( blkSize, new Codec[]{ new N5BlockCodec() }, - new DeterministicSizeCodec[]{}, + new DeterministicSizeCodec[]{new RawBytes<>()}, IndexLocation.END ) ); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index a4681b8ab..2768ceada 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -56,13 +56,13 @@ public static Collection data() { final ArrayList params = new ArrayList<>(); for (IndexLocation indexLoc : IndexLocation.values()) { - for (ByteOrder blockByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { - for (ByteOrder indexByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { + for (ByteOrder blockByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN}) { + for (ByteOrder indexByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN}) { params.add(new Object[]{indexLoc, blockByteOrder, indexByteOrder}); } } } - final int numParams = LOCAL_DEBUG ? 1 : params.size(); + final int numParams = params.size(); final Object[][] paramArray = new Object[numParams][]; Arrays.setAll(paramArray, params::get); return Arrays.asList(paramArray); @@ -93,7 +93,7 @@ private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, new ShardingCodec<>( blockSize, new Codec[]{new N5BlockCodec<>()}, //, new GzipCompression(4)}, - new DeterministicSizeCodec[]{new N5BlockCodec<>(), new Crc32cChecksumCodec()}, + new DeterministicSizeCodec[]{new RawBytes<>(), new Crc32cChecksumCodec()}, indexLocation ) ); From 1c5f0c1165010b5febac7181c651b91b6a2f5dae Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 May 2025 09:37:48 -0400 Subject: [PATCH 221/423] chore: feedback --- .../java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java | 1 + src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java index b22d358be..abe2e3ab3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java @@ -38,6 +38,7 @@ * @param * type of the data contained in the DataBlock */ +//TODO Caleb: replace with ArrayCodec? Or use for Factory naming? public interface DataBlockCodec { ReadData encode(DataBlock dataBlock) throws IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 7e91827f0..0ecbb1552 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -62,6 +62,7 @@ public interface ReadData { * @throws IOException * if an I/O error occurs while trying to get the length */ + //TODO N5IOException default long length() throws IOException { return -1; } From c27421382b60b4e4cccc19596acf69019556845e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 May 2025 10:13:52 -0400 Subject: [PATCH 222/423] fix: write ShardIndex before closing output stream fix(test): shardIndex writeRead --- .../janelia/saalfeldlab/n5/shard/Shard.java | 6 +- .../saalfeldlab/n5/shard/ShardIndexTest.java | 48 +++---------- .../saalfeldlab/n5/shard/ShardTest.java | 70 +++++++++---------- 3 files changed, 47 insertions(+), 77 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index b4ddc0476..abf7c5213 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -168,9 +168,9 @@ default ReadData createReadData() throws IOException { index.set(blockOffset.getAndAdd(blockWrittenSize), blockWrittenSize, blockPosition); } } - } - synchronized (index) { - ShardIndex.write(out, index); + synchronized (index) { + ShardIndex.write(out, index); + } } }); } else { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index ee34a5771..f2637126f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -2,6 +2,7 @@ import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5FSTest; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; @@ -61,44 +62,7 @@ public void testOffsetIndex() { } @Test - public void testReadVirtual() throws IOException { - - final N5KeyValueWriter writer = (N5KeyValueWriter)tempN5Factory.createTempN5Writer(); - final KeyValueAccess kva = writer.getKeyValueAccess(); - - final int[] shardBlockGridSize = new int[]{6, 5}; - final IndexLocation indexLocation = IndexLocation.END; - final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[] { new RawBytes<>(), - new Crc32cChecksumCodec()}; - - final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "0").toString(); - - final ShardIndex index = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); - index.set(0, 6, new int[]{0, 0}); - index.set(19, 32, new int[]{1, 0}); - index.set(93, 111, new int[]{3, 0}); - index.set(143, 1, new int[]{1, 2}); - - final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, kva.size(path)); - try ( - final LockedChannel channel = kva.lockForWriting(path, bounds.start, index.numBytes()); - final OutputStream out = channel.newOutputStream() - ) { - ShardIndex.write(index, out); - } - - final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); - try ( - final LockedChannel channel = kva.lockForReading(path, bounds.start, index.numBytes()); - final InputStream in = channel.newInputStream() - ) { - ShardIndex.read(in, indexRead); - } - assertEquals(index, indexRead); - } - - @Test - public void testReadInMemory() throws IOException { + public void writeReadTest() throws IOException { final N5KeyValueWriter writer = (N5KeyValueWriter)tempN5Factory.createTempN5Writer(); final KeyValueAccess kva = writer.getKeyValueAccess(); @@ -116,7 +80,13 @@ public void testReadInMemory() throws IOException { index.set(93, 111, new int[]{3, 0}); index.set(143, 1, new int[]{1, 2}); - final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, kva.size(path)); + long currentSize; + try { + currentSize = kva.size(path); + } catch (N5Exception.N5NoSuchKeyException e) { + currentSize = -1; + } + final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, currentSize); try ( final LockedChannel channel = kva.lockForWriting(path, bounds.start, index.numBytes()); final OutputStream out = channel.newOutputStream() diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 2768ceada..72864f2ed 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -221,29 +221,29 @@ public void writeReadBlockTest() { final HashMap writtenBlocks = new HashMap<>(); -// for (int idx1 = 1; idx1 >= 0; idx1--) { -// for (int idx2 = 1; idx2 >= 0; idx2--) { -// final long[] gridPosition = {idx1, idx2}; -// final DataBlock dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); -// byte[] data = (byte[])dataBlock.getData(); -// for (int i = 0; i < data.length; i++) { -// data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); -// } -// writer.writeBlock(dataset, datasetAttributes, dataBlock); -// -// final DataBlock block = writer.readBlock(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); -// Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); -// -// for (Map.Entry entry : writtenBlocks.entrySet()) { -// final long[] otherGridPosition = entry.getKey(); -// final byte[] otherData = entry.getValue(); -// final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); -// Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); -// } -// -// writtenBlocks.put(gridPosition, data); -// } -// } + for (int idx1 = 1; idx1 >= 0; idx1--) { + for (int idx2 = 1; idx2 >= 0; idx2--) { + final long[] gridPosition = {idx1, idx2}; + final DataBlock dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); + byte[] data = (byte[])dataBlock.getData(); + for (int i = 0; i < data.length; i++) { + data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); + } + writer.writeBlock(dataset, datasetAttributes, dataBlock); + + final DataBlock block = writer.readBlock(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + + for (Map.Entry entry : writtenBlocks.entrySet()) { + final long[] otherGridPosition = entry.getKey(); + final byte[] otherData = entry.getValue(); + final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); + Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); + } + + writtenBlocks.put(gridPosition, data); + } + } } @Test @@ -265,18 +265,18 @@ public void writeReadShardTest() { final InMemoryShard shard = new InMemoryShard<>(datasetAttributes, new long[]{0, 0}); -// for (int idx1 = 1; idx1 >= 0; idx1--) { -// for (int idx2 = 1; idx2 >= 0; idx2--) { -// final long[] gridPosition = {idx1, idx2}; -// final DataBlock dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); -// byte[] data = (byte[])dataBlock.getData(); -// for (int i = 0; i < data.length; i++) { -// data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); -// } -// shard.addBlock((DataBlock)dataBlock); -// writtenBlocks.put(gridPosition, data); -// } -// } + for (int idx1 = 1; idx1 >= 0; idx1--) { + for (int idx2 = 1; idx2 >= 0; idx2--) { + final long[] gridPosition = {idx1, idx2}; + final DataBlock dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); + byte[] data = (byte[])dataBlock.getData(); + for (int i = 0; i < data.length; i++) { + data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); + } + shard.addBlock((DataBlock)dataBlock); + writtenBlocks.put(gridPosition, data); + } + } writer.writeShard(dataset, datasetAttributes, shard); From 3486b0f47ee425a70860044c4ce88d9b326ebb26 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 23 May 2025 10:20:29 -0400 Subject: [PATCH 223/423] wip: FileSplittableReadData now wraps a kva --- .../n5/FileSystemKeyValueAccess.java | 17 +--------- .../saalfeldlab/n5/KeyValueAccess.java | 17 ++++++++++ .../n5/readdata/FileSplittableReadData.java | 33 +++++++++++-------- .../saalfeldlab/n5/shard/ShardTest.java | 2 +- 4 files changed, 39 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index c8d29d611..a7a947091 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -249,22 +249,7 @@ public LockedFileChannel lockForReading(final String normalPath, final long star @Override public FileSplittableReadData createReadData(final String normalPath) { - return new FileSplittableReadData(Paths.get(normalPath), 0, -1); - } - - public ReadData read(final String normalPath, final long startByte, final long length) throws IOException { - final FileChannel channel = FileChannel.open(fileSystem.getPath(normalPath), new OpenOption[]{StandardOpenOption.READ}); - channel.position(startByte); - - final int sz = (int)(length < 0 ? channel.size() : (int)length); - final byte[] data = new byte[sz]; - final ByteBuffer buf = ByteBuffer.wrap(data); - channel.read(buf); - return ReadData.from(data); - } - - public ReadData readFully(final String normalPath) throws IOException { - return read(normalPath, 0, -1); + return new FileSplittableReadData(this, normalPath, 0, -1); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 4f058c3c6..b4d34adc4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -62,6 +62,8 @@ import java.util.Arrays; import java.util.stream.Collectors; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + /** * Key value read primitives used by {@link N5KeyValueReader} * implementations. This interface implements a subset of access primitives @@ -254,6 +256,21 @@ default URI uri(final String uriString) throws URISyntaxException { */ public boolean isFile(String normalPath); // TODO: Looks un-used. Remove? + /** + * Create a {@link ReadData} through which data at the normal key can be read. + *

    + * Implementations should read lazily if possible. Consumers may call {@link ReadData#materialize()} to force + * a read operation if needed. + *

    + * Partial reads are possible using {@link ReadData#slice()} on the output, if supported by this KeyValueAccess + * implementation. + * + * @param normalKey is expected to be in normalized form, no further efforts are made to normalize it + * @param startByte the starting byte + * @param length the number of bytes to read + * @return a materialized Read data + * @throws IOException if an error occurs + */ default ReadData createReadData(final String normalPath) throws IOException { return ReadData.from(this, normalPath); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java index 8382b0b59..d08b8956e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java @@ -5,30 +5,32 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.NoSuchFileException; -import java.nio.file.OpenOption; -import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; public class FileSplittableReadData implements SplittableReadData { private ByteArraySplittableReadData materialized; - private final Path path; + private final KeyValueAccess kva; + private final String normalKey; private final long offset; - private final long length; + private long length; - public FileSplittableReadData(Path path, long offset, long length) { - this.path = path; + public FileSplittableReadData(KeyValueAccess kva, String normalKey, long offset, long length) { + this.kva = kva; + this.normalKey = normalKey; this.offset = offset; this.length = length; } - public FileSplittableReadData(Path path, long offset) { - this(path, offset, -1); + public FileSplittableReadData(KeyValueAccess kva, String normalKey, long offset) { + this(kva, normalKey, offset, -1); } @Override @@ -36,12 +38,17 @@ public long length() { if (materialized != null) return materialized.length(); + if (length < 0) { + try { + length = kva.size(normalKey); + } catch (IOException e) {} + } return length; } @Override public InputStream inputStream() throws IOException, IllegalStateException { - return materialize().inputStream(); + return materialize().inputStream(); } @Override @@ -61,7 +68,7 @@ private void read() throws IOException { final FileChannel channel; try { - channel = FileChannel.open(path, StandardOpenOption.READ); + channel = FileChannel.open(Paths.get(normalKey), StandardOpenOption.READ); } catch (final NoSuchFileException e) { throw new N5Exception.N5NoSuchKeyException(e); } @@ -83,7 +90,7 @@ public ReadData slice(long offset, long length) throws IOException { if (materialized != null) return materialize().slice(offset, length); - return new FileSplittableReadData(path, this.offset + offset, length); + return new FileSplittableReadData(kva, normalKey, this.offset + offset, length); } @Override @@ -99,8 +106,8 @@ public Pair split(long pivot) throws IOException { final long lenR = this.length - pivot; return new ImmutablePair( - new FileSplittableReadData(path, offsetL, lenL), - new FileSplittableReadData(path, offsetR, lenR)); + new FileSplittableReadData(kva, normalKey, offsetL, lenL), + new FileSplittableReadData(kva, normalKey, offsetR, lenR)); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 2768ceada..0b3f2a11b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -93,7 +93,7 @@ private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, new ShardingCodec<>( blockSize, new Codec[]{new N5BlockCodec<>()}, //, new GzipCompression(4)}, - new DeterministicSizeCodec[]{new RawBytes<>(), new Crc32cChecksumCodec()}, + new DeterministicSizeCodec[]{new N5BlockCodec<>(), new Crc32cChecksumCodec()}, indexLocation ) ); From 11cc2e2901968eb4269741beaf6d5f32d9968068 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 23 May 2025 10:21:22 -0400 Subject: [PATCH 224/423] rm ReadDataBenchmarksKvaReadFully --- .../ReadDataBenchmarksKvaReadFully.java | 43 ------------------- 1 file changed, 43 deletions(-) delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadFully.java diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadFully.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadFully.java deleted file mode 100644 index f526f83da..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadFully.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.janelia.saalfeldlab.n5.benchmarks; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; - -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.runner.Runner; -import org.openjdk.jmh.runner.RunnerException; -import org.openjdk.jmh.runner.options.Options; -import org.openjdk.jmh.runner.options.OptionsBuilder; - -@State(Scope.Benchmark) -@Warmup(iterations = 10, time = 100, timeUnit = TimeUnit.MICROSECONDS) -@Measurement(iterations = 100, time = 100, timeUnit = TimeUnit.MICROSECONDS) -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.MICROSECONDS) -@Fork(1) -public class ReadDataBenchmarksKvaReadFully extends ReadDataBenchmarks { - - public static void main(String... args) throws RunnerException { - - final Options options = new OptionsBuilder().include(ReadDataBenchmarksKvaReadFully.class.getSimpleName() + "\\.") - .build(); - - new Runner(options).run(); - } - - public ReadData read() throws IOException { - - return ((FileSystemKeyValueAccess)kva).readFully(getPath().toString()); - } - -} \ No newline at end of file From 6edda5f01d7a9cd060543db3de049fd2e8b6bcac Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 23 May 2025 10:35:57 -0400 Subject: [PATCH 225/423] test: FileSplittableReadData knows its length --- .../java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index 68bf3a778..64aa60ae1 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -147,7 +147,7 @@ public void testFileSplittableReadData() throws IOException { final FileSplittableReadData readData = new FileSystemKeyValueAccess(FileSystems.getDefault()) .createReadData(tmpF.getAbsolutePath()); - assertEquals("file read data length unknown", -1, readData.length()); + assertEquals("file read data length", 128, readData.length()); splittableReadDataTestHelper(readData.materialize(), N, 5); } From 521b6aad987ace8866eab06af4074bd5d4318c36 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 May 2025 10:49:05 -0400 Subject: [PATCH 226/423] feat,BREAKING: wrap IO as N5IO and rethrow; don't throw IOException in KVA for size --- .../saalfeldlab/n5/FileSystemKeyValueAccess.java | 5 ++++- .../janelia/saalfeldlab/n5/HttpKeyValueAccess.java | 2 +- .../org/janelia/saalfeldlab/n5/KeyValueAccess.java | 2 +- .../n5/readdata/FileSplittableReadData.java | 11 +++++++---- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index a7a947091..3f6fbc070 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -62,6 +62,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.Reader; +import java.io.UncheckedIOException; import java.io.Writer; import java.net.URI; import java.net.URISyntaxException; @@ -301,12 +302,14 @@ public boolean exists(final String normalPath) { } @Override - public long size(final String normalPath) throws IOException { + public long size(final String normalPath) { try { return Files.size(fileSystem.getPath(normalPath)); } catch (NoSuchFileException e) { throw new N5Exception.N5NoSuchKeyException("No such file", e); + } catch (IOException | UncheckedIOException e) { + throw new N5Exception.N5IOException(e); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index 7a95fbcf9..975731105 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -127,7 +127,7 @@ public boolean exists(final String normalPath) { } } - @Override public long size(String normalPath) throws IOException { + @Override public long size(String normalPath) { final HttpURLConnection head = requireValidHttpResponse(normalPath, "HEAD", "Error checking existence: " + normalPath, true); return head.getContentLengthLong(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index b4d34adc4..fe168b494 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -234,7 +234,7 @@ default URI uri(final String uriString) throws URISyntaxException { */ public boolean exists(final String normalPath); - public long size(final String normalPath) throws IOException; + public long size(final String normalPath); /** * Test whether the path is a directory. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java index d08b8956e..34e58c9bf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java @@ -2,9 +2,11 @@ import java.io.IOException; import java.io.InputStream; +import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; @@ -39,9 +41,7 @@ public long length() { return materialized.length(); if (length < 0) { - try { - length = kva.size(normalKey); - } catch (IOException e) {} + length = kva.size(normalKey); } return length; } @@ -68,9 +68,12 @@ private void read() throws IOException { final FileChannel channel; try { - channel = FileChannel.open(Paths.get(normalKey), StandardOpenOption.READ); + Path path = Paths.get(kva.uri(normalKey)); + channel = FileChannel.open(path, StandardOpenOption.READ); } catch (final NoSuchFileException e) { throw new N5Exception.N5NoSuchKeyException(e); + } catch (URISyntaxException e) { + throw new N5Exception(e); } channel.position(offset); From 3242dc716e63f950e657c8ff7f866f524f7f99b8 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 May 2025 10:49:22 -0400 Subject: [PATCH 227/423] fix(test): index should have RawBytesCodec --- src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index f2eb1508a..72864f2ed 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -93,7 +93,7 @@ private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, new ShardingCodec<>( blockSize, new Codec[]{new N5BlockCodec<>()}, //, new GzipCompression(4)}, - new DeterministicSizeCodec[]{new N5BlockCodec<>(), new Crc32cChecksumCodec()}, + new DeterministicSizeCodec[]{new RawBytes<>(), new Crc32cChecksumCodec()}, indexLocation ) ); From a56c80b1ef930a50f04ef56e623374225a91bd22 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 23 May 2025 14:38:53 -0400 Subject: [PATCH 228/423] wip: refactor KvaSplittableReadData * add abstract KeyValueAccessSplittableReadData * ReadData.length does now throw IOException --- .../n5/FileSystemKeyValueAccess.java | 66 ++++++---- .../saalfeldlab/n5/HttpKeyValueAccess.java | 32 +++-- .../saalfeldlab/n5/KeyValueAccess.java | 41 +++---- .../n5/KeyValueAccessSplittableReadData.java | 95 ++++++++++++++ .../n5/readdata/FileSplittableReadData.java | 116 ------------------ .../saalfeldlab/n5/readdata/LazyReadData.java | 10 +- .../saalfeldlab/n5/readdata/ReadData.java | 2 +- .../n5/readdata/ReadDataTests.java | 7 +- 8 files changed, 184 insertions(+), 185 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 3f6fbc070..ab902b99a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -26,36 +26,11 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import org.apache.commons.io.input.BoundedInputStream; -import org.janelia.saalfeldlab.n5.readdata.FileSplittableReadData; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.SplittableReadData; import java.io.File; import java.io.IOException; @@ -249,7 +224,7 @@ public LockedFileChannel lockForReading(final String normalPath, final long star } @Override - public FileSplittableReadData createReadData(final String normalPath) { + public SplittableReadData createReadData(final String normalPath) { return new FileSplittableReadData(this, normalPath, 0, -1); } @@ -622,4 +597,41 @@ protected static void createAndCheckIsDirectory( throw x; } } + + private class FileSplittableReadData extends KeyValueAccessSplittableReadData { + + public FileSplittableReadData(FileSystemKeyValueAccess kva, String normalKey, long offset, long length) { + super(kva, normalKey, offset, length); + } + + @Override + void read() throws IOException { + + final FileChannel channel; + try { + Path path = Paths.get(kva.uri(normalKey)); + channel = FileChannel.open(path, StandardOpenOption.READ); + } catch (final NoSuchFileException e) { + throw new N5Exception.N5NoSuchKeyException(e); + } catch (URISyntaxException e) { + throw new N5Exception(e); + } + channel.position(offset); + + if (length > Integer.MAX_VALUE) + throw new IOException("Attempt to materialize too large data"); + + final int sz = (int)(length < 0 ? channel.size() : (int)length); + final byte[] data = new byte[sz]; + final ByteBuffer buf = ByteBuffer.wrap(data); + channel.read(buf); + materialized = (SplittableReadData)ReadData.from(data); + } + + @Override + KeyValueAccessSplittableReadData readOperationSlice(long offset, long length) throws IOException { + return new FileSplittableReadData(kva, normalKey, offset, length); + } + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index 975731105..45d536aef 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -221,6 +221,11 @@ private HttpURLConnection httpRequest(String normalPath, String method) throws I return connection; } + @Override + public HttpSplittableReadData createReadData(final String normalPath) { + return new HttpSplittableReadData(this, normalPath, 0, -1); + } + @Override public LockedChannel lockForReading(final String normalPath) throws IOException { return lockForReading(normalPath, 0, -1); @@ -352,14 +357,6 @@ private class HttpObjectChannel implements LockedChannel { private final long size; private final ArrayList resources = new ArrayList<>(); - protected HttpObjectChannel(final URI uri) { - this(uri, 0, -1); - } - - protected HttpObjectChannel(final URI uri, int size) { - this(uri, 0, size); - } - protected HttpObjectChannel(final URI uri, long startByte, long size) { this.uri = uri; @@ -432,4 +429,23 @@ public void close() throws IOException { } } + private class HttpSplittableReadData extends KeyValueAccessSplittableReadData { + + public HttpSplittableReadData(HttpKeyValueAccess kva, String normalKey, long offset, long length) { + super(kva, normalKey, offset, length); + } + + @Override + void read() throws IOException { + try( final HttpObjectChannel ch = new HttpObjectChannel(kva.uri(normalKey), offset, length) ) { + materialized = ReadData.from(ch.newInputStream()).materialize(); + } catch (URISyntaxException e) {} + } + + @Override + KeyValueAccessSplittableReadData readOperationSlice(long offset, long length) throws IOException { + return new HttpSplittableReadData(kva, normalKey, offset, length); + } + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index fe168b494..a8f6e86f7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -26,43 +26,21 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.SplittableReadData; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.FileSystem; import java.util.Arrays; import java.util.stream.Collectors; -import org.janelia.saalfeldlab.n5.readdata.ReadData; /** * Key value read primitives used by {@link N5KeyValueReader} @@ -335,6 +313,15 @@ public LockedChannel lockForWriting(String normalPath, final long startByte, fin * List all children of a path. * * @param normalPath + + Latest + New (30) + Unread (239) + Top + Categories + +Topic list, column headers with buttons are sortable. + * is expected to be in normalized form, no further * efforts are made to normalize it. * @return the the child paths @@ -366,5 +353,5 @@ public LockedChannel lockForWriting(String normalPath, final long startByte, fin * if an error occurs during deletion */ public void delete(final String normalPath) throws IOException; - + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java new file mode 100644 index 000000000..56bf5351a --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java @@ -0,0 +1,95 @@ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.SplittableReadData; + +abstract class KeyValueAccessSplittableReadData implements SplittableReadData { + + protected SplittableReadData materialized; + + protected final K kva; + protected final String normalKey; + protected final long offset; + protected long length; + + public KeyValueAccessSplittableReadData(K kva, String normalKey, long offset, long length) { + + this.kva = kva; + this.normalKey = normalKey; + this.offset = offset; + this.length = length; + } + + public KeyValueAccessSplittableReadData(K kva, String normalKey, long offset) { + + this(kva, normalKey, offset, -1); + } + + @Override + public long length() { + + if (materialized != null) + return materialized.length(); + + if (length < 0) { + length = kva.size(normalKey); + } + return length; + } + + @Override + public InputStream inputStream() throws IOException, IllegalStateException { + + return materialize().inputStream(); + } + + @Override + public byte[] allBytes() throws IOException, IllegalStateException { + + return materialize().allBytes(); + } + + @Override + public SplittableReadData materialize() throws IOException { + + if (materialized == null) + read(); + + return (SplittableReadData)materialized; + } + + abstract void read() throws IOException; + + abstract KeyValueAccessSplittableReadData readOperationSlice(long offset, long length) throws IOException; + + @Override + public ReadData slice(long offset, long length) throws IOException { + + if (materialized != null) + return materialize().slice(offset, length); + + return readOperationSlice(this.offset + offset, length); + } + + @Override + public Pair split(long pivot) throws IOException { + + if (materialized != null) + return materialize().split(pivot); + + final long offsetL = 0; + final long lenL = pivot; + + final long offsetR = offset + pivot; + final long lenR = this.length - pivot; + + return new ImmutablePair( + readOperationSlice(offsetL, lenL), + readOperationSlice(offsetR, lenR)); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java deleted file mode 100644 index 34e58c9bf..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/FileSplittableReadData.java +++ /dev/null @@ -1,116 +0,0 @@ -package org.janelia.saalfeldlab.n5.readdata; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URISyntaxException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; - -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; -import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.N5Exception; - -public class FileSplittableReadData implements SplittableReadData { - - private ByteArraySplittableReadData materialized; - - private final KeyValueAccess kva; - private final String normalKey; - private final long offset; - private long length; - - public FileSplittableReadData(KeyValueAccess kva, String normalKey, long offset, long length) { - this.kva = kva; - this.normalKey = normalKey; - this.offset = offset; - this.length = length; - } - - public FileSplittableReadData(KeyValueAccess kva, String normalKey, long offset) { - this(kva, normalKey, offset, -1); - } - - @Override - public long length() { - if (materialized != null) - return materialized.length(); - - if (length < 0) { - length = kva.size(normalKey); - } - return length; - } - - @Override - public InputStream inputStream() throws IOException, IllegalStateException { - return materialize().inputStream(); - } - - @Override - public byte[] allBytes() throws IOException, IllegalStateException { - return materialize().allBytes(); - } - - @Override - public SplittableReadData materialize() throws IOException { - if (materialized == null) - read(); - - return materialized; - } - - private void read() throws IOException { - - final FileChannel channel; - try { - Path path = Paths.get(kva.uri(normalKey)); - channel = FileChannel.open(path, StandardOpenOption.READ); - } catch (final NoSuchFileException e) { - throw new N5Exception.N5NoSuchKeyException(e); - } catch (URISyntaxException e) { - throw new N5Exception(e); - } - channel.position(offset); - - if (length > Integer.MAX_VALUE) - throw new IOException("Attempt to materialize too large data"); - - final int sz = (int)(length < 0 ? channel.size() : (int)length); - final byte[] data = new byte[sz]; - final ByteBuffer buf = ByteBuffer.wrap(data); - channel.read(buf); - materialized = new ByteArraySplittableReadData(data); - } - - @Override - public ReadData slice(long offset, long length) throws IOException { - - if (materialized != null) - return materialize().slice(offset, length); - - return new FileSplittableReadData(kva, normalKey, this.offset + offset, length); - } - - @Override - public Pair split(long pivot) throws IOException { - - if (materialized != null) - return materialize().split(pivot); - - final long offsetL = 0; - final long lenL = pivot; - - final long offsetR = offset + pivot; - final long lenR = this.length - pivot; - - return new ImmutablePair( - new FileSplittableReadData(kva, normalKey, offsetL, lenL), - new FileSplittableReadData(kva, normalKey, offsetR, lenR)); - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java index 7d2f61be1..0420ee4a7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -33,6 +33,7 @@ import java.io.InputStream; import java.io.OutputStream; import org.apache.commons.io.output.ProxyOutputStream; +import org.janelia.saalfeldlab.n5.N5Exception; class LazyReadData implements ReadData { @@ -72,8 +73,13 @@ public SplittableReadData materialize() throws IOException { } @Override - public long length() throws IOException { - return materialize().length(); + public long length() { + + try { + return materialize().length(); + } catch (IOException e) { + throw new N5Exception.N5IOException(e); + } } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 0ecbb1552..1b5079d8c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -63,7 +63,7 @@ public interface ReadData { * if an I/O error occurs while trying to get the length */ //TODO N5IOException - default long length() throws IOException { + default long length() { return -1; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index 64aa60ae1..58c8113af 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -131,7 +131,7 @@ public int read() throws IOException { } @Test - public void testFileSplittableReadData() throws IOException { + public void testFileKvaReadData() throws IOException { int N = 128; byte[] data = new byte[N]; @@ -142,15 +142,14 @@ public void testFileSplittableReadData() throws IOException { tmpF.deleteOnExit(); try (FileOutputStream os = new FileOutputStream(tmpF)) { os.write(data); - os.close(); } - final FileSplittableReadData readData = new FileSystemKeyValueAccess(FileSystems.getDefault()) + final SplittableReadData readData = new FileSystemKeyValueAccess(FileSystems.getDefault()) .createReadData(tmpF.getAbsolutePath()); assertEquals("file read data length", 128, readData.length()); splittableReadDataTestHelper(readData.materialize(), N, 5); } - + private class ByteFun implements OutputStreamOperator { IntUnaryOperator fun; From 15737489dd17dd631801b48240fd45be27159856 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 May 2025 16:59:16 -0400 Subject: [PATCH 229/423] fix: N5BlockCodec adds header size to encoded size --- .../saalfeldlab/n5/codec/N5BlockCodec.java | 10 +++++ .../saalfeldlab/n5/codec/N5Codecs.java | 14 +++++++ .../saalfeldlab/n5/shard/ShardIndex.java | 17 ++++++-- .../saalfeldlab/n5/shard/ShardIndexTest.java | 9 +++-- .../saalfeldlab/n5/shard/ShardTest.java | 39 +++++-------------- 5 files changed, 53 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java index 457b6d23f..a9d28397e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java @@ -16,12 +16,15 @@ public class N5BlockCodec implements Codec.ArrayCodec { private DataBlockCodec dataBlockCodec; + private DatasetAttributes attributes; + //TODO Caleb: Extract to factory that returns N5BlockCodec wrapper for datablockCodec @Override public void initialize(final DatasetAttributes attributes, final Codec.BytesCodec[] byteCodecs) { /*TODO: Consider an attributes.createDataBlockCodec() without parameters? */ final ConcatenatedBytesCodec concatenatedBytesCodec = new ConcatenatedBytesCodec(byteCodecs); this.dataBlockCodec = N5Codecs.createDataBlockCodec(attributes.getDataType(), concatenatedBytesCodec); + this.attributes = attributes; } @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { @@ -34,6 +37,13 @@ public class N5BlockCodec implements Codec.ArrayCodec { return dataBlockCodec.encode(dataBlock); } + @Override public long encodedSize(long size) { + + final int[] blockSize = attributes.getBlockSize(); + int headerSize = new N5Codecs.BlockHeader(blockSize, DataBlock.getNumElements(blockSize)).getSize(); + return headerSize + size; + } + @Override public String getType() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index 084be2d64..113e7ba9a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -273,6 +273,20 @@ static class BlockHeader { this.numElements = numElements; } + public int getSize() { + + switch (mode) { + case MODE_DEFAULT: + return 2 + 4 * blockSize.length; + case MODE_VARLENGTH: + return 2 + 4 * blockSize.length + 4; + case MODE_OBJECT: + return 2 + 4; + default: + throw new IllegalArgumentException("Unexpected mode: " + mode); + } + } + public int[] blockSize() { return blockSize; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index a72ad4508..66f423b34 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -35,12 +35,14 @@ public class ShardIndex extends LongArrayDataBlock { private final IndexLocation location; private final DeterministicSizeCodec[] codecs; + private final DatasetAttributes indexAttributes; public ShardIndex(int[] shardBlockGridSize, long[] data, IndexLocation location, final DeterministicSizeCodec... codecs) { super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, data); this.codecs = codecs; this.location = location; + this.indexAttributes = createIndexAttributes(); } public ShardIndex(int[] shardBlockGridSize, IndexLocation location, DeterministicSizeCodec... codecs) { @@ -67,7 +69,9 @@ public boolean exists(int blockNum) { public int getNumBlocks() { - return Arrays.stream(getSize()).reduce(1, (x, y) -> x * y); + /* getSize() is the number of data entries; each block takes 2 entries (offset and length) + * so the product of the dimension sizes, divided by 2, is the number of blocks. */ + return Arrays.stream(getSize()).reduce(1, (x, y) -> x * y) / 2; } public boolean isEmpty() { @@ -155,7 +159,7 @@ public static boolean read(byte[] data, final ShardIndex index) { public static void read(InputStream in, final ShardIndex index) throws IOException { - @SuppressWarnings("unchecked") final DataBlock indexBlock = (DataBlock)DefaultBlockReader.readBlock(in, index.getIndexAttributes(), index.gridPosition); + @SuppressWarnings("unchecked") final DataBlock indexBlock = (DataBlock)DefaultBlockReader.readBlock(in, index.createIndexAttributes(), index.gridPosition); final long[] indexData = indexBlock.getData(); System.arraycopy(indexData, 0, index.data, 0, index.data.length); } @@ -189,10 +193,15 @@ public static void write( public static void write(final ShardIndex index, OutputStream out) throws IOException { - DefaultBlockWriter.writeBlock(out, index.getIndexAttributes(), index); + DefaultBlockWriter.writeBlock(out, index.createIndexAttributes(), index); } - private DatasetAttributes getIndexAttributes() { + public Codec.ArrayCodec getArrayCodec() { + return indexAttributes.getArrayCodec(); + } + + // TODO: Caleb static? + private DatasetAttributes createIndexAttributes() { final DatasetAttributes indexAttributes = new DatasetAttributes( diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index f2637126f..0432150ab 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -6,6 +6,7 @@ import org.janelia.saalfeldlab.n5.N5FSTest; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; @@ -70,7 +71,7 @@ public void writeReadTest() throws IOException { final int[] shardBlockGridSize = new int[]{6, 5}; final IndexLocation indexLocation = IndexLocation.END; final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[]{ - new RawBytes<>(), + new N5BlockCodec<>(), new Crc32cChecksumCodec()}; final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "indexTest").toString(); @@ -80,6 +81,8 @@ public void writeReadTest() throws IOException { index.set(93, 111, new int[]{3, 0}); index.set(143, 1, new int[]{1, 2}); + final long indexSize = index.getArrayCodec().encodedSize(index.numBytes()); + long currentSize; try { currentSize = kva.size(path); @@ -88,7 +91,7 @@ public void writeReadTest() throws IOException { } final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, currentSize); try ( - final LockedChannel channel = kva.lockForWriting(path, bounds.start, index.numBytes()); + final LockedChannel channel = kva.lockForWriting(path, bounds.start, indexSize); final OutputStream out = channel.newOutputStream() ) { ShardIndex.write(index, out); @@ -96,7 +99,7 @@ public void writeReadTest() throws IOException { final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); try ( - final LockedChannel channel = kva.lockForReading(path, bounds.start, index.numBytes()); + final LockedChannel channel = kva.lockForReading(path, bounds.start, indexSize); final InputStream in = channel.newInputStream() ) { ShardIndex.read(in, indexRead); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 72864f2ed..2819dfd1a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -36,7 +36,7 @@ @RunWith(Parameterized.class) public class ShardTest { - private static final boolean LOCAL_DEBUG = true; + private static final boolean LOCAL_DEBUG = false; private static final N5FSTest tempN5Factory = new N5FSTest() { @@ -56,10 +56,8 @@ public static Collection data() { final ArrayList params = new ArrayList<>(); for (IndexLocation indexLoc : IndexLocation.values()) { - for (ByteOrder blockByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN}) { - for (ByteOrder indexByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN}) { - params.add(new Object[]{indexLoc, blockByteOrder, indexByteOrder}); - } + for (ByteOrder indexByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { + params.add(new Object[]{indexLoc, indexByteOrder}); } } final int numParams = params.size(); @@ -72,9 +70,6 @@ public static Collection data() { public IndexLocation indexLocation; @Parameterized.Parameter(1) - public ByteOrder dataByteOrder; - - @Parameterized.Parameter(2) public ByteOrder indexByteOrder; @After @@ -289,7 +284,7 @@ public void writeReadShardTest() { } @Test - @Ignore + @Ignore("Not currently supported ") public void writeReadNestedShards() { int[] blockSize = new int[]{4, 4}; @@ -305,7 +300,8 @@ public void writeReadNestedShards() { writer.writeBlocks("nestedShards", datasetAttributes, new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), new ByteArrayDataBlock(blockSize, new long[]{0, 2}, data), - new ByteArrayDataBlock(blockSize, new long[]{2, 1}, data)); + new ByteArrayDataBlock(blockSize, new long[]{2, 1}, data) + ); assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 1, 1).getData()); assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 0, 2).getData()); @@ -338,41 +334,26 @@ private DatasetAttributes getNestedShardCodecsAttributes(int[] blockSize) { ); } - public static void main(String[] args) { - - final long[] imageSize = new long[]{32, 27}; - final int[] shardSize = new int[]{16, 9}; - final int[] blockSize = new int[]{4, 3}; - final int numBlockElements = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y); - - try (final N5Writer n5 = new N5Factory().openWriter("n5:/tmp/tests/codeReview/sharded.n5")) { - - final DatasetAttributes attributes = getDatasetAttributes(imageSize, shardSize, blockSize); - } - - } - private static DatasetAttributes getDatasetAttributes(long[] imageSize, int[] shardSize, int[] blockSize) { - final DatasetAttributes attributes = new DatasetAttributes( + return new DatasetAttributes( imageSize, shardSize, blockSize, DataType.INT32, - new ShardingCodec( + new ShardingCodec<>( blockSize, new Codec[]{ // codecs applied to image data - new RawBytes(ByteOrder.BIG_ENDIAN), + new RawBytes<>(ByteOrder.BIG_ENDIAN), }, new DeterministicSizeCodec[]{ // codecs applied to the shard index, must not be compressors - new RawBytes(ByteOrder.LITTLE_ENDIAN), + new RawBytes<>(ByteOrder.LITTLE_ENDIAN), new Crc32cChecksumCodec() }, IndexLocation.START ) ); - return attributes; } } From d8504eb922b8e8f65e2a00dddf9e13c7d872147b Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 May 2025 16:59:40 -0400 Subject: [PATCH 230/423] refactor: branch sharding logic on ShardingCodec --- .../java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 414b974b7..609dade84 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -61,6 +61,7 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.shard.InMemoryShard; import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.util.Position; import java.io.IOException; @@ -249,7 +250,7 @@ default boolean removeAttributes(final String pathName, final List attri final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { - if (datasetAttributes.getShardSize() != null) { + if (datasetAttributes.getArrayCodec() instanceof ShardingCodec) { /* Group blocks by shard index */ final Map>> shardBlockMap = datasetAttributes.groupBlocks( From 65345d9f1e0ae308c10039eb95102081c6d529d7 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 23 May 2025 17:10:48 -0400 Subject: [PATCH 231/423] doc: SplittableReadData.slice --- .../n5/readdata/SplittableReadData.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java index f5195bef5..1be480f40 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java @@ -10,8 +10,22 @@ default ReadData limit(final long length) throws IOException { return slice(0, length); } + /** + * Returns a new {@link ReadData} representing a slice, or subset + * of this ReadData. + * + * @param offset the offset relative to this + * @param length of the returned ReadData + * @return a slice + * @throws IOException an exception + */ ReadData slice(final long offset, final long length) throws IOException; - // TODO do we want this? how should it work? + /* + * TODO do we want this? how should it work? + * this could be useful for infinite data, or data of unknown length? + * + * tail below would be equivalent to slice(pivot, -1) + */ Pair split(final long pivot) throws IOException; } From b9130deb09368268140ad3189f6e0768d10d188e Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 23 May 2025 17:11:51 -0400 Subject: [PATCH 232/423] refactor: hide KeyValueAccessSplittableReadData --- .../saalfeldlab/n5/KeyValueAccessSplittableReadData.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java index 56bf5351a..b1d1fb842 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java @@ -17,7 +17,7 @@ abstract class KeyValueAccessSplittableReadData implem protected final long offset; protected long length; - public KeyValueAccessSplittableReadData(K kva, String normalKey, long offset, long length) { + KeyValueAccessSplittableReadData(K kva, String normalKey, long offset, long length) { this.kva = kva; this.normalKey = normalKey; @@ -25,7 +25,7 @@ public KeyValueAccessSplittableReadData(K kva, String normalKey, long offset, lo this.length = length; } - public KeyValueAccessSplittableReadData(K kva, String normalKey, long offset) { + KeyValueAccessSplittableReadData(K kva, String normalKey, long offset) { this(kva, normalKey, offset, -1); } From 2243a4f5277a0c24e2f949fae88de25133fdbf8c Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 23 May 2025 17:13:03 -0400 Subject: [PATCH 233/423] perf: FileSystemKva's ReadData uses lockForReading --- .../n5/FileSystemKeyValueAccess.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index ab902b99a..e0f89d3f3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -147,6 +147,10 @@ protected LockedFileChannel(final Path path, final boolean readOnly, final long } } + protected FileChannel getFileChannel() { + return channel; + } + @Override public long size() throws IOException { @@ -607,25 +611,20 @@ public FileSplittableReadData(FileSystemKeyValueAccess kva, String normalKey, lo @Override void read() throws IOException { - final FileChannel channel; - try { - Path path = Paths.get(kva.uri(normalKey)); - channel = FileChannel.open(path, StandardOpenOption.READ); + try (FileChannel channel = kva.lockForReading(normalKey).getFileChannel()) { + channel.position(offset); + if (length > Integer.MAX_VALUE) + throw new IOException("Attempt to materialize too large data"); + + final int sz = (int)(length < 0 ? channel.size() : (int)length); + final byte[] data = new byte[sz]; + final ByteBuffer buf = ByteBuffer.wrap(data); + channel.read(buf); + materialized = (SplittableReadData)ReadData.from(data); } catch (final NoSuchFileException e) { throw new N5Exception.N5NoSuchKeyException(e); - } catch (URISyntaxException e) { - throw new N5Exception(e); } - channel.position(offset); - - if (length > Integer.MAX_VALUE) - throw new IOException("Attempt to materialize too large data"); - final int sz = (int)(length < 0 ? channel.size() : (int)length); - final byte[] data = new byte[sz]; - final ByteBuffer buf = ByteBuffer.wrap(data); - channel.read(buf); - materialized = (SplittableReadData)ReadData.from(data); } @Override From 19af7cc5411c3a5da1b929e87f49eb4686b9826d Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 23 May 2025 17:16:27 -0400 Subject: [PATCH 234/423] test: add CodecTests * testing ConcatenatedBytesCodec --- .../saalfeldlab/n5/codec/CodecTests.java | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/CodecTests.java diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/CodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/CodecTests.java new file mode 100644 index 000000000..4fbdc5381 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/CodecTests.java @@ -0,0 +1,98 @@ +package org.janelia.saalfeldlab.n5.codec; + +import static org.junit.Assert.assertArrayEquals; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Random; +import java.util.function.IntUnaryOperator; + +import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.ReadData.OutputStreamOperator; +import org.janelia.saalfeldlab.n5.readdata.SplittableReadData; +import org.junit.BeforeClass; +import org.junit.Test; + +public class CodecTests { + + static Random random; + + @BeforeClass + public static void setup() { + random = new Random(7777); + } + + @Test + public void concatenatedBytesCodecTest() throws IOException { + + int N = 16; + ReadData data = ReadData.from( new InputStream() { + @Override + public int read() throws IOException { + return Math.abs(random.nextInt()) % 32; + } + }, N ).materialize(); + + final byte[] bytes = data.allBytes(); + final byte[] expected = new byte[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + expected[i] = (byte)(2 * bytes[i] + 3); + } + + final BytesCodec a = new ByteFunctionCodec(x -> 2 * x, x -> x / 2); + final BytesCodec b = new ByteFunctionCodec(x -> x + 3, x -> x - 3 ); + final ConcatenatedBytesCodec ab = new ConcatenatedBytesCodec(a, b); + + final SplittableReadData encodedData = ab.encode(data).materialize(); + assertArrayEquals(expected, encodedData.allBytes()); + + final SplittableReadData decodedData = ab.decode(encodedData).materialize(); + assertArrayEquals(bytes, decodedData.allBytes()); + } + + public static class ByteFunctionCodec implements BytesCodec { + + IntUnaryOperator encoder; + IntUnaryOperator decoder; + + public ByteFunctionCodec( IntUnaryOperator encoder, IntUnaryOperator decoder ) { + this.encoder = encoder; + this.decoder = decoder; + } + + @Override + public String getType() { + return "byteFunction"; + } + + public ReadData decode(ReadData data) throws IOException { + return data.encode(new ByteFun(decoder)); + } + + public ReadData encode(ReadData data) throws IOException { + return data.encode(new ByteFun(encoder)); + } + } + + private static class ByteFun implements OutputStreamOperator { + + IntUnaryOperator fun; + public ByteFun(IntUnaryOperator fun) { + this.fun = fun; + } + + @Override + public OutputStream apply(OutputStream o) throws IOException { + return new OutputStream() { + @Override + public void write(int b) throws IOException { + o.write(fun.applyAsInt(b)); + } + }; + } + } + +} From c3a87cbb906f7303f1b18b099c1d3e6a000fa7d8 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 27 May 2025 11:50:10 -0400 Subject: [PATCH 235/423] refactor: remove deprecated api methods, some cleanup --- .../saalfeldlab/n5/BlockParameters.java | 11 -- .../saalfeldlab/n5/DatasetAttributes.java | 30 +---- .../n5/ShardedDatasetAttributes.java | 124 ------------------ .../saalfeldlab/n5/codec/N5Codecs.java | 1 - .../saalfeldlab/n5/shard/InMemoryShard.java | 34 +---- .../janelia/saalfeldlab/n5/shard/Shard.java | 8 -- .../saalfeldlab/n5/shard/ShardParameters.java | 16 ++- .../saalfeldlab/n5/shard/VirtualShard.java | 1 + .../saalfeldlab/n5/AbstractN5Test.java | 4 +- .../saalfeldlab/n5/demo/BlockIterators.java | 25 ++-- 10 files changed, 32 insertions(+), 222 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/BlockParameters.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BlockParameters.java b/src/main/java/org/janelia/saalfeldlab/n5/BlockParameters.java deleted file mode 100644 index 65a214972..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/BlockParameters.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.janelia.saalfeldlab.n5; - -public interface BlockParameters { - - public long[] getDimensions(); - - public int getNumDimensions(); - - public int[] getBlockSize(); - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 7fa306ec0..22cc6a5ab 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -85,8 +85,8 @@ * * @author Stephan Saalfeld */ -//TODO Caleb: try to delete ShardParameters? -public class DatasetAttributes implements BlockParameters, ShardParameters, Serializable { +//TODO : inline ShardParameters and delete interface +public class DatasetAttributes implements ShardParameters, Serializable { private static final long serialVersionUID = -4521467080388947553L; @@ -174,27 +174,6 @@ public DatasetAttributes( this( dimensions, null, blockSize, dataType, codecs ); } - /** - * Deprecated. {@link Compression} are {@link Codec}. Use {@code Code...} constructor instead - * Constructs a DatasetAttributes instance with specified dimensions, block size, data type, - * and compression scheme. This constructor is deprecated and redirects to another constructor - * with codec support. - * - * @param dimensions the dimensions of the dataset - * @param blockSize the size of the blocks in the dataset - * @param dataType the data type of the dataset - * @param compression the compression scheme used for storing the dataset - */ - @Deprecated - public DatasetAttributes( - final long[] dimensions, - final int[] blockSize, - final DataType dataType, - final Compression compression) { - - this(dimensions, blockSize, dataType, (Codec)compression); - } - @Override public long[] getDimensions() { @@ -220,10 +199,11 @@ public int[] getBlockSize() { } /** - * Deprecated. {@link Compression} is no longer a special case. prefer to reference {@link #getCodecs()} + * Only used for deserialization for N5 backwards compatibility. + * {@link Compression} is no longer a special case. Prefer to reference {@link #getCodecs()} * Will return {@link RawCompression} if no compression is otherwise provided, for legacy compatibility. * - * @return compression Codec, if one was present + * @return compression Codec, if one was present, or else RawCompression */ @Deprecated public Compression getCompression() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java deleted file mode 100644 index 335c252ed..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardedDatasetAttributes.java +++ /dev/null @@ -1,124 +0,0 @@ -package org.janelia.saalfeldlab.n5; - -import java.util.Arrays; - -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; -import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; -import org.janelia.saalfeldlab.n5.shard.ShardParameters; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; - -@Deprecated -public class ShardedDatasetAttributes extends DatasetAttributes implements ShardParameters { - - private static final long serialVersionUID = -4559068841006651814L; - - private final int[] shardSize; - - private final ShardingCodec shardingCodec; - - public ShardedDatasetAttributes ( - final long[] dimensions, - final int[] shardSize, //in pixels - final int[] blockSize, //in pixels - final DataType dataType, - final Codec[] blocksCodecs, - final DeterministicSizeCodec[] indexCodecs, - final IndexLocation indexLocation - ) { - //TODO Caleb: Can we just let the super codecs() return this ShardCodec? - super(dimensions, blockSize, dataType, blocksCodecs); - - if (!validateShardBlockSize(shardSize, blockSize)) { - throw new N5Exception(String.format("Invalid shard %s / block size %s", - Arrays.toString(shardSize), - Arrays.toString(blockSize))); - } - - this.shardSize = shardSize; - this.shardingCodec = new ShardingCodec( - blockSize, - blocksCodecs, - indexCodecs, - indexLocation - ); - } - - public ShardedDatasetAttributes( - final long[] dimensions, - final int[] shardSize, //in pixels - final int[] blockSize, //in pixels - final DataType dataType, - final ShardingCodec codec) { - super(dimensions, blockSize, dataType, null, null); - this.shardSize = shardSize; - this.shardingCodec = codec; - } - - /** - * Returns whether the given shard and block sizes are valid. Specifically, is - * the shard size a multiple of the block size in every dimension. - * - * @param shardSize size of the shard in pixels - * @param blockSize size of a block in pixels - * @return - */ - public static boolean validateShardBlockSize(final int[] shardSize, final int[] blockSize) { - - if (shardSize.length != blockSize.length) - return false; - - for (int i = 0; i < shardSize.length; i++) { - if (shardSize[i] % blockSize[i] != 0) - return false; - } - return true; - } - - public ShardingCodec getShardingCodec() { - return shardingCodec; - } - - @Override public ArrayCodec getArrayCodec() { - - return shardingCodec.getArrayCodec(); - } - - @Override public BytesCodec[] getCodecs() { - - return shardingCodec.getCodecs(); - } - - @Override - protected Codec[] concatenateCodecs() { - - return new Codec[] { shardingCodec }; - } - - public IndexLocation getIndexLocation() { - - return getShardingCodec().getIndexLocation(); - } - - /** - * The size of the blocks in pixel units. - * - * @return the number of pixels per dimension for this shard. - */ - @Override - public int[] getShardSize() { - - return shardSize; - } - - public static int[] getBlockSize(Codec[] codecs) { - - for (final Codec codec : codecs) - if (codec instanceof ShardingCodec) - return ((ShardingCodec)codec).getBlockSize(); - - return null; - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index 113e7ba9a..937a4d23e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -35,7 +35,6 @@ import java.io.OutputStream; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; -import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataBlock.DataBlockFactory; import org.janelia.saalfeldlab.n5.DataType; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 38019beb0..83d222102 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -17,11 +17,8 @@ public class InMemoryShard extends AbstractShard { private final Map> blocks; private ShardIndexBuilder indexBuilder; - /* - * TODO: - * Use morton- or c-ording instead of writing blocks out in the order they're added? - * (later) - */ + //TODO delegated shard constructor? Or new class? + public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] shardPosition) { this(datasetAttributes, shardPosition, null); @@ -57,39 +54,12 @@ public void addBlock(DataBlock block) { storeBlock(block); } - public int numBlocks() { - - return blocks.size(); - } - @Override public List> getBlocks() { return new ArrayList<>(blocks.values()); } - public List> getBlocks( int[] blockIndexes ) { - - final ArrayList> out = new ArrayList<>(); - final int[] blocksPerShard = getDatasetAttributes().getBlocksPerShard(); - - long[] position = new long[ getSize().length ]; - for( int idx : blockIndexes ) { - GridIterator.indexToPosition(idx, blocksPerShard, position); - DataBlock blk = getBlock(position); - if( blk != null ) - out.add(blk); - } - return out; - } - protected IndexLocation indexLocation() { - - if (index != null) - return index.getLocation(); - else - return indexBuilder.getLocation(); - } - @Override public ShardIndex getIndex() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index abf7c5213..ca758f28b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -135,14 +135,6 @@ default Iterator blockPositionIterator() { ShardIndex getIndex(); - static Shard createEmpty(final DatasetAttributes attributes, long... shardPosition) { - - final long[] emptyIndex = new long[(int)(2 * attributes.getNumBlocks())]; - Arrays.fill(emptyIndex, ShardIndex.EMPTY_INDEX_NBYTES); - final ShardIndex shardIndex = new ShardIndex(attributes.getBlocksPerShard(), emptyIndex, ShardingCodec.IndexLocation.END); - return new InMemoryShard(attributes, shardPosition, shardIndex); - } - default ReadData createReadData() throws IOException { final DatasetAttributes datasetAttributes = getDatasetAttributes(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java index 714c0cbea..900dd6f3f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java @@ -13,13 +13,18 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; -import org.janelia.saalfeldlab.n5.BlockParameters; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.janelia.saalfeldlab.n5.util.Position; @Deprecated -public interface ShardParameters extends BlockParameters { +public interface ShardParameters { + + public long[] getDimensions(); + + public int getNumDimensions(); + + public int[] getBlockSize(); /** @@ -55,7 +60,7 @@ default int[] getBlocksPerShard() { */ default long[] blocksPerImage() { return IntStream.range(0, getNumDimensions()) - .mapToLong(i -> (long) Math.ceil(getDimensions()[i] / getBlockSize()[i])) + .mapToLong(i -> (long) Math.ceil((double)getDimensions()[i] / getBlockSize()[i])) .toArray(); } @@ -66,7 +71,7 @@ default long[] blocksPerImage() { */ default long[] shardsPerImage() { return IntStream.range(0, getNumDimensions()) - .mapToLong(i -> (long)Math.ceil(getDimensions()[i] / getShardSize()[i])) + .mapToLong(i -> (long)Math.ceil((double)getDimensions()[i] / getShardSize()[i])) .toArray(); } @@ -199,7 +204,7 @@ default Map>> groupBlocks(final List x * y); } @@ -220,5 +225,4 @@ static Stream toStream( final Iterator it ) { it, Spliterator.ORDERED), false); } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index c651e02e8..d5cc7fb97 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -18,6 +18,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +//TODO : consider different names? LazyShard? ShardView? PartialReadShard? public class VirtualShard extends AbstractShard { private final SplittableReadData shardData; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 986ea984d..b891a9f46 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -269,7 +269,7 @@ public void testCreateDataset() { assertArrayEquals(dimensions, info.getDimensions()); assertArrayEquals(blockSize, info.getBlockSize()); assertEquals(DataType.UINT64, info.getDataType()); - assertTrue(info.getCompression() instanceof RawCompression); + assertEquals(0, info.getCodecs().length); } @Test @@ -1661,6 +1661,6 @@ protected void assertDatasetAttributesEquals(final DatasetAttributes expected, f assertArrayEquals(expected.getDimensions(), actual.getDimensions()); assertArrayEquals(expected.getBlockSize(), actual.getBlockSize()); assertEquals(expected.getDataType(), actual.getDataType()); - assertEquals(expected.getCompression(), actual.getCompression()); + assertArrayEquals(expected.getCodecs(), actual.getCodecs()); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java index 091faf615..f876ec85b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java @@ -11,11 +11,11 @@ import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.RawCompression; -import org.janelia.saalfeldlab.n5.ShardedDatasetAttributes; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; @@ -29,14 +29,17 @@ public static void main(String[] args) { public static void shardBlockIterator() { - final ShardedDatasetAttributes attrs = new ShardedDatasetAttributes( + final DatasetAttributes attrs = new DatasetAttributes( new long[] {12, 8}, // image size new int[] {6, 4}, // shard size new int[] {2, 2}, // block size DataType.UINT8, - new Codec[] { new N5BlockCodec<>() }, - new DeterministicSizeCodec[] { new RawBytes<>() }, - IndexLocation.END); + new ShardingCodec<>( + new int[] {2, 2}, + new Codec[] { new N5BlockCodec<>() }, + new DeterministicSizeCodec[] { new RawBytes<>() }, + IndexLocation.END + )); shardPositions(attrs) .forEach(x -> System.out.println(Arrays.toString(x))); @@ -56,18 +59,14 @@ public static void blockIterator() { public static long[] blockGridSize(final DatasetAttributes attrs ) { // this could be a nice method for DatasetAttributes - return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> { - return (long)Math.ceil(attrs.getDimensions()[i] / attrs.getBlockSize()[i]); - }).toArray(); + return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getBlockSize()[i])).toArray(); } - public static long[] shardGridSize(final ShardedDatasetAttributes attrs ) { + public static long[] shardGridSize(final DatasetAttributes attrs ) { // this could be a nice method for DatasetAttributes - return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> { - return (long)Math.ceil(attrs.getDimensions()[i] / attrs.getShardSize()[i]); - }).toArray(); + return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getShardSize()[i])).toArray(); } @@ -75,7 +74,7 @@ public static Stream blockPositions( DatasetAttributes attrs ) { return toStream(new GridIterator(blockGridSize(attrs))); } - public static Stream shardPositions( ShardedDatasetAttributes attrs ) { + public static Stream shardPositions( DatasetAttributes attrs ) { final int[] blocksPerShard = attrs.getBlocksPerShard(); return toStream( new GridIterator(shardGridSize(attrs))) From db0e1a3cdfc249dbf6261d8673f4ed0df0c67a3c Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 27 May 2025 11:51:39 -0400 Subject: [PATCH 236/423] feat: Shard as a the base case for writing --- .../saalfeldlab/n5/DatasetAttributes.java | 30 +++++++++++++++---- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 2 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 6 ++-- .../saalfeldlab/n5/shard/InMemoryShard.java | 6 ++-- .../janelia/saalfeldlab/n5/shard/Shard.java | 5 ++-- .../saalfeldlab/n5/shard/ShardTest.java | 2 +- 6 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 22cc6a5ab..213856b3d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -59,6 +59,7 @@ import java.util.HashMap; import java.util.stream.Stream; +import org.janelia.saalfeldlab.n5.N5Exception.N5ShardException; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; @@ -72,6 +73,7 @@ import com.google.gson.JsonParseException; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; /** * Mandatory dataset attributes: @@ -134,11 +136,11 @@ public DatasetAttributes( final Codec[] filteredCodecs = Arrays.stream(codecs).filter(it -> !(it instanceof RawCompression)).toArray(Codec[]::new); if (filteredCodecs.length == 0) { byteCodecs = new BytesCodec[]{}; - arrayCodec = new N5BlockCodec(); + arrayCodec = new N5BlockCodec<>(); } else if (filteredCodecs.length == 1 && filteredCodecs[0] instanceof Compression) { final BytesCodec compression = (BytesCodec)filteredCodecs[0]; byteCodecs = compression instanceof RawCompression ? new BytesCodec[]{} : new BytesCodec[]{compression}; - arrayCodec = new N5BlockCodec(); + arrayCodec = new N5BlockCodec<>(); } else { if (!(filteredCodecs[0] instanceof ArrayCodec)) throw new N5Exception("Expected first element of filteredCodecs to be ArrayCodec, but was: " + filteredCodecs[0].getClass()); @@ -171,7 +173,7 @@ public DatasetAttributes( final int[] blockSize, final DataType dataType, final Codec... codecs) { - this( dimensions, null, blockSize, dataType, codecs ); + this( dimensions, blockSize, blockSize, dataType, codecs ); } @Override @@ -198,6 +200,23 @@ public int[] getBlockSize() { return blockSize; } + public boolean isSharded() { + return getArrayCodec() instanceof ShardingCodec; + } + + /** + * If this dataset is sharded, return the ArrayCodec as a ShardingCodec + * @return the ShardingCodec + * + * @throws N5ShardException if the dataset is not sharded + */ + public ShardingCodec getShardingCodec() throws N5ShardException { + if (getArrayCodec() instanceof ShardingCodec) + return (ShardingCodec)getArrayCodec(); + else + throw new N5ShardException("Dataset is not Sharded"); + } + /** * Only used for deserialization for N5 backwards compatibility. * {@link Compression} is no longer a special case. Prefer to reference {@link #getCodecs()} @@ -205,8 +224,7 @@ public int[] getBlockSize() { * * @return compression Codec, if one was present, or else RawCompression */ - @Deprecated - public Compression getCompression() { + private Compression getCompression() { return Arrays.stream(byteCodecs) .filter(it -> it instanceof Compression) @@ -230,9 +248,11 @@ public DataType getDataType() { */ public ArrayCodec getArrayCodec() { + //noinspection unchecked return (ArrayCodec) arrayCodec; } + public BytesCodec[] getCodecs() { return byteCodecs; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 73637de22..1cec971e7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -142,7 +142,7 @@ default List> readBlocks( final List blockPositions) throws N5Exception { // TODO which interface should have this implementation? - if (datasetAttributes.getArrayCodec() instanceof ShardingCodec) { + if (datasetAttributes.isSharded()) { /* Group by shard position */ final Map> shardBlockMap = datasetAttributes.groupBlockPositions(blockPositions); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 609dade84..43d7e446b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -250,7 +250,7 @@ default boolean removeAttributes(final String pathName, final List attri final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { - if (datasetAttributes.getArrayCodec() instanceof ShardingCodec) { + if (datasetAttributes.isSharded()) { /* Group blocks by shard index */ final Map>> shardBlockMap = datasetAttributes.groupBlocks( @@ -284,12 +284,12 @@ default void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception { - if (datasetAttributes.getShardSize() != null) { + if (datasetAttributes.isSharded()) { writeBlocks(path, datasetAttributes, dataBlock); return; } - final long[] keyPos = datasetAttributes.getArrayCodec().getPositionForBlock(datasetAttributes, dataBlock); + final long[] keyPos = datasetAttributes.getShardingCodec().getPositionForBlock(datasetAttributes, dataBlock); final String keyPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), keyPos); try ( final LockedChannel channel = getKeyValueAccess().lockForWriting(keyPath); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 83d222102..3e05fff08 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -3,7 +3,6 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; -import org.janelia.saalfeldlab.n5.util.GridIterator; import org.janelia.saalfeldlab.n5.util.Position; import java.util.ArrayList; @@ -13,8 +12,9 @@ public class InMemoryShard extends AbstractShard { - /* Map of a hash of the DataBlocks `gridPosition` to the block */ + /** Map {@link DataBlock#getGridPosition} as hashable {@link Position} to the block */ private final Map> blocks; + private ShardIndexBuilder indexBuilder; //TODO delegated shard constructor? Or new class? @@ -23,7 +23,7 @@ public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] sha this(datasetAttributes, shardPosition, null); indexBuilder = new ShardIndexBuilder(this); - final IndexLocation indexLocation = ((ShardingCodec)datasetAttributes.getArrayCodec()).getIndexLocation(); + final IndexLocation indexLocation = datasetAttributes.getShardingCodec().getIndexLocation(); indexBuilder.indexLocation(indexLocation); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index ca758f28b..8b055fb9e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -140,8 +140,7 @@ default ReadData createReadData() throws IOException { final DatasetAttributes datasetAttributes = getDatasetAttributes(); final ShardIndex index = ShardIndex.createIndex(datasetAttributes); - @SuppressWarnings("unchecked") - ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); + ShardingCodec shardingCodec = datasetAttributes.getShardingCodec(); final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); long blocksStartBytes = index.getLocation() == ShardingCodec.IndexLocation.START ? index.numBytes() : 0; final AtomicLong blockOffset = new AtomicLong(blocksStartBytes); @@ -205,7 +204,7 @@ public DataBlockIterator(final Shard shard) { @Override public boolean hasNext() { - for (int i = blockIndex; i < attributes.getNumBlocks(); i++) { + for (int i = blockIndex; i < attributes.getNumBlocksPerShard(); i++) { if (index.exists(i)) return true; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 2819dfd1a..fe83d3828 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -284,7 +284,7 @@ public void writeReadShardTest() { } @Test - @Ignore("Not currently supported ") + @Ignore("Nested sharding not supported ") public void writeReadNestedShards() { int[] blockSize = new int[]{4, 4}; From c8c433585c209a76a2e1e187bd5ab30d4e3faa79 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 27 May 2025 11:51:54 -0400 Subject: [PATCH 237/423] fix: NameConfig for Bzip2 --- src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index c34a559d0..5b80183a4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -68,6 +68,7 @@ public class Bzip2Compression implements Compression { private static final long serialVersionUID = -4873117458390529118L; @CompressionParameter + @NameConfig.Parameter private final int blockSize; public Bzip2Compression(final int blockSize) { From 4f29c338c6bd276b5e7812d796104ff01ac3f729 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 27 May 2025 11:53:07 -0400 Subject: [PATCH 238/423] fix: getArrayCodec not ShardingCodec --- .../java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 43d7e446b..4eb081796 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -289,7 +289,7 @@ default void writeBlock( return; } - final long[] keyPos = datasetAttributes.getShardingCodec().getPositionForBlock(datasetAttributes, dataBlock); + final long[] keyPos = datasetAttributes.getArrayCodec().getPositionForBlock(datasetAttributes, dataBlock); final String keyPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), keyPos); try ( final LockedChannel channel = getKeyValueAccess().lockForWriting(keyPath); From ceaf5c30848df5cc5ae225491f9cadadce7b3b53 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 27 May 2025 13:29:20 -0400 Subject: [PATCH 239/423] chore: remove duplicate/outdated license header --- .../saalfeldlab/n5/AbstractDataBlock.java | 25 ---------- .../saalfeldlab/n5/ByteArrayDataBlock.java | 25 ---------- .../saalfeldlab/n5/Bzip2Compression.java | 25 ---------- .../janelia/saalfeldlab/n5/Compression.java | 25 ---------- .../saalfeldlab/n5/CompressionAdapter.java | 29 +----------- .../org/janelia/saalfeldlab/n5/DataBlock.java | 25 ---------- .../org/janelia/saalfeldlab/n5/DataType.java | 25 ---------- .../saalfeldlab/n5/DatasetAttributes.java | 25 ---------- .../saalfeldlab/n5/DefaultBlockReader.java | 25 ---------- .../saalfeldlab/n5/DefaultBlockWriter.java | 25 ---------- .../saalfeldlab/n5/DoubleArrayDataBlock.java | 25 ---------- .../saalfeldlab/n5/FloatArrayDataBlock.java | 25 ---------- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 25 ---------- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 25 ---------- .../saalfeldlab/n5/GzipCompression.java | 25 ---------- .../saalfeldlab/n5/HttpKeyValueAccess.java | 46 +++++++++++-------- .../saalfeldlab/n5/IntArrayDataBlock.java | 29 +----------- .../janelia/saalfeldlab/n5/LockedChannel.java | 25 ---------- .../saalfeldlab/n5/LongArrayDataBlock.java | 29 +----------- .../saalfeldlab/n5/Lz4Compression.java | 25 ---------- .../janelia/saalfeldlab/n5/N5FSReader.java | 25 ---------- .../janelia/saalfeldlab/n5/N5FSWriter.java | 25 ---------- .../org/janelia/saalfeldlab/n5/N5Reader.java | 25 ---------- .../org/janelia/saalfeldlab/n5/N5Writer.java | 25 ---------- .../saalfeldlab/n5/NameConfigAdapter.java | 23 ++++++---- .../saalfeldlab/n5/RawCompression.java | 25 ---------- .../saalfeldlab/n5/ReflectionUtils.java | 29 +----------- .../saalfeldlab/n5/ShortArrayDataBlock.java | 29 +----------- .../saalfeldlab/n5/StringDataBlock.java | 29 +----------- .../janelia/saalfeldlab/n5/XzCompression.java | 25 ---------- .../saalfeldlab/n5/AbstractN5Test.java | 25 ---------- .../janelia/saalfeldlab/n5/N5Benchmark.java | 25 ---------- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 25 ---------- 33 files changed, 52 insertions(+), 816 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index 2158e07f1..f1cf6809f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.util.function.ToIntFunction; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index c3586a9f5..ca9664be0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; public class ByteArrayDataBlock extends AbstractDataBlock { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 5b80183a4..a03b42d12 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index 5d1e1831c..83481948c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java index 266304016..96d636b66 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java @@ -6,32 +6,6 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. @@ -42,7 +16,7 @@ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS @@ -50,6 +24,7 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. + * #L% */ package org.janelia.saalfeldlab.n5; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index ec1e26eeb..0324add04 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 73c0fb6cd..843c52521 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.lang.reflect.Type; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 213856b3d..bf3321d51 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.Serializable; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index c4068f12b..64645cdf0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index 5e1f5070c..9d6ea0463 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 5b8b41ebf..c958afd11 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; public class DoubleArrayDataBlock extends AbstractDataBlock { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index 4881db375..2f3287b6c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; public class FloatArrayDataBlock extends AbstractDataBlock { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 4eb081796..0d8d3aded 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import com.google.gson.Gson; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index c607f0f33..e441f363e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index 8ff096c41..2ca4d1692 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index 45d536aef..b3c370b0b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -1,22 +1,30 @@ -/** - * Copyright (c) 2017, Stephan Saalfeld All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, this list of conditions - * and the following disclaimer. 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY - * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/*- + * #%L + * Not HDF5 + * %% + * Copyright (C) 2017 - 2025 Stephan Saalfeld + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% */ package org.janelia.saalfeldlab.n5; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 4bea296c2..9bc5b831e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -6,32 +6,6 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. @@ -42,7 +16,7 @@ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS @@ -50,6 +24,7 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. + * #L% */ package org.janelia.saalfeldlab.n5; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java index 5adbd7bbc..420b1f231 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.Closeable; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index bad39ec6d..4780e1299 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -6,32 +6,6 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. @@ -42,7 +16,7 @@ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS @@ -50,6 +24,7 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. + * #L% */ package org.janelia.saalfeldlab.n5; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index 4918dfd5a..e0fdde456 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import net.jpountz.lz4.LZ4BlockInputStream; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 8d332098b..4397b4544 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.nio.file.FileSystems; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 589b566db..d856ebb22 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.nio.file.FileSystems; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 4a707cbbc..4c5063a5c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import org.janelia.saalfeldlab.n5.shard.Shard; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 5b066ecbc..514a239f0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import org.janelia.saalfeldlab.n5.codec.Codec; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java index 804a2cde8..7fd05a262 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java @@ -1,20 +1,22 @@ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - *

    +/*- + * #%L + * Not HDF5 + * %% + * Copyright (C) 2017 - 2025 Stephan Saalfeld + * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - *

    + * * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS @@ -22,6 +24,7 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. + * #L% */ package org.janelia.saalfeldlab.n5; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index d13a4a0ab..a64a4fd9c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java index eeff6fdc3..52d598d69 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java @@ -6,32 +6,6 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. @@ -42,7 +16,7 @@ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS @@ -50,6 +24,7 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. + * #L% */ package org.janelia.saalfeldlab.n5; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index a181e589c..753ad5e50 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -6,32 +6,6 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. @@ -42,7 +16,7 @@ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS @@ -50,6 +24,7 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. + * #L% */ package org.janelia.saalfeldlab.n5; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 5b811b5e4..f26d6cf23 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -6,32 +6,6 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. @@ -42,7 +16,7 @@ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS @@ -50,6 +24,7 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. + * #L% */ package org.janelia.saalfeldlab.n5; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index 1f086c620..bae52b688 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.IOException; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index b891a9f46..77d120dee 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import static org.junit.Assert.assertArrayEquals; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java index 5793615c9..815b8f119 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import static org.junit.Assert.fail; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 58b58fa8a..9eaeca325 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import static org.junit.Assert.fail; From d7c9d35e64f44546cbc740eee0fe06c87cbfdfd3 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 27 May 2025 13:38:17 -0400 Subject: [PATCH 240/423] test: writing / reading of invalid blocks --- .../saalfeldlab/n5/AbstractN5Test.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 77d120dee..aeaf428bc 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -467,6 +467,57 @@ public void testWriteReadSerializableBlock() throws ClassNotFoundException { } } + @Test + public void testWriteInvalidBlock() { + + final Compression compression = getCompressions()[0]; + final DataType dataType = DataType.UINT8; + + final int[] biggerBlockSize = Arrays.stream(blockSize).map(x -> x + 2).toArray(); + int nBigger = Arrays.stream(biggerBlockSize).reduce(1, (x, y) -> x * y); + + final int[] smallerBlockSize = Arrays.stream(blockSize).map(x -> x - 2).toArray(); + int nSmaller = Arrays.stream(smallerBlockSize).reduce(1, (x, y) -> x * y); + + int N = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y); + + final Random rnd = new Random(7560); + final byte[] biggerData = new byte[nBigger]; + rnd.nextBytes(biggerData); + + final byte[] smallerData = new byte[nSmaller]; + rnd.nextBytes(smallerData); + + final float[] floatData = new float[N]; + + try (final N5Writer n5 = createTempN5Writer()) { + + n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); + final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + + // write a block that is too large + final ByteArrayDataBlock bigDataBlock = new ByteArrayDataBlock(biggerBlockSize, new long[]{0, 0, 0}, biggerData); + n5.writeBlock(datasetName, attributes, bigDataBlock); + + final DataBlock loadedBigDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + assertArrayEquals(biggerData, (byte[])loadedBigDataBlock.getData()); + + // write a block that is too small + final ByteArrayDataBlock smallDataBlock = new ByteArrayDataBlock(smallerBlockSize, new long[]{0, 0, 0}, smallerData); + n5.writeBlock(datasetName, attributes, smallDataBlock); + + final DataBlock loadedSmallDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + System.out.println(((byte[])loadedSmallDataBlock.getData()).length); + assertArrayEquals(smallerData, (byte[])loadedSmallDataBlock.getData()); + + // write a block of the wrong type + final FloatArrayDataBlock floatDataBlock = new FloatArrayDataBlock(blockSize, new long[]{0, 0, 0}, floatData); + assertThrows(ClassCastException.class, () -> { + n5.writeBlock(datasetName, attributes, floatDataBlock); + }); + } + } + @Test public void testOverwriteBlock() { From 5d13af6d8b9f757da34dc3c1fe04cedac38533b2 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 2 Jun 2025 11:01:12 -0400 Subject: [PATCH 241/423] refactor: extract Pattern matching to be reused for attribute path normalization --- .../org/janelia/saalfeldlab/n5/N5URI.java | 118 ++++++++++++------ 1 file changed, 82 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java index 7d992fc96..0fde0783f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java @@ -454,6 +454,82 @@ public static String normalizeGroupPath(final String path) { return normalizePath(path.startsWith("/") || path.startsWith("\\") ? path.substring(1) : path); } + private enum N5UriPattern { + + /** + * matches any `/` or `[N]` where `N` is non-negative + */ + MULTI_PART_ATTRIBUTE(Pattern.compile(".*((?\\[[0-9]+])")), + /** + * matches `A[N]` where A is some non-empty preceding path + */ + ATTRIBUTE_ARRAY_EXCEPT_START(Pattern.compile("((?\\[[0-9]+]))")), + /** + * The following Pattern has 4 possible matches. + * It is intended to be used to remove matching portions iteratively until no further matches are found: + *

    + * The first 3 matches can remove redundant separators of the form: + *

      + *
    • (?<=/)/+ : `a///b` -> `a/b`
      • + *
    • (?<=(/|^))(\./)+ : `a/./b` -> `a/b`
      • + *
    • ((/|(?<=/))\.)$ : `a/b/` -> `a/b`
      • + *
    + * The next match avoids removing `/` when it is NOT redundant (e.g. only character, or escaped): + *
      + *
    • (? `/ , `/a/b/\\/` -> `/a/b/\\/`
      • + *
    + * The last match resolves relative paths: + *
      + *
    • 5. ((?<=^/)|^|(?<=(/|^))[^/]+(? + *
        + *
      • `a/../b` -> `b`
        • + *
      • `/a/../b` -> `/b`
        • + *
      • `../a/../b` -> `b`
        • + *
      • `/../a/../b` -> `/b`
        • + *
      • `/../a/../../b` -> `/b`
        • + *
      + *
    + *

    + */ + RELATIVE_ATTRIBUTE_PARTS(Pattern.compile( "((?<=/)/+|(?<=(/|^))(\\./)+|((/|(?<=/))\\.)$|(? @@ -503,50 +579,20 @@ public static String normalizeAttributePath(final String attributePath) { * Short circuit if there are no non-escaped `/` or array indices (e.g. * [N] where N is a non-negative integer) */ - if (!attributePath.matches(".*((? `[10]/b` */ - final String attrPathPlusFirstIndexSeparator = attributePath.replaceAll("^(?\\[[0-9]+])", "${array}/"); + final String attrPathPlusFirstIndexSeparator = N5UriPattern.appendSlashAfterArrayStart(attributePath); + /* * Add separator before and after arrays not at the beginning `a[10]b` * -> `a/[10]/b` */ - final String attrPathPlusIndexSeparators = attrPathPlusFirstIndexSeparator - .replaceAll("((?\\[[0-9]+]))", "/${array}/"); + final String attrPathPlusIndexSeparators = N5UriPattern.addSlashAroundArrayExceptStart(attrPathPlusFirstIndexSeparator); - /* - * The following has 4 possible matches, in each case it removes the - * match: - * The first 3 remove redundant separators of the form: - * 1.`a///b` -> `a/b` : (?<=/)/+ - * 2.`a/./b` -> `a/b` : (?<=(/|^))(\./)+ - * 3.`a/b/` -> `a/b` : ((/|(?<=/))\.)$ - * The next avoids removing `/` when it is NOT redundant (e.g. only character, or escaped): - * 4. `/` -> `/ , `/a/b/\\/` -> `/a/b/\\/` : (? `b` - * - `/a/../b` -> `/b` - * - `../a/../b` -> `b` - * - `/../a/../b` -> `/b` - * - `/../a/../../b` -> `/b` - * - * This is run iteratively, since earlier removals may cause later - * removals to be valid, - * as well as the need to match once per relative `../` pattern. - */ - final Pattern relativePathPattern = Pattern.compile( - "((?<=/)/+|(?<=(/|^))(\\./)+|((/|(?<=/))\\.)$|(? Date: Mon, 2 Jun 2025 14:23:41 -0400 Subject: [PATCH 242/423] refactor: migrate IOExceptions to N5IOExceptions for ReadData and related logic --- .../saalfeldlab/n5/Bzip2Compression.java | 14 +- .../janelia/saalfeldlab/n5/Compression.java | 5 +- .../n5/FileSystemKeyValueAccess.java | 23 ++- .../saalfeldlab/n5/GzipCompression.java | 11 +- .../saalfeldlab/n5/HttpKeyValueAccess.java | 15 +- .../n5/KeyValueAccessSplittableReadData.java | 9 +- .../saalfeldlab/n5/Lz4Compression.java | 5 +- .../janelia/saalfeldlab/n5/XzCompression.java | 12 +- .../saalfeldlab/n5/codec/AsTypeCodec.java | 9 +- .../janelia/saalfeldlab/n5/codec/Codec.java | 5 +- .../n5/codec/ConcatenatedBytesCodec.java | 8 +- .../saalfeldlab/n5/codec/DataBlockCodec.java | 7 +- .../saalfeldlab/n5/codec/DataCodec.java | 52 +++--- .../n5/codec/FixedScaleOffsetCodec.java | 8 +- .../saalfeldlab/n5/codec/IdentityCodec.java | 9 +- .../saalfeldlab/n5/codec/N5BlockCodec.java | 8 +- .../saalfeldlab/n5/codec/N5Codecs.java | 155 ++++++++++-------- .../saalfeldlab/n5/codec/RawBlockCodecs.java | 7 +- .../saalfeldlab/n5/codec/RawBytes.java | 7 +- .../n5/codec/checksum/ChecksumCodec.java | 10 +- .../readdata/AbstractInputStreamReadData.java | 10 +- .../readdata/ByteArraySplittableReadData.java | 6 +- .../n5/readdata/KeyValueAccessReadData.java | 27 +-- .../saalfeldlab/n5/readdata/LazyReadData.java | 29 ++-- .../saalfeldlab/n5/readdata/ReadData.java | 30 ++-- .../readdata/ReadDataSplittableReadData.java | 19 ++- .../janelia/saalfeldlab/n5/shard/Shard.java | 6 +- .../saalfeldlab/n5/shard/ShardIndex.java | 2 +- .../saalfeldlab/n5/shard/ShardingCodec.java | 7 +- .../saalfeldlab/n5/codec/CodecTests.java | 8 +- .../n5/readdata/ReadDataTests.java | 4 +- 31 files changed, 310 insertions(+), 217 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index a03b42d12..6480baaad 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -28,14 +28,15 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; - import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; +import java.io.IOException; + @CompressionType("bzip2") @NameConfig.Name("bzip2") public class Bzip2Compression implements Compression { @@ -66,9 +67,12 @@ public boolean equals(final Object other) { } @Override - public ReadData decode(final ReadData readData) throws IOException { - - return ReadData.from(new BZip2CompressorInputStream(readData.inputStream())); + public ReadData decode(final ReadData readData) throws N5IOException { + try { + return ReadData.from(new BZip2CompressorInputStream(readData.inputStream())); + } catch (IOException e) { + throw new N5IOException(e); + } } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index 83481948c..627c13a70 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -36,6 +36,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.scijava.annotations.Indexable; @@ -101,7 +102,7 @@ default String getType() { * @throws IOException * if any I/O error occurs */ - ReadData decode(ReadData readData) throws IOException; + ReadData decode(ReadData readData) throws N5IOException; /** * Encode the given {@code readData}. @@ -117,6 +118,6 @@ default String getType() { * @throws IOException * if any I/O error occurs */ - ReadData encode(ReadData readData) throws IOException; + ReadData encode(ReadData readData) throws N5IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index e0f89d3f3..9fd9ace0c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -29,6 +29,8 @@ package org.janelia.saalfeldlab.n5; import org.apache.commons.io.input.BoundedInputStream; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.SplittableReadData; @@ -212,7 +214,7 @@ public LockedFileChannel lockForReading(final String normalPath) throws IOExcept try { return new LockedFileChannel(normalPath, true); } catch (final NoSuchFileException e) { - throw new N5Exception.N5NoSuchKeyException("No such file", e); + throw new N5NoSuchKeyException("No such file", e); } } @@ -223,7 +225,7 @@ public LockedFileChannel lockForReading(final String normalPath, final long star try { return new LockedFileChannel(normalPath, true, startByte, size); } catch (final NoSuchFileException e) { - throw new N5Exception.N5NoSuchKeyException("No such file", e); + throw new N5NoSuchKeyException("No such file", e); } } @@ -250,7 +252,7 @@ public LockedFileChannel lockForReading(final Path path) throws IOException { try { return new LockedFileChannel(path, true); } catch (final NoSuchFileException e) { - throw new N5Exception.N5NoSuchKeyException("No such file", e); + throw new N5NoSuchKeyException("No such file", e); } } @@ -286,9 +288,9 @@ public long size(final String normalPath) { try { return Files.size(fileSystem.getPath(normalPath)); } catch (NoSuchFileException e) { - throw new N5Exception.N5NoSuchKeyException("No such file", e); + throw new N5NoSuchKeyException("No such file", e); } catch (IOException | UncheckedIOException e) { - throw new N5Exception.N5IOException(e); + throw new N5IOException(e); } } @@ -609,9 +611,12 @@ public FileSplittableReadData(FileSystemKeyValueAccess kva, String normalKey, lo } @Override - void read() throws IOException { + void read() throws N5IOException { - try (FileChannel channel = kva.lockForReading(normalKey).getFileChannel()) { + try ( + final LockedFileChannel lockedFileChannel = kva.lockForReading(normalKey); + final FileChannel channel = lockedFileChannel.getFileChannel() + ) { channel.position(offset); if (length > Integer.MAX_VALUE) throw new IOException("Attempt to materialize too large data"); @@ -622,7 +627,9 @@ void read() throws IOException { channel.read(buf); materialized = (SplittableReadData)ReadData.from(data); } catch (final NoSuchFileException e) { - throw new N5Exception.N5NoSuchKeyException(e); + throw new N5NoSuchKeyException(e); + } catch (final IOException e) { + throw new N5IOException(e); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index 2ca4d1692..9c81fef76 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -37,6 +37,7 @@ import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; import org.apache.commons.compress.compressors.gzip.GzipParameters; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @@ -96,13 +97,17 @@ private InputStream decode(final InputStream in) throws IOException { } @Override - public ReadData decode(final ReadData readData) throws IOException { + public ReadData decode(final ReadData readData) throws N5IOException { - return ReadData.from(decode(readData.inputStream())); + try { + return ReadData.from(decode(readData.inputStream())); + } catch (IOException e) { + throw new N5IOException(e); + } } @Override - public ReadData encode(final ReadData readData) { + public ReadData encode(final ReadData readData) { if (useZlib) { return readData.encode(out -> new DeflaterOutputStream(out, new Deflater(level))); } else { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index b3c370b0b..dfaa96793 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -31,6 +31,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.function.TriFunction; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.http.ListResponseParser; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -74,7 +75,7 @@ public class HttpKeyValueAccess implements KeyValueAccess { /** * Opens an {@link HttpKeyValueAccess} * - * @throws N5Exception.N5IOException if the access could not be created + * @throws N5IOException if the access could not be created */ public HttpKeyValueAccess() { @@ -305,7 +306,7 @@ private String[] queryListEntries(String normalPath, ListResponseParser parser, final String listResponse = responseToString(http.getInputStream()); return parser.parseListResponse(listResponse); } catch (IOException e) { - throw new N5Exception.N5IOException("Error listing directory at " + normalPath, e); + throw new N5IOException("Error listing directory at " + normalPath, e); } } @@ -333,7 +334,7 @@ private HttpURLConnection requireValidHttpResponse(String uri, String method, Tr code = http.getResponseCode(); responseMsg = http.getResponseMessage(); } catch (IOException e) { - throw new N5Exception.N5IOException("Could not validate HTTP Response", e); + throw new N5IOException("Could not validate HTTP Response", e); } final N5Exception cause = filterCode.apply(code, responseMsg, http); @@ -444,10 +445,14 @@ public HttpSplittableReadData(HttpKeyValueAccess kva, String normalKey, long off } @Override - void read() throws IOException { + void read() throws N5IOException { try( final HttpObjectChannel ch = new HttpObjectChannel(kva.uri(normalKey), offset, length) ) { materialized = ReadData.from(ch.newInputStream()).materialize(); - } catch (URISyntaxException e) {} + } catch (URISyntaxException e) { + throw new N5Exception(e); + } catch (IOException e) { + throw new N5IOException(e); + } } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java index b1d1fb842..3fe3f6301 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java @@ -5,6 +5,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.SplittableReadData; @@ -43,19 +44,19 @@ public long length() { } @Override - public InputStream inputStream() throws IOException, IllegalStateException { + public InputStream inputStream() throws N5IOException, IllegalStateException { return materialize().inputStream(); } @Override - public byte[] allBytes() throws IOException, IllegalStateException { + public byte[] allBytes() throws N5IOException, IllegalStateException { return materialize().allBytes(); } @Override - public SplittableReadData materialize() throws IOException { + public SplittableReadData materialize() throws N5IOException { if (materialized == null) read(); @@ -63,7 +64,7 @@ public SplittableReadData materialize() throws IOException { return (SplittableReadData)materialized; } - abstract void read() throws IOException; + abstract void read() throws N5IOException; abstract KeyValueAccessSplittableReadData readOperationSlice(long offset, long length) throws IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index e0fdde456..3770c6b7b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -31,11 +31,10 @@ import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import java.io.IOException; - @CompressionType("lz4") @NameConfig.Name("lz4") public class Lz4Compression implements Compression { @@ -66,7 +65,7 @@ public boolean equals(final Object other) { } @Override - public ReadData decode(final ReadData readData) throws IOException { + public ReadData decode(final ReadData readData) throws N5IOException { return ReadData.from(new LZ4BlockInputStream(readData.inputStream())); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index bae52b688..ca8b4e945 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -28,13 +28,15 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; +import java.io.IOException; + @CompressionType("xz") @NameConfig.Name("xz") public class XzCompression implements Compression { @@ -65,9 +67,13 @@ public boolean equals(final Object other) { } @Override - public ReadData decode(final ReadData readData) throws IOException { + public ReadData decode(final ReadData readData) throws N5IOException { - return ReadData.from(new XZCompressorInputStream(readData.inputStream())); + try { + return ReadData.from(new XZCompressorInputStream(readData.inputStream())); + } catch (IOException e) { + throw new N5IOException(e); + } } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java index 0cc2bdd19..2ea5bda57 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java @@ -1,12 +1,11 @@ package org.janelia.saalfeldlab.n5.codec; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.function.BiConsumer; import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @@ -56,7 +55,7 @@ public DataType getEncodedDataType() { return encodedType; } - @Override public ReadData encode(ReadData readData) throws IOException { + @Override public ReadData encode(ReadData readData) { numBytes = bytes(dataType); @@ -74,7 +73,7 @@ public DataType getEncodedDataType() { }); } - @Override public ReadData decode(ReadData readData) throws IOException { + @Override public ReadData decode(ReadData readData) throws N5IOException { return ReadData.from(out -> { numBytes = bytes(dataType); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index d9e6178be..e1c5f2ee8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -2,6 +2,7 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @@ -36,7 +37,7 @@ interface BytesCodec extends Codec { * @return decoded ReadData * @throws IOException if any I/O error occurs */ - ReadData decode(ReadData readData) throws IOException; + ReadData decode(ReadData readData) throws N5IOException; /** * Encode the given {@code readData}. @@ -48,7 +49,7 @@ interface BytesCodec extends Codec { * @return encoded ReadData * @throws IOException if any I/O error occurs */ - ReadData encode(ReadData readData) throws IOException; + ReadData encode(ReadData readData) throws N5IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java index 1507db6ea..ef30963fa 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java @@ -1,9 +1,9 @@ package org.janelia.saalfeldlab.n5.codec; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; -import java.io.IOException; - class ConcatenatedBytesCodec implements Codec.BytesCodec { private final BytesCodec[] codecs; @@ -12,7 +12,7 @@ class ConcatenatedBytesCodec implements Codec.BytesCodec { this.codecs = codecs; } - @Override public ReadData encode(ReadData readData) throws IOException { + @Override public ReadData encode(ReadData readData) throws N5IOException { ReadData encodeData = readData; if (codecs != null) { @@ -23,7 +23,7 @@ class ConcatenatedBytesCodec implements Codec.BytesCodec { return encodeData; } - @Override public ReadData decode(ReadData readData) throws IOException { + @Override public ReadData decode(ReadData readData) throws N5IOException { ReadData decodeData = readData; if (codecs != null) { for (int i = codecs.length - 1; i >= 0; i--) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java index abe2e3ab3..fa4ff1fc9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java @@ -28,8 +28,9 @@ */ package org.janelia.saalfeldlab.n5.codec; -import java.io.IOException; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** @@ -41,7 +42,7 @@ //TODO Caleb: replace with ArrayCodec? Or use for Factory naming? public interface DataBlockCodec { - ReadData encode(DataBlock dataBlock) throws IOException; + ReadData encode(DataBlock dataBlock) throws N5IOException; - DataBlock decode(ReadData readData, long[] gridPosition) throws IOException; + DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java index af5d3a455..94bb14c45 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -38,6 +38,8 @@ import java.util.Arrays; import java.util.function.IntFunction; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** @@ -53,9 +55,9 @@ */ public abstract class DataCodec { - public abstract ReadData serialize(T data) throws IOException; + public abstract ReadData serialize(T data) throws N5IOException; - public abstract T deserialize(ReadData readData, int numElements) throws IOException; + public abstract T deserialize(ReadData readData, int numElements) throws N5IOException; public int bytesPerElement() { return bytesPerElement; @@ -128,9 +130,13 @@ public ReadData serialize(final byte[] data) { } @Override - public byte[] deserialize(final ReadData readData, int numElements) throws IOException { + public byte[] deserialize(final ReadData readData, int numElements) throws N5IOException { final byte[] data = createData(numElements); - new DataInputStream(readData.inputStream()).readFully(data); + try { + new DataInputStream(readData.inputStream()).readFully(data); + } catch (IOException e) { + throw new N5IOException(e); + } return data; } } @@ -145,14 +151,14 @@ private static final class ShortDataCodec extends DataCodec { } @Override - public ReadData serialize(final short[] data) { + public ReadData serialize(final short[] data) throws N5IOException { final ByteBuffer serialized = ByteBuffer.allocate(Short.BYTES * data.length); serialized.order(order).asShortBuffer().put(data); return ReadData.from(serialized); } @Override - public short[] deserialize(final ReadData readData, int numElements) throws IOException { + public short[] deserialize(final ReadData readData, int numElements) throws N5IOException { final short[] data = createData(numElements); readData.toByteBuffer().order(order).asShortBuffer().get(data); return data; @@ -169,14 +175,14 @@ private static final class IntDataCodec extends DataCodec { } @Override - public ReadData serialize(final int[] data) { + public ReadData serialize(final int[] data) throws N5IOException { final ByteBuffer serialized = ByteBuffer.allocate(Integer.BYTES * data.length); serialized.order(order).asIntBuffer().put(data); return ReadData.from(serialized); } @Override - public int[] deserialize(final ReadData readData, int numElements) throws IOException { + public int[] deserialize(final ReadData readData, int numElements) throws N5IOException { final int[] data = createData(numElements); final ByteBuffer byteBuffer = readData.toByteBuffer(); final IntBuffer intBuffer = byteBuffer.order(order).asIntBuffer(); @@ -195,14 +201,14 @@ private static final class LongDataCodec extends DataCodec { } @Override - public ReadData serialize(final long[] data) { + public ReadData serialize(final long[] data) throws N5IOException { final ByteBuffer serialized = ByteBuffer.allocate(Long.BYTES * data.length); serialized.order(order).asLongBuffer().put(data); return ReadData.from(serialized); } @Override - public long[] deserialize(final ReadData readData, int numElements) throws IOException { + public long[] deserialize(final ReadData readData, int numElements) throws N5IOException { final long[] data = createData(numElements); readData.toByteBuffer().order(order).asLongBuffer().get(data); return data; @@ -219,14 +225,14 @@ private static final class FloatDataCodec extends DataCodec { } @Override - public ReadData serialize(final float[] data) { + public ReadData serialize(final float[] data) throws N5IOException { final ByteBuffer serialized = ByteBuffer.allocate(Float.BYTES * data.length); serialized.order(order).asFloatBuffer().put(data); return ReadData.from(serialized); } @Override - public float[] deserialize(final ReadData readData, int numElements) throws IOException { + public float[] deserialize(final ReadData readData, int numElements) throws N5IOException { final float[] data = createData(numElements); readData.toByteBuffer().order(order).asFloatBuffer().get(data); return data; @@ -243,14 +249,14 @@ private static final class DoubleDataCodec extends DataCodec { } @Override - public ReadData serialize(final double[] data) { + public ReadData serialize(final double[] data) throws N5IOException { final ByteBuffer serialized = ByteBuffer.allocate(Double.BYTES * data.length); serialized.order(order).asDoubleBuffer().put(data); return ReadData.from(serialized); } @Override - public double[] deserialize(final ReadData readData, int numElements) throws IOException { + public double[] deserialize(final ReadData readData, int numElements) throws N5IOException { final double[] data = createData(numElements); readData.toByteBuffer().order(order).asDoubleBuffer().get(data); return data; @@ -267,13 +273,13 @@ private static final class N5StringDataCodec extends DataCodec { } @Override - public ReadData serialize(String[] data) { + public ReadData serialize(String[] data) throws N5IOException { final String flattenedArray = String.join(NULLCHAR, data) + NULLCHAR; return ReadData.from(flattenedArray.getBytes(ENCODING)); } @Override - public String[] deserialize(ReadData readData, int numElements) throws IOException { + public String[] deserialize(ReadData readData, int numElements) throws N5IOException { final byte[] serializedData = readData.allBytes(); final String rawChars = new String(serializedData, ENCODING); return rawChars.split(NULLCHAR); @@ -289,7 +295,7 @@ private static final class ZarrStringDataCodec extends DataCodec { } @Override - public ReadData serialize(String[] data) { + public ReadData serialize(String[] data) throws N5IOException { final int N = data.length; final byte[][] encodedStrings = Arrays.stream(data).map(str -> str.getBytes(ENCODING)).toArray(byte[][]::new); final int[] lengths = Arrays.stream(encodedStrings).mapToInt(a -> a.length).toArray(); @@ -305,7 +311,7 @@ public ReadData serialize(String[] data) { } @Override - public String[] deserialize(ReadData readData, int numElements) throws IOException { + public String[] deserialize(ReadData readData, int numElements) throws N5IOException { final ByteBuffer serialized = readData.toByteBuffer(); serialized.order(ByteOrder.LITTLE_ENDIAN); @@ -335,14 +341,18 @@ private static final class ObjectDataCodec extends DataCodec { } @Override - public ReadData serialize(byte[] data) { + public ReadData serialize(byte[] data) throws N5IOException { return ReadData.from(data); } @Override - public byte[] deserialize(ReadData readData, int numElements) throws IOException { + public byte[] deserialize(ReadData readData, int numElements) throws N5IOException { final byte[] data = createData(numElements); - new DataInputStream(readData.inputStream()).readFully(data); + try { + new DataInputStream(readData.inputStream()).readFully(data); + } catch (IOException e) { + throw new N5IOException(e); + } return data; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java index f5cde7ec0..33c90f0af 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java @@ -1,11 +1,11 @@ package org.janelia.saalfeldlab.n5.codec; import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; import java.util.function.BiConsumer; @@ -95,14 +95,14 @@ public String getType() { return TYPE; } - @Override public ReadData decode(ReadData readData) throws IOException { + @Override public ReadData decode(ReadData readData) throws N5IOException { numBytes = bytes(dataType); numEncodedBytes = bytes(encodedType); return ReadData.from(new FixedLengthConvertedInputStream(numEncodedBytes, numBytes, this.decoder, readData.inputStream())); } - @Override public ReadData encode(ReadData readData) throws IOException { + @Override public ReadData encode(ReadData readData) { return readData.encode(out -> { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java index 2050f593f..fcc368e15 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -1,9 +1,6 @@ package org.janelia.saalfeldlab.n5.codec; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - +import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @@ -20,12 +17,12 @@ public String getType() { return TYPE; } - @Override public ReadData decode(ReadData readData) throws IOException { + @Override public ReadData decode(ReadData readData) { return readData; } - @Override public ReadData encode(ReadData readData) throws IOException { + @Override public ReadData encode(ReadData readData) { return readData; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java index a9d28397e..e43f9255f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java @@ -2,11 +2,11 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import java.io.IOException; - @NameConfig.Name(value = N5BlockCodec.TYPE) public class N5BlockCodec implements Codec.ArrayCodec { @@ -27,12 +27,12 @@ public class N5BlockCodec implements Codec.ArrayCodec { this.attributes = attributes; } - @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException { return dataBlockCodec.decode(readData, gridPosition); } - @Override public ReadData encode(DataBlock dataBlock) throws IOException { + @Override public ReadData encode(DataBlock dataBlock) throws N5IOException { return dataBlockCodec.encode(dataBlock); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index 937a4d23e..c03e464b3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -42,10 +42,12 @@ import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; import org.janelia.saalfeldlab.n5.IntArrayDataBlock; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; import org.janelia.saalfeldlab.n5.StringDataBlock; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import static org.janelia.saalfeldlab.n5.N5Exception.*; import static org.janelia.saalfeldlab.n5.codec.N5Codecs.BlockHeader.MODE_DEFAULT; import static org.janelia.saalfeldlab.n5.codec.N5Codecs.BlockHeader.MODE_OBJECT; import static org.janelia.saalfeldlab.n5.codec.N5Codecs.BlockHeader.MODE_VARLENGTH; @@ -136,9 +138,9 @@ abstract static class AbstractDataBlockCodec implements DataBlockCodec { } - abstract BlockHeader createBlockHeader(final DataBlock dataBlock, ReadData blockData) throws IOException; + abstract BlockHeader createBlockHeader(final DataBlock dataBlock, ReadData blockData) throws N5IOException; - @Override public ReadData encode(DataBlock dataBlock) throws IOException { + @Override public ReadData encode(DataBlock dataBlock) throws N5IOException { return ReadData.from(out -> { final ReadData dataReadData = dataCodec.serialize(dataBlock.getData()); final ReadData encodedData = codec.encode(dataReadData); @@ -149,10 +151,10 @@ abstract static class AbstractDataBlockCodec implements DataBlockCodec { }); } - abstract BlockHeader decodeBlockHeader(final InputStream in) throws IOException; + abstract BlockHeader decodeBlockHeader(final InputStream in) throws N5IOException; @Override - public DataBlock decode(final ReadData readData, final long[] gridPosition) throws IOException { + public DataBlock decode(final ReadData readData, final long[] gridPosition) throws N5IOException { try(final InputStream in = readData.inputStream()) { final BlockHeader header = decodeBlockHeader(in); @@ -167,6 +169,8 @@ public DataBlock decode(final ReadData readData, final long[] gridPosition) t final ReadData decodeData = codec.decode(blockData); final T data = dataCodec.deserialize(decodeData, numElements); return dataBlockFactory.createDataBlock(header.blockSize(), gridPosition, data); + } catch (IOException e) { + throw new N5IOException(e); } } } @@ -190,7 +194,7 @@ protected BlockHeader createBlockHeader(final DataBlock dataBlock, ReadData b } @Override - protected BlockHeader decodeBlockHeader(final InputStream in) throws IOException { + protected BlockHeader decodeBlockHeader(final InputStream in) throws N5IOException { return BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); } @@ -207,13 +211,13 @@ private static class StringDataBlockCodec extends AbstractDataBlockCodec dataBlock, ReadData blockData) throws IOException { + protected BlockHeader createBlockHeader(final DataBlock dataBlock, ReadData blockData) throws N5IOException { return new BlockHeader(dataBlock.getSize(), (int)blockData.length()); } @Override - protected BlockHeader decodeBlockHeader(final InputStream in) throws IOException { + protected BlockHeader decodeBlockHeader(final InputStream in) throws N5IOException { return BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); } @@ -236,7 +240,7 @@ protected BlockHeader createBlockHeader(DataBlock dataBlock, ReadData bl } @Override - protected BlockHeader decodeBlockHeader(final InputStream in) throws IOException { + protected BlockHeader decodeBlockHeader(final InputStream in) throws N5IOException { return BlockHeader.readFrom(in, MODE_OBJECT); } @@ -296,78 +300,95 @@ public int numElements() { return numElements; } - private static int[] readBlockSize(final DataInputStream dis) throws IOException { + private static int[] readBlockSize(final DataInputStream dis) throws N5IOException { - final int nDim = dis.readShort(); - final int[] blockSize = new int[nDim]; - for (int d = 0; d < nDim; ++d) - blockSize[d] = dis.readInt(); - return blockSize; - } - - private static void writeBlockSize(final int[] blockSize, final DataOutputStream dos) throws IOException { - - dos.writeShort(blockSize.length); - for (final int size : blockSize) - dos.writeInt(size); + try { + final int nDim = dis.readShort(); + final int[] blockSize = new int[nDim]; + for (int d = 0; d < nDim; ++d) + blockSize[d] = dis.readInt(); + return blockSize; + } catch (IOException e) { + throw new N5IOException(e); + } } - void writeTo(final OutputStream out) throws IOException { + private static void writeBlockSize(final int[] blockSize, final DataOutputStream dos) throws N5IOException { - final DataOutputStream dos = new DataOutputStream(out); - dos.writeShort(mode); - switch (mode) { - case MODE_DEFAULT:// default - writeBlockSize(blockSize, dos); - break; - case MODE_VARLENGTH:// varlength - writeBlockSize(blockSize, dos); - dos.writeInt(numElements); - break; - case MODE_OBJECT: // object - dos.writeInt(numElements); - break; - default: - throw new IOException("unexpected mode: " + mode); + try { + dos.writeShort(blockSize.length); + for (final int size : blockSize) + dos.writeInt(size); + } catch (IOException e) { + throw new N5IOException(e); } - dos.flush(); } - static BlockHeader readFrom(final InputStream in, short... allowedModes) throws IOException { + void writeTo(final OutputStream out) throws N5IOException { - final DataInputStream dis = new DataInputStream(in); - final short mode = dis.readShort(); - final int[] blockSize; - final int numElements; - switch (mode) { - case MODE_DEFAULT:// default - blockSize = readBlockSize(dis); - numElements = DataBlock.getNumElements(blockSize); - break; - case MODE_VARLENGTH:// varlength - blockSize = readBlockSize(dis); - numElements = dis.readInt(); - break; - case MODE_OBJECT: // object - blockSize = null; - numElements = dis.readInt(); - break; - default: - throw new IOException("Unexpected mode: " + mode); + try { + final DataOutputStream dos = new DataOutputStream(out); + dos.writeShort(mode); + switch (mode) { + case MODE_DEFAULT:// default + writeBlockSize(blockSize, dos); + break; + case MODE_VARLENGTH:// varlength + writeBlockSize(blockSize, dos); + dos.writeInt(numElements); + break; + case MODE_OBJECT: // object + dos.writeInt(numElements); + break; + default: + throw new N5Exception("unexpected mode: " + mode); + } + dos.flush(); + } catch (IOException e) { + throw new N5IOException(e); } - boolean modeIsOk = allowedModes == null || allowedModes.length == 0; - for (int i = 0; !modeIsOk && i < allowedModes.length; ++i) { - if (mode == allowedModes[i]) { - modeIsOk = true; + } + + static BlockHeader readFrom(final InputStream in, short... allowedModes) throws N5IOException, N5Exception { + + try { + final DataInputStream dis = new DataInputStream(in); + final short mode = dis.readShort(); + final int[] blockSize; + final int numElements; + switch (mode) { + case MODE_DEFAULT:// default + blockSize = readBlockSize(dis); + numElements = DataBlock.getNumElements(blockSize); + break; + case MODE_VARLENGTH:// varlength + blockSize = readBlockSize(dis); + numElements = dis.readInt(); break; + case MODE_OBJECT: // object + blockSize = null; + numElements = dis.readInt(); + break; + default: + throw new N5Exception("Unexpected mode: " + mode); } - } - if (!modeIsOk) { - throw new IOException("Unexpected mode: " + mode); - } - return new BlockHeader(mode, blockSize, numElements); + boolean modeIsOk = allowedModes == null || allowedModes.length == 0; + for (int i = 0; !modeIsOk && i < allowedModes.length; ++i) { + if (mode == allowedModes[i]) { + modeIsOk = true; + break; + } + } + if (!modeIsOk) { + throw new N5Exception("Unexpected mode: " + mode); + } + + return new BlockHeader(mode, blockSize, numElements); + } catch (IOException e) { + throw new N5IOException(e); + } } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java index 49493ad12..389e7a116 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java @@ -7,6 +7,7 @@ import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; import org.janelia.saalfeldlab.n5.IntArrayDataBlock; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; import org.janelia.saalfeldlab.n5.StringDataBlock; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -107,12 +108,12 @@ private int numElements() { }); } - @Override N5Codecs.BlockHeader decodeBlockHeader(InputStream in) throws IOException { + @Override N5Codecs.BlockHeader decodeBlockHeader(InputStream in) { return null; } - @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException { try (final InputStream in = readData.inputStream()) { final int bytesPerElement = dataCodec.bytesPerElement(); @@ -121,6 +122,8 @@ private int numElements() { final T data = dataCodec.deserialize(decodeData, numElements()); return dataBlockFactory.createDataBlock(blockSize, gridPosition, data); + } catch (IOException e) { + throw new N5IOException(e); } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java index 22f844697..9a0d7f1eb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java @@ -10,10 +10,11 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import java.io.IOException; import java.nio.ByteOrder; @@ -49,12 +50,12 @@ public ByteOrder getByteOrder() { this.dataBlockCodec = RawBlockCodecs.createDataBlockCodec(attributes.getDataType(), getByteOrder(), attributes.getBlockSize(), concatenatedBytesCodec); } - @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException { return dataBlockCodec.decode(readData, gridPosition); } - @Override public ReadData encode(DataBlock dataBlock) throws IOException { + @Override public ReadData encode(DataBlock dataBlock) throws N5IOException { return dataBlockCodec.encode(dataBlock); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java index 88a6d8534..afc78beb9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java @@ -8,6 +8,8 @@ import java.util.zip.CheckedOutputStream; import java.util.zip.Checksum; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; @@ -40,8 +42,8 @@ public int numChecksumBytes() { return numChecksumBytes; } - private CheckedOutputStream createStream(OutputStream out) throws IOException { - return new CheckedOutputStream(out, getChecksum()) { + private CheckedOutputStream createStream(OutputStream out) { + return new CheckedOutputStream(out, getChecksum()) { private boolean closed = false; @Override public void close() throws IOException { @@ -55,13 +57,13 @@ private CheckedOutputStream createStream(OutputStream out) throws IOException { }; } - @Override public ReadData encode(ReadData readData) throws IOException { + @Override public ReadData encode(ReadData readData) { return readData.encode(this::createStream); } - @Override public ReadData decode(ReadData readData) throws IOException { + @Override public ReadData decode(ReadData readData) throws N5IOException { return ReadData.from(new CheckedInputStream(readData.inputStream(), getChecksum())); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java index 0a1b8173d..21307ba14 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java @@ -33,6 +33,8 @@ import java.io.InputStream; import org.apache.commons.io.IOUtils; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; // not thread-safe abstract class AbstractInputStreamReadData implements ReadData { @@ -40,7 +42,7 @@ abstract class AbstractInputStreamReadData implements ReadData { private ByteArraySplittableReadData bytes; @Override - public SplittableReadData materialize() throws IOException { + public SplittableReadData materialize() throws N5IOException { if (bytes == null) { final byte[] data; final int length = (int) length(); @@ -48,10 +50,14 @@ public SplittableReadData materialize() throws IOException { data = new byte[length]; try( InputStream is = inputStream()) { new DataInputStream(is).readFully(data); + } catch (IOException e) { + throw new N5IOException(e); } } else { try( InputStream is = inputStream()) { data = IOUtils.toByteArray(is); + } catch (IOException e) { + throw new N5IOException(e); } } bytes = new ByteArraySplittableReadData(data); @@ -60,7 +66,7 @@ public SplittableReadData materialize() throws IOException { } @Override - public byte[] allBytes() throws IOException, IllegalStateException { + public byte[] allBytes() throws N5IOException, IllegalStateException { return materialize().allBytes(); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java index 9bf86ef9c..e34a144bb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java @@ -35,6 +35,8 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; class ByteArraySplittableReadData implements SplittableReadData { @@ -61,7 +63,7 @@ public long length() { } @Override - public InputStream inputStream() throws IOException { + public InputStream inputStream() { return new ByteArrayInputStream(data, offset, length); } @@ -77,7 +79,7 @@ public byte[] allBytes() { } @Override - public SplittableReadData materialize() throws IOException { + public SplittableReadData materialize() { return this; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java index a79a55b5c..036811430 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java @@ -33,6 +33,8 @@ import org.apache.commons.io.input.ProxyInputStream; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; class KeyValueAccessReadData extends AbstractInputStreamReadData { @@ -53,19 +55,24 @@ class KeyValueAccessReadData extends AbstractInputStreamReadData { * * @return an InputStream on this data * - * @throws IOException + * @throws N5IOException * if any I/O error occurs */ @Override - public InputStream inputStream() throws IOException { - final LockedChannel channel = keyValueAccess.lockForReading(normalPath); - return new ProxyInputStream(channel.newInputStream()) { + public InputStream inputStream() throws N5IOException { + try { + @SuppressWarnings("resource") + final LockedChannel channel = keyValueAccess.lockForReading(normalPath); + return new ProxyInputStream(channel.newInputStream()) { - @Override - public void close() throws IOException { - in.close(); - channel.close(); - } - }; + @Override + public void close() throws IOException { + in.close(); + channel.close(); + } + }; + } catch (final IOException e) { + throw new N5IOException(e); + } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java index 0420ee4a7..fd1d2e269 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -34,6 +34,7 @@ import java.io.OutputStream; import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; class LazyReadData implements ReadData { @@ -63,7 +64,7 @@ class LazyReadData implements ReadData { private ByteArraySplittableReadData bytes; @Override - public SplittableReadData materialize() throws IOException { + public SplittableReadData materialize() throws N5IOException { if (bytes == null) { final ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); writeTo(baos); @@ -73,31 +74,31 @@ public SplittableReadData materialize() throws IOException { } @Override - public long length() { + public long length() throws N5IOException { - try { - return materialize().length(); - } catch (IOException e) { - throw new N5Exception.N5IOException(e); - } + return materialize().length(); } @Override - public InputStream inputStream() throws IOException, IllegalStateException { + public InputStream inputStream() throws N5IOException, IllegalStateException { return materialize().inputStream(); } @Override - public byte[] allBytes() throws IOException, IllegalStateException { + public byte[] allBytes() throws N5IOException, IllegalStateException { return materialize().allBytes(); } @Override - public void writeTo(final OutputStream outputStream) throws IOException, IllegalStateException { - if (bytes != null) { - outputStream.write(bytes.allBytes()); - } else { - writer.writeTo(outputStream); + public void writeTo(final OutputStream outputStream) throws N5IOException, IllegalStateException { + try { + if (bytes != null) { + outputStream.write(bytes.allBytes()); + } else { + writer.writeTo(outputStream); + } + } catch (IOException e) { + throw new N5IOException(e); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 1b5079d8c..f717e15ac 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -33,6 +33,7 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; /** * An abstraction over {@code byte[]} data. @@ -59,11 +60,10 @@ public interface ReadData { * * @return number of bytes, if known, or -1 * - * @throws IOException + * @throws N5IOException * if an I/O error occurs while trying to get the length */ - //TODO N5IOException - default long length() { + default long length() throws N5IOException { return -1; } @@ -78,12 +78,12 @@ default long length() { * * @return an InputStream on this data * - * @throws IOException + * @throws N5IOException * if any I/O error occurs * @throws IllegalStateException * if this method was already called once and cannot be called again. */ - InputStream inputStream() throws IOException, IllegalStateException; + InputStream inputStream() throws N5IOException, IllegalStateException; /** * Return the contained data as a {@code byte[]} array. @@ -97,12 +97,12 @@ default long length() { * * @return all contained data as a byte[] array * - * @throws IOException + * @throws N5IOException * if any I/O error occurs * @throws IllegalStateException * if {@link #inputStream()} was already called once and cannot be called again. */ - byte[] allBytes() throws IOException, IllegalStateException; + byte[] allBytes() throws N5IOException, IllegalStateException; /** * Return the contained data as a {@code ByteBuffer}. @@ -117,12 +117,12 @@ default long length() { * * @return all contained data as a ByteBuffer * - * @throws IOException + * @throws N5IOException * if any I/O error occurs * @throws IllegalStateException * if {@link #inputStream()} was already called once and cannot be called again. */ - default ByteBuffer toByteBuffer() throws IOException, IllegalStateException { + default ByteBuffer toByteBuffer() throws N5IOException, IllegalStateException { return ByteBuffer.wrap(allBytes()); } @@ -134,7 +134,7 @@ default ByteBuffer toByteBuffer() throws IOException, IllegalStateException { * The returned {@code ReadData} has a known {@link #length} and multiple * {@link #inputStream InputStreams} can be opened on it. */ - SplittableReadData materialize() throws IOException; + SplittableReadData materialize() throws N5IOException; /** * Write the contained data into an {@code OutputStream}. @@ -149,13 +149,17 @@ default ByteBuffer toByteBuffer() throws IOException, IllegalStateException { * @param outputStream * destination to write to * - * @throws IOException + * @throws N5IOException * if any I/O error occurs * @throws IllegalStateException * if {@link #inputStream()} was already called once and cannot be called again. */ - default void writeTo(OutputStream outputStream) throws IOException, IllegalStateException { - outputStream.write(allBytes()); + default void writeTo(OutputStream outputStream) throws N5IOException, IllegalStateException { + try { + outputStream.write(allBytes()); + } catch (IOException e) { + throw new N5IOException(e); + } } // ------------- Encoding / Decoding ---------------- diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadDataSplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadDataSplittableReadData.java index 58add721f..d5cfd3ea5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadDataSplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadDataSplittableReadData.java @@ -32,6 +32,7 @@ import org.apache.commons.io.input.ProxyInputStream; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import java.io.IOException; import java.io.InputStream; @@ -57,7 +58,7 @@ public long length() { } @Override - public InputStream inputStream() throws IOException { + public InputStream inputStream() throws N5IOException { final InputStream offsetInputStream = new ProxyInputStream(readData.inputStream()) { @@ -75,14 +76,18 @@ public InputStream inputStream() throws IOException { } }; - return BoundedInputStream.builder() - .setInputStream(offsetInputStream) - .setBufferSize((int)length) - .get(); + try { + return BoundedInputStream.builder() + .setInputStream(offsetInputStream) + .setBufferSize((int)length) + .get(); + } catch (IOException e) { + throw new N5IOException(e); + } } @Override - public byte[] allBytes() throws IOException, IllegalStateException { + public byte[] allBytes() throws N5IOException, IllegalStateException { final byte[] bytes = readData.allBytes(); if (offset == 0 && bytes.length == length) { @@ -93,7 +98,7 @@ public byte[] allBytes() throws IOException, IllegalStateException { } @Override - public SplittableReadData materialize() throws IOException { + public SplittableReadData materialize() { return this; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 8b055fb9e..860bce62a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -14,6 +14,8 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.util.GridIterator; +import static org.janelia.saalfeldlab.n5.N5Exception.*; + public interface Shard extends Iterable> { /** @@ -135,7 +137,7 @@ default Iterator blockPositionIterator() { ShardIndex getIndex(); - default ReadData createReadData() throws IOException { + default ReadData createReadData() throws N5IOException { final DatasetAttributes datasetAttributes = getDatasetAttributes(); final ShardIndex index = ShardIndex.createIndex(datasetAttributes); @@ -189,7 +191,7 @@ class DataBlockIterator implements Iterator> { private final Shard shard; private final ShardIndex index; // TODO ShardParameters is deprecated? - private final ShardParameters attributes; + private final DatasetAttributes attributes; private int blockIndex = 0; public DataBlockIterator(final Shard shard) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 66f423b34..13593aa82 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -182,7 +182,7 @@ public static boolean read( public static void write( final OutputStream outputStream, final ShardIndex index - ) throws IOException { + ) throws N5IOException { try { write(index, outputStream); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index b45781c78..5de495639 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -9,6 +9,8 @@ import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -16,7 +18,6 @@ import org.janelia.saalfeldlab.n5.serialization.N5Annotations; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import java.io.IOException; import java.lang.reflect.Type; import java.util.Objects; @@ -120,12 +121,12 @@ public DeterministicSizeCodec[] getIndexCodecs() { getArrayCodec().initialize(attributes, getCodecs()); } - @Override public ReadData encode(DataBlock dataBlock) throws IOException { + @Override public ReadData encode(DataBlock dataBlock) throws N5IOException { return getArrayCodec().encode(dataBlock); } - @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException { final SplittableReadData splitableReadData = readData.materialize(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/CodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/CodecTests.java index 4fbdc5381..2a6883dbe 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/CodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/CodecTests.java @@ -5,10 +5,10 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.Arrays; import java.util.Random; import java.util.function.IntUnaryOperator; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.ReadData.OutputStreamOperator; @@ -68,11 +68,11 @@ public String getType() { return "byteFunction"; } - public ReadData decode(ReadData data) throws IOException { + public ReadData decode(ReadData data) { return data.encode(new ByteFun(decoder)); } - public ReadData encode(ReadData data) throws IOException { + public ReadData encode(ReadData data) { return data.encode(new ByteFun(encoder)); } } @@ -85,7 +85,7 @@ public ByteFun(IntUnaryOperator fun) { } @Override - public OutputStream apply(OutputStream o) throws IOException { + public OutputStream apply(OutputStream o) { return new OutputStream() { @Override public void write(int b) throws IOException { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index 58c8113af..1b1e42433 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -17,6 +17,8 @@ import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.lang3.tuple.Pair; import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData.OutputStreamOperator; import org.junit.Test; @@ -158,7 +160,7 @@ public ByteFun(IntUnaryOperator fun) { } @Override - public OutputStream apply(OutputStream o) throws IOException { + public OutputStream apply(OutputStream o) { return new OutputStream() { @Override public void write(int b) throws IOException { From 214466087b3a735fc1de492798236c9bffea3629 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 10 Jun 2025 16:17:46 -0400 Subject: [PATCH 243/423] feat: support writing blocks through shards on non-sharded datasets --- .../janelia/saalfeldlab/n5/Compression.java | 1 - .../saalfeldlab/n5/DatasetAttributes.java | 6 +- .../saalfeldlab/n5/NameConfigAdapter.java | 7 ++ .../saalfeldlab/n5/shard/AbstractShard.java | 11 +- .../n5/shard/BlockAsShardCodec.java | 112 ++++++++++++++++++ .../saalfeldlab/n5/shard/InMemoryShard.java | 31 +++-- .../janelia/saalfeldlab/n5/shard/Shard.java | 12 ++ .../saalfeldlab/n5/shard/ShardIndex.java | 58 +++------ .../n5/shard/ShardIndexBuilder.java | 86 -------------- .../saalfeldlab/n5/shard/ShardParameters.java | 1 - .../saalfeldlab/n5/shard/VirtualShard.java | 24 +--- .../saalfeldlab/n5/AbstractN5Test.java | 44 +++++++ .../saalfeldlab/n5/shard/ShardTest.java | 2 +- 13 files changed, 222 insertions(+), 173 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index 627c13a70..0f0632016 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -28,7 +28,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; import java.io.Serializable; import java.lang.annotation.ElementType; import java.lang.annotation.Inherited; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index bf3321d51..f5094e439 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -39,6 +39,7 @@ import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; +import org.janelia.saalfeldlab.n5.shard.BlockAsShardCodec; import org.janelia.saalfeldlab.n5.shard.ShardParameters; import com.google.gson.JsonDeserializationContext; @@ -188,8 +189,9 @@ public boolean isSharded() { public ShardingCodec getShardingCodec() throws N5ShardException { if (getArrayCodec() instanceof ShardingCodec) return (ShardingCodec)getArrayCodec(); - else - throw new N5ShardException("Dataset is not Sharded"); + else { + return new BlockAsShardCodec<>(getArrayCodec()); + } } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java index 7fd05a262..d957a92a3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java @@ -38,6 +38,7 @@ import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.serialization.N5Annotations; import org.janelia.saalfeldlab.n5.serialization.NameConfig; +import org.janelia.saalfeldlab.n5.shard.BlockAsShardCodec; import org.scijava.annotations.Index; import org.scijava.annotations.IndexItem; @@ -87,6 +88,12 @@ public static synchronized void update(final NameConfigAdapter adapter) { Class clazz; try { clazz = (Class)Class.forName(item.className()); + + /* BlockAsShardCodec is not serializable; it is an internal memory-only + * interpretation codec for interacting with un-sharded datasets as sharded datasets. */ + if (clazz == BlockAsShardCodec.class) + continue; + final String name = clazz.getAnnotation(NameConfig.Name.class).value(); final String type = prefix + "." + name; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java index 4e2ac8319..9aa81b740 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java @@ -11,7 +11,9 @@ public abstract class AbstractShard implements Shard { private final long[] gridPosition; - public AbstractShard(final DatasetAttributes datasetAttributes, final long[] gridPosition, + public AbstractShard( + final DatasetAttributes datasetAttributes, + final long[] gridPosition, final ShardIndex index) { this.datasetAttributes = datasetAttributes; @@ -42,11 +44,4 @@ public long[] getGridPosition() { return gridPosition; } - - @Override - public ShardIndex getIndex() { - - return index; - } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java new file mode 100644 index 000000000..e5d64f0d6 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java @@ -0,0 +1,112 @@ +package org.janelia.saalfeldlab.n5.shard; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.codec.RawBytes; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +public class BlockAsShardCodec extends ShardingCodec { + + private static final RawBytes NO_OP_ARRAY_CODEC = new RawBytes() { + + @Override public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { + + return ReadData.from(new byte[0]); + } + + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException { + + throw new UnsupportedOperationException(" NO_OP_ARRAY_CODEC is used for `encode` only. "); + } + }; + + private static final BytesCodec[] EMPTY_SHARD_CODECS = new BytesCodec[0]; + private static final DeterministicSizeCodec[] NO_OP_INDEX_CODECS = new DeterministicSizeCodec[]{NO_OP_ARRAY_CODEC}; + + final ArrayCodec datasetArrayCodec; + private DatasetAttributes datasetAttributes; + + public BlockAsShardCodec(ArrayCodec datasetArrayCodec) { + + super(null, EMPTY_SHARD_CODECS, NO_OP_INDEX_CODECS, IndexLocation.END); + this.datasetArrayCodec = datasetArrayCodec; + } + + @Override public ShardIndex createIndex(DatasetAttributes attributes) { + + return new ShardIndex(attributes.getBlocksPerShard(), getIndexLocation(), NO_OP_INDEX_CODECS) { + + @Override public void readFrom(ReadData shardData) throws N5Exception.N5IOException { + + if (shardData.length() == -1) + shardData.materialize(); + + final long length = shardData.length(); + if (length == -1) + throw new N5Exception.N5IOException("ReadData for shard index must have a valid length, but was " + length); + + data[0] = 0; + data[1] = length; + } + + @Override public long numBytes() { + + return 0; + } + }; + } + + @Override public int[] getBlockSize() { + + return datasetAttributes.getBlockSize(); + } + + @Override public ArrayCodec getArrayCodec() { + + return datasetArrayCodec; + } + + @Override public long[] getPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { + + return datasetArrayCodec.getPositionForBlock(attributes, datablock); + } + + @Override public long[] getPositionForBlock(DatasetAttributes attributes, long... blockPosition) { + + return datasetArrayCodec.getPositionForBlock(attributes, blockPosition); + } + + @Override public void initialize(DatasetAttributes attributes, BytesCodec... codecs) { + + this.datasetAttributes = attributes; + datasetArrayCodec.initialize(attributes, codecs); + } + + @Override public long encodedSize(long size) { + + return datasetArrayCodec.encodedSize(size); + } + + @Override public long decodedSize(long size) { + + return datasetArrayCodec.decodedSize(size); + } + + @Override public String getType() { + + //TODO Caleb: can we ensure this is never called? + return datasetArrayCodec.getType(); + } + + @Override public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { + + return datasetArrayCodec.encode(dataBlock); + } + + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException { + + return datasetArrayCodec.decode(readData, gridPosition); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 3e05fff08..60f2d5177 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -6,6 +6,7 @@ import org.janelia.saalfeldlab.n5.util.Position; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -15,19 +16,14 @@ public class InMemoryShard extends AbstractShard { /** Map {@link DataBlock#getGridPosition} as hashable {@link Position} to the block */ private final Map> blocks; - private ShardIndexBuilder indexBuilder; - - //TODO delegated shard constructor? Or new class? - public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] shardPosition) { this(datasetAttributes, shardPosition, null); - indexBuilder = new ShardIndexBuilder(this); - final IndexLocation indexLocation = datasetAttributes.getShardingCodec().getIndexLocation(); - indexBuilder.indexLocation(indexLocation); } - public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] gridPosition, + public InMemoryShard( + final DatasetAttributes datasetAttributes, + final long[] gridPosition, ShardIndex index) { super(datasetAttributes, gridPosition, index); @@ -49,9 +45,20 @@ private void storeBlock(DataBlock block) { return blocks.get(Position.wrap(blockGridPosition)); } - public void addBlock(DataBlock block) { + /** + * Add the {@code block} to this shard. If the block is not contained in this shard, do not add it. + * + * @param block to add the shard + * @return whether the block was added + */ + public boolean addBlock(DataBlock block) { + + final long[] shardPositionForBlock = datasetAttributes.getShardPositionForBlock(block.getGridPosition()); + if (!Arrays.equals(shardPositionForBlock, getGridPosition())) + return false; storeBlock(block); + return true; } @Override @@ -63,10 +70,8 @@ public List> getBlocks() { @Override public ShardIndex getIndex() { - if (index != null) - return index; - else - return indexBuilder.build(); + index = index != null ? index : createIndex(); + return index; } public static InMemoryShard fromShard(Shard shard) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 860bce62a..053988fb1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -135,8 +135,20 @@ default Iterator blockPositionIterator() { return new GridIterator(GridIterator.int2long(getBlockGridSize()), min); } + /** + * @return the ShardIndex for this shard, or a new ShardIndex if the Shard is non-existent + */ ShardIndex getIndex(); + /** + * @return and empty index of the correct size for the dataset + */ + default ShardIndex createIndex() { + + final DatasetAttributes datasetAttributes = getDatasetAttributes(); + return datasetAttributes.getShardingCodec().createIndex(datasetAttributes); + } + default ReadData createReadData() throws N5IOException { final DatasetAttributes datasetAttributes = getDatasetAttributes(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 13593aa82..c421e3db7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -12,6 +12,7 @@ import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.SplittableReadData; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import java.io.ByteArrayInputStream; @@ -140,21 +141,25 @@ public long numBytes() { return totalNumBytes; } - public static boolean read(byte[] data, final ShardIndex index) { + public void readFrom(ReadData shardData) throws N5IOException { - final IndexByteBounds byteBounds = byteBounds(index, data.length); - final ByteArrayInputStream is = new ByteArrayInputStream(data); - is.skip(byteBounds.start); + /* we require a length, so materialize if we don't have one. */ + if (shardData.length() == -1) + shardData.materialize(); + final long length = shardData.length(); + if (length == -1) + throw new N5IOException("ReadData for shard index must have a valid length, but was " + length); + + final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(this, length); + final ReadData indexData; try { - BoundedInputStream bIs = BoundedInputStream.builder() - .setInputStream(is) - .setMaxCount(index.numBytes()).get(); - read(bIs, index); - return true; + indexData = ((SplittableReadData)shardData).slice(bounds.start, this.numBytes()); } catch (IOException e) { - return false; + throw new N5IOException("Failed to read shard index", e); } + ShardIndex.read(indexData, this); + } public static void read(InputStream in, final ShardIndex index) throws IOException { @@ -215,8 +220,7 @@ private DatasetAttributes createIndexAttributes() { public static IndexByteBounds byteBounds(DatasetAttributes datasetAttributes, final long objectSize) { - ShardingCodec shardCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); - final ShardIndex index = shardCodec.createIndex(datasetAttributes); + final ShardIndex index = datasetAttributes.getShardingCodec().createIndex(datasetAttributes); final long indexSize = index.numBytes(); return byteBounds(indexSize, index.location, objectSize); @@ -250,31 +254,6 @@ public IndexByteBounds(long start, long end) { } } - //TODO Caleb: Probably don't need to keep this eventually - public static ShardIndex read(FileChannel channel, DatasetAttributes datasetAttributes) throws IOException { - - // TODO need codecs - // TODO FileChannel is too specific - generalize - ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); - final int[] indexShape = prepend(2, datasetAttributes.getBlocksPerShard()); - final int indexSize = (int)Arrays.stream(indexShape).reduce(1, (x, y) -> x * y); - final int indexBytes = BYTES_PER_LONG * indexSize; - - if (shardingCodec.getIndexLocation() == IndexLocation.END) { - channel.position(channel.size() - indexBytes); - } - - final InputStream is = Channels.newInputStream(channel); - final DataInputStream dis = new DataInputStream(is); - - final long[] indexes = new long[indexSize]; - for (int i = 0; i < indexSize; i++) { - indexes[i] = dis.readLong(); - } - - return new ShardIndex(indexShape, indexes, IndexLocation.END); - } - private static long[] emptyIndexData(final int[] size) { final int N = 2 * Arrays.stream(size).reduce(1, (x, y) -> x * y); @@ -312,8 +291,9 @@ public boolean equals(Object other) { public static ShardIndex createIndex(final DatasetAttributes attributes) { - ShardingCodec shardingCodec = (ShardingCodec)attributes.getArrayCodec(); - return shardingCodec.createIndex(attributes); + return attributes.getShardingCodec().createIndex(attributes); } + + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java deleted file mode 100644 index 41d505af0..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndexBuilder.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import java.util.Arrays; - -import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; - -public class ShardIndexBuilder { - - private final Shard shard; - - private ShardIndex temporaryIndex; - - private IndexLocation location = IndexLocation.END; - - private DeterministicSizeCodec[] codecs; - - private long currentOffset = 0; - - public ShardIndexBuilder(Shard shard) { - - this.shard = shard; - this.temporaryIndex = new ShardIndex(shard.getBlockGridSize(), location); - } - - public ShardIndex build() { - - return new ShardIndex( - shard.getBlockGridSize(), - temporaryIndex.getData(), - location, - codecs); - } - - public ShardIndexBuilder indexLocation(IndexLocation location) { - - this.location = location; - this.temporaryIndex = new ShardIndex(shard.getBlockGridSize(), location); - updateInitialOffset(); - return this; - } - - public IndexLocation getLocation() { - - return this.location; - } - - public ShardIndexBuilder setCodecs(DeterministicSizeCodec... codecs) { - - this.codecs = codecs; - final ShardIndex newIndex = new ShardIndex(shard.getBlockGridSize(), temporaryIndex.getLocation(), codecs); - this.temporaryIndex = newIndex; - updateInitialOffset(); - return this; - } - - public ShardIndexBuilder addBlock(long[] blockPosition, long numBytes) { - //TODO Caleb: Maybe move to ShardIndex? - final int[] blockPositionInShard = shard.getDatasetAttributes().getBlockPositionInShard( - shard.getGridPosition(), - blockPosition); - - if (blockPositionInShard == null) { - throw new IllegalArgumentException(String.format( - "The block at position %s is not contained in the shard at position : %s and size : %s )", - Arrays.toString(blockPosition), - Arrays.toString(shard.getGridPosition()), - Arrays.toString(shard.getSize()))); - } - - temporaryIndex.set(currentOffset, numBytes, blockPositionInShard); - currentOffset += numBytes; - - return this; - } - - private void updateInitialOffset() { - - if (location == IndexLocation.END) - currentOffset = 0; - else - currentOffset = temporaryIndex.numBytes(); - - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java index 900dd6f3f..f2f8d734d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java @@ -43,7 +43,6 @@ public interface ShardParameters { default int[] getBlocksPerShard() { final int[] shardSize = getShardSize(); - Objects.requireNonNull(shardSize, "getShardSize() must not be null"); final int nd = getNumDimensions(); final int[] blocksPerShard = new int[nd]; final int[] blockSize = getBlockSize(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index d5cc7fb97..996f1ddd9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -35,8 +35,7 @@ public VirtualShard( @SuppressWarnings("unchecked") public DataBlock getBlock(ReadData blockData, long... blockGridPosition) throws IOException { - ShardingCodec shardingCodec = (ShardingCodec)datasetAttributes.getArrayCodec(); - return shardingCodec.getArrayCodec().decode(blockData, blockGridPosition); + return datasetAttributes.getShardingCodec().getArrayCodec().decode(blockData, blockGridPosition); } @Override @@ -119,35 +118,16 @@ public DataBlock getBlock(long... blockGridPosition) { } } - public ShardIndex createIndex() { - - // Empty index of the correct size - return ((ShardingCodec)getDatasetAttributes().getArrayCodec()).createIndex(getDatasetAttributes()); - } - @Override public ShardIndex getIndex() { //TODO Caleb: How to handle when this shard doesn't exist (splitableData.getSize() <= 0) index = createIndex(); - final ReadData indexData; try { - /* we require a length, so materialize if we don't have one. */ - if (shardData.length() == -1) { - shardData.materialize(); - } - final long length = shardData.length(); - if (length == -1) - throw new N5IOException("ReadData for shard index must have a valid length, but was " + length); - - final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, length); - indexData = shardData.slice(bounds.start, index.numBytes()); + index.readFrom(shardData); } catch (N5Exception.N5NoSuchKeyException e) { return null; - } catch (IOException | UncheckedIOException e) { - throw new N5IOException(e); } - ShardIndex.read(indexData, index); return index; } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index aeaf428bc..3e5f6e103 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -55,11 +55,15 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5ClassCastException; import org.janelia.saalfeldlab.n5.N5Reader.Version; +import org.janelia.saalfeldlab.n5.shard.InMemoryShard; +import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.VirtualShard; import org.janelia.saalfeldlab.n5.url.UriAttributeTest; import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -1683,6 +1687,46 @@ public void testPathsWithIllegalUriCharacters() throws IOException, URISyntaxExc } } + @Test + public void testWriteReadShardOnUnshardedDataset() throws Exception { + try (N5Writer writer = createTempN5Writer()) { + final String datasetName = "testWriteShardOnUnshardedDataset"; + final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64, new RawCompression()); + writer.createDataset(datasetName, datasetAttributes); + + + final DataBlock block0 = (DataBlock) DataType.UINT64.createDataBlock(blockSize, new long[]{0,0,0}); + final DataBlock block1 = (DataBlock) DataType.UINT64.createDataBlock(blockSize, new long[]{1,0,0}); + + final InMemoryShard writeShard = new InMemoryShard<>(datasetAttributes, new long[]{0, 0, 0}); + writeShard.addBlock(block0); + + writeShard.addBlock(block1); + + final List> writeBlocks = writeShard.getBlocks(); + assertEquals("block as shard should not have the second block which is outside this shard",1, writeBlocks.size()); + assertEquals(block1, writeBlocks.get(0)); + + writer.writeShard(datasetName, datasetAttributes, writeShard); + final Shard readShard = writer.readShard(datasetName, datasetAttributes, writeShard.getGridPosition()); + + Assert.assertArrayEquals("shard read position should be same as write position", writeShard.getGridPosition(), readShard.getGridPosition()); + Assert.assertArrayEquals("shard position should be the same as block position when unsharded", block0.getGridPosition(), readShard.getGridPosition()); + Assert.assertArrayEquals("shard size should equal block size when unsharded", readShard.getBlockSize(), readShard.getSize()); + + + final List> readBlocks = readShard.getBlocks(); + assertEquals("read shard should contain one block", 1, readBlocks.size()); + final DataBlock readBlock = readBlocks.get(0); + Assert.assertArrayEquals("read block position should be same as block position when unsharded", block0.getGridPosition(), readBlock.getGridPosition()); + Assert.assertArrayEquals("read block size should equal block size when unsharded", readBlock.getSize(), readBlock.getSize()); + + assertArrayEquals("block written through shard should be identical", (long[])readBlock.getData(), (long[])block0.getData()); + + } + + } + protected void assertDatasetAttributesEquals(final DatasetAttributes expected, final DatasetAttributes actual) { assertArrayEquals(expected.getDimensions(), actual.getDimensions()); assertArrayEquals(expected.getBlockSize(), actual.getBlockSize()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index fe83d3828..c8a041002 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -51,7 +51,7 @@ public class ShardTest { } }; - @Parameterized.Parameters(name = "IndexLocation({0}), Block ByteOrder({1}), Index ByteOrder({2})") + @Parameterized.Parameters(name = "IndexLocation({0}), Index ByteOrder({1})") public static Collection data() { final ArrayList params = new ArrayList<>(); From 6c4156c1092337b47a626790bf8082615d7b0fa2 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 12 Jun 2025 09:38:43 -0400 Subject: [PATCH 244/423] feat: more block as shard logic --- .../n5/shard/BlockAsShardCodec.java | 6 ---- .../saalfeldlab/n5/shard/ShardIndex.java | 35 ++++++++++--------- .../saalfeldlab/n5/AbstractN5Test.java | 10 +++--- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java index e5d64f0d6..ceed0d6a0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java @@ -50,11 +50,6 @@ public BlockAsShardCodec(ArrayCodec datasetArrayCodec) { data[0] = 0; data[1] = length; } - - @Override public long numBytes() { - - return 0; - } }; } @@ -95,7 +90,6 @@ public BlockAsShardCodec(ArrayCodec datasetArrayCodec) { } @Override public String getType() { - //TODO Caleb: can we ensure this is never called? return datasetArrayCodec.getType(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index c421e3db7..f07d30a67 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -36,14 +36,14 @@ public class ShardIndex extends LongArrayDataBlock { private final IndexLocation location; private final DeterministicSizeCodec[] codecs; - private final DatasetAttributes indexAttributes; + private final ShardIndexAttributes indexAttributes; public ShardIndex(int[] shardBlockGridSize, long[] data, IndexLocation location, final DeterministicSizeCodec... codecs) { super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, data); this.codecs = codecs; this.location = location; - this.indexAttributes = createIndexAttributes(); + this.indexAttributes = new ShardIndexAttributes(this); } public ShardIndex(int[] shardBlockGridSize, IndexLocation location, DeterministicSizeCodec... codecs) { @@ -164,9 +164,10 @@ public void readFrom(ReadData shardData) throws N5IOException { public static void read(InputStream in, final ShardIndex index) throws IOException { - @SuppressWarnings("unchecked") final DataBlock indexBlock = (DataBlock)DefaultBlockReader.readBlock(in, index.createIndexAttributes(), index.gridPosition); - final long[] indexData = indexBlock.getData(); - System.arraycopy(indexData, 0, index.data, 0, index.data.length); + final ReadData dataIn = ReadData.from(in); + final Codec.ArrayCodec shardIndexCodec = index.indexAttributes.getArrayCodec(); + final DataBlock indexBlock = shardIndexCodec.decode(dataIn, index.gridPosition); + System.arraycopy(indexBlock.getData(), 0, index.data, 0, index.data.length); } public static boolean read( @@ -198,24 +199,24 @@ public static void write( public static void write(final ShardIndex index, OutputStream out) throws IOException { - DefaultBlockWriter.writeBlock(out, index.createIndexAttributes(), index); + final Codec.ArrayCodec indexCodec = index.indexAttributes.getArrayCodec(); + indexCodec.encode(index).writeTo(out); } public Codec.ArrayCodec getArrayCodec() { return indexAttributes.getArrayCodec(); } - // TODO: Caleb static? - private DatasetAttributes createIndexAttributes() { - - final DatasetAttributes indexAttributes = - new DatasetAttributes( - Arrays.stream(getSize()).mapToLong(it -> it).toArray(), - getSize(), - DataType.UINT64, - codecs - ); - return indexAttributes; + private static class ShardIndexAttributes extends DatasetAttributes { + + public ShardIndexAttributes(ShardIndex index) { + super( + Arrays.stream(index.getSize()).mapToLong(it -> it).toArray(), + index.getSize(), + DataType.UINT64, + index.codecs + ); + } } public static IndexByteBounds byteBounds(DatasetAttributes datasetAttributes, final long objectSize) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 3e5f6e103..11de497a8 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1699,13 +1699,15 @@ public void testWriteReadShardOnUnshardedDataset() throws Exception { final DataBlock block1 = (DataBlock) DataType.UINT64.createDataBlock(blockSize, new long[]{1,0,0}); final InMemoryShard writeShard = new InMemoryShard<>(datasetAttributes, new long[]{0, 0, 0}); - writeShard.addBlock(block0); + boolean added0 = writeShard.addBlock(block0); + boolean added1 = writeShard.addBlock(block1); - writeShard.addBlock(block1); + assertTrue("Block 0 should be added to shard", added0); + assertFalse("Block 1 should not be added to shard because it's in a different shard position", added1); final List> writeBlocks = writeShard.getBlocks(); - assertEquals("block as shard should not have the second block which is outside this shard",1, writeBlocks.size()); - assertEquals(block1, writeBlocks.get(0)); + assertEquals("block as shard should not have the second block which is outside this shard", 1, writeBlocks.size()); + assertEquals(block0, writeBlocks.get(0)); writer.writeShard(datasetName, datasetAttributes, writeShard); final Shard readShard = writer.readShard(datasetName, datasetAttributes, writeShard.getGridPosition()); From 74e1f700cb1fdf0c4b4bfa3d5da07f991f9be822 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 12 Jun 2025 10:21:19 -0400 Subject: [PATCH 245/423] test: make test data smaller --- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 11de497a8..6ea60cc0a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -89,8 +89,8 @@ public abstract class AbstractN5Test { static protected final String groupName = "/test/group"; static protected final String[] subGroupNames = new String[]{"a", "b", "c"}; static protected final String datasetName = "/test/group/dataset"; - static protected final long[] dimensions = new long[]{100, 200, 300}; - static protected final int[] blockSize = new int[]{44, 33, 22}; + static protected final long[] dimensions = new long[]{10, 20, 30}; + static protected final int[] blockSize = new int[]{33, 22, 11}; static protected final int blockNumElements = blockSize[0] * blockSize[1] * blockSize[2]; static protected byte[] byteBlock; From 3c20ea017956e123fecf13f636cd32732a8051f9 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 12 Jun 2025 10:22:08 -0400 Subject: [PATCH 246/423] refactor: static readFromShard --- .../readdata/ByteArraySplittableReadData.java | 13 +++-- .../n5/shard/BlockAsShardCodec.java | 22 +++---- .../saalfeldlab/n5/shard/ShardIndex.java | 58 +++++-------------- .../saalfeldlab/n5/shard/VirtualShard.java | 3 +- .../saalfeldlab/n5/shard/ShardIndexTest.java | 2 +- 5 files changed, 32 insertions(+), 66 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java index e34a144bb..77f067186 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArraySplittableReadData.java @@ -53,7 +53,11 @@ class ByteArraySplittableReadData implements SplittableReadData { this.data = data; this.offset = offset; - this.length = length; + + if( length < 0 ) + this.length = data.length - offset; + else + this.length = length; } @Override @@ -87,12 +91,11 @@ public SplittableReadData materialize() { @Override public SplittableReadData slice(final long offset, final long length) throws IOException { - if (offset < 0 || offset >= this.length || length < 0) { + if (offset < 0 || offset >= this.length ) throw new IndexOutOfBoundsException(); - } + final int o = this.offset + (int)offset; - final int l = Math.min((int)length, this.length - o); - return new ByteArraySplittableReadData(data, o, l); + return new ByteArraySplittableReadData(data, o, (int)length); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java index ceed0d6a0..99fba596a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java @@ -2,6 +2,7 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.codec.RawBytes; @@ -9,7 +10,7 @@ public class BlockAsShardCodec extends ShardingCodec { - private static final RawBytes NO_OP_ARRAY_CODEC = new RawBytes() { + private static final RawBytes VIRTUAL_SHARD_INDEX_CODEC = new RawBytes() { @Override public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { @@ -18,19 +19,20 @@ public class BlockAsShardCodec extends ShardingCodec { @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException { - throw new UnsupportedOperationException(" NO_OP_ARRAY_CODEC is used for `encode` only. "); + final long[] data = new long[]{ 0, -1}; + return new LongArrayDataBlock(new int[0], new long[0], data); } }; private static final BytesCodec[] EMPTY_SHARD_CODECS = new BytesCodec[0]; - private static final DeterministicSizeCodec[] NO_OP_INDEX_CODECS = new DeterministicSizeCodec[]{NO_OP_ARRAY_CODEC}; + private static final DeterministicSizeCodec[] NO_OP_INDEX_CODECS = new DeterministicSizeCodec[]{VIRTUAL_SHARD_INDEX_CODEC}; final ArrayCodec datasetArrayCodec; private DatasetAttributes datasetAttributes; public BlockAsShardCodec(ArrayCodec datasetArrayCodec) { - super(null, EMPTY_SHARD_CODECS, NO_OP_INDEX_CODECS, IndexLocation.END); + super(null, EMPTY_SHARD_CODECS, NO_OP_INDEX_CODECS, IndexLocation.START); this.datasetArrayCodec = datasetArrayCodec; } @@ -38,17 +40,9 @@ public BlockAsShardCodec(ArrayCodec datasetArrayCodec) { return new ShardIndex(attributes.getBlocksPerShard(), getIndexLocation(), NO_OP_INDEX_CODECS) { - @Override public void readFrom(ReadData shardData) throws N5Exception.N5IOException { + @Override public long numBytes() { - if (shardData.length() == -1) - shardData.materialize(); - - final long length = shardData.length(); - if (length == -1) - throw new N5Exception.N5IOException("ReadData for shard index must have a valid length, but was " + length); - - data[0] = 0; - data[1] = length; + return 0; } }; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index f07d30a67..dda9a6ad4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -1,11 +1,8 @@ package org.janelia.saalfeldlab.n5.shard; -import org.apache.commons.io.input.BoundedInputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.DefaultBlockReader; -import org.janelia.saalfeldlab.n5.DefaultBlockWriter; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; @@ -15,14 +12,10 @@ import org.janelia.saalfeldlab.n5.readdata.SplittableReadData; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; import java.util.Arrays; import java.util.stream.IntStream; @@ -141,7 +134,7 @@ public long numBytes() { return totalNumBytes; } - public void readFrom(ReadData shardData) throws N5IOException { + public static void readFromShard(ReadData shardData, ShardIndex index) throws N5IOException { /* we require a length, so materialize if we don't have one. */ if (shardData.length() == -1) @@ -151,29 +144,17 @@ public void readFrom(ReadData shardData) throws N5IOException { if (length == -1) throw new N5IOException("ReadData for shard index must have a valid length, but was " + length); - final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(this, length); + final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, length); final ReadData indexData; try { - indexData = ((SplittableReadData)shardData).slice(bounds.start, this.numBytes()); + indexData = ((SplittableReadData)shardData).slice(bounds.start, index.numBytes()); } catch (IOException e) { throw new N5IOException("Failed to read shard index", e); } - ShardIndex.read(indexData, this); - + ShardIndex.read(indexData, index); } - public static void read(InputStream in, final ShardIndex index) throws IOException { - - final ReadData dataIn = ReadData.from(in); - final Codec.ArrayCodec shardIndexCodec = index.indexAttributes.getArrayCodec(); - final DataBlock indexBlock = shardIndexCodec.decode(dataIn, index.gridPosition); - System.arraycopy(indexBlock.getData(), 0, index.data, 0, index.data.length); - } - - public static boolean read( - final ReadData indexData, - final ShardIndex index - ) { + public static boolean read( final ReadData indexData, final ShardIndex index ) { try (final InputStream in = indexData.inputStream()) { read(in, index); @@ -185,24 +166,21 @@ public static boolean read( } } - public static void write( - final OutputStream outputStream, - final ShardIndex index - ) throws N5IOException { + public static void read(InputStream indexIn, final ShardIndex index) throws N5IOException { - try { - write(index, outputStream); - } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to write shard index", e); - } + final ReadData dataIn = ReadData.from(indexIn); + final Codec.ArrayCodec shardIndexCodec = index.indexAttributes.getArrayCodec(); + final DataBlock indexBlock = shardIndexCodec.decode(dataIn, index.gridPosition); + System.arraycopy(indexBlock.getData(), 0, index.data, 0, index.data.length); } - public static void write(final ShardIndex index, OutputStream out) throws IOException { + public static void write( final OutputStream outputStream, final ShardIndex index ) throws N5IOException { - final Codec.ArrayCodec indexCodec = index.indexAttributes.getArrayCodec(); - indexCodec.encode(index).writeTo(out); + final Codec.ArrayCodec indexCodec = index.indexAttributes.getArrayCodec(); + indexCodec.encode(index).writeTo(outputStream); } + public Codec.ArrayCodec getArrayCodec() { return indexAttributes.getArrayCodec(); } @@ -219,14 +197,6 @@ public ShardIndexAttributes(ShardIndex index) { } } - public static IndexByteBounds byteBounds(DatasetAttributes datasetAttributes, final long objectSize) { - - final ShardIndex index = datasetAttributes.getShardingCodec().createIndex(datasetAttributes); - - final long indexSize = index.numBytes(); - return byteBounds(indexSize, index.location, objectSize); - } - public static IndexByteBounds byteBounds(final ShardIndex index, long objectSize) { return byteBounds(index.numBytes(), index.location, objectSize); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 996f1ddd9..724566921 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -9,7 +9,6 @@ import org.janelia.saalfeldlab.n5.util.GridIterator; import java.io.IOException; -import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Arrays; @@ -124,7 +123,7 @@ public ShardIndex getIndex() { //TODO Caleb: How to handle when this shard doesn't exist (splitableData.getSize() <= 0) index = createIndex(); try { - index.readFrom(shardData); + ShardIndex.readFromShard(shardData, index); } catch (N5Exception.N5NoSuchKeyException e) { return null; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index 0432150ab..4fc92e713 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -94,7 +94,7 @@ public void writeReadTest() throws IOException { final LockedChannel channel = kva.lockForWriting(path, bounds.start, indexSize); final OutputStream out = channel.newOutputStream() ) { - ShardIndex.write(index, out); + ShardIndex.write(out, index); } final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecs); From fff741b38a5ca4c4f7893aa7cb6136080ef78782 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 12 Jun 2025 10:31:19 -0400 Subject: [PATCH 247/423] test: write shard for http fs tests --- .../n5/http/HttpReaderFsWriter.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index dc6e9dd1e..6556bfcaa 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -29,6 +29,7 @@ package org.janelia.saalfeldlab.n5.http; import com.google.gson.Gson; +import com.google.gson.JsonElement; import org.janelia.saalfeldlab.n5.CachedGsonKeyValueN5Reader; import org.janelia.saalfeldlab.n5.CachedGsonKeyValueN5Writer; import org.janelia.saalfeldlab.n5.Compression; @@ -39,6 +40,8 @@ import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.shard.Shard; import java.io.Serializable; import java.lang.reflect.Field; @@ -280,4 +283,39 @@ public HttpRead writer.writeSerializedBlock(object, datasetPath, datasetAttributes, gridPosition); } + + @Override public void setVersion(String path) { + + writer.setVersion(path); + } + + @Override public void writeAttributes(String normalGroupPath, JsonElement attributes) throws N5Exception { + + writer.writeAttributes(normalGroupPath, attributes); + } + + @Override public void setAttributes(String path, JsonElement attributes) throws N5Exception { + + writer.setAttributes(path, attributes); + } + + @Override public void writeAttributes(String normalGroupPath, Map attributes) throws N5Exception { + + writer.writeAttributes(normalGroupPath, attributes); + } + + @Override public void writeBlocks(String datasetPath, DatasetAttributes datasetAttributes, DataBlock... dataBlocks) throws N5Exception { + + writer.writeBlocks(datasetPath, datasetAttributes, dataBlocks); + } + + @Override public void writeShard(String path, DatasetAttributes datasetAttributes, Shard shard) throws N5Exception { + + writer.writeShard(path, datasetAttributes, shard); + } + + @Override public void createDataset(String datasetPath, long[] dimensions, int[] blockSize, DataType dataType, Codec... codecs) throws N5Exception { + + writer.createDataset(datasetPath, dimensions, blockSize, dataType, codecs); + } } From bf0920c5626b1562f79cbcf1915e893f5cc1c781 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 12 Jun 2025 11:05:07 -0400 Subject: [PATCH 248/423] feat: annotation to indicate a non-serializable NameConfig --- .../org/janelia/saalfeldlab/n5/NameConfigAdapter.java | 5 ++--- .../janelia/saalfeldlab/n5/serialization/NameConfig.java | 8 ++++++++ .../janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java | 2 ++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java index d957a92a3..3bfff5216 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java @@ -89,9 +89,8 @@ public static synchronized void update(final NameConfigAdapter adapter) { try { clazz = (Class)Class.forName(item.className()); - /* BlockAsShardCodec is not serializable; it is an internal memory-only - * interpretation codec for interacting with un-sharded datasets as sharded datasets. */ - if (clazz == BlockAsShardCodec.class) + final NameConfig.Serialize serialize = clazz.getAnnotation(NameConfig.Serialize.class); + if (serialize != null && !serialize.value()) continue; final String name = clazz.getAnnotation(NameConfig.Name.class).value(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java b/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java index 2ccb122ea..dfcb9332f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java @@ -26,6 +26,14 @@ public interface NameConfig extends Serializable { String value(); } + @Retention(RetentionPolicy.RUNTIME) + @Inherited + @Target(ElementType.TYPE) + @Indexable + @interface Serialize { + boolean value() default true; + } + @Retention(RetentionPolicy.RUNTIME) @Inherited @Target(ElementType.FIELD) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java index 99fba596a..1549315fb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java @@ -7,7 +7,9 @@ import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; +@NameConfig.Serialize(false) public class BlockAsShardCodec extends ShardingCodec { private static final RawBytes VIRTUAL_SHARD_INDEX_CODEC = new RawBytes() { From 481f75afa324dd07af1cbaa0760b276a4c034855 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 12 Jun 2025 11:05:46 -0400 Subject: [PATCH 249/423] misc: cleanup more blockAsShard stuff --- .../org/janelia/saalfeldlab/n5/shard/InMemoryShard.java | 3 +-- src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java | 6 ++++-- .../java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java | 7 ------- .../org/janelia/saalfeldlab/n5/shard/VirtualShard.java | 1 - 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 60f2d5177..2bb17b3dd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -70,8 +70,7 @@ public List> getBlocks() { @Override public ShardIndex getIndex() { - index = index != null ? index : createIndex(); - return index; + return index = index != null ? index : createIndex(); } public static InMemoryShard fromShard(Shard shard) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 053988fb1..a48a1cfe1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -151,11 +151,13 @@ default ShardIndex createIndex() { default ReadData createReadData() throws N5IOException { - final DatasetAttributes datasetAttributes = getDatasetAttributes(); - final ShardIndex index = ShardIndex.createIndex(datasetAttributes); + + final DatasetAttributes datasetAttributes = getDatasetAttributes(); ShardingCodec shardingCodec = datasetAttributes.getShardingCodec(); final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); + + final ShardIndex index = createIndex(); long blocksStartBytes = index.getLocation() == ShardingCodec.IndexLocation.START ? index.numBytes() : 0; final AtomicLong blockOffset = new AtomicLong(blocksStartBytes); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index dda9a6ad4..b859fa0a9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -259,12 +259,5 @@ public boolean equals(Object other) { } return true; } - - public static ShardIndex createIndex(final DatasetAttributes attributes) { - - return attributes.getShardingCodec().createIndex(attributes); - } - - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index 724566921..c24e94de3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -120,7 +120,6 @@ public DataBlock getBlock(long... blockGridPosition) { @Override public ShardIndex getIndex() { - //TODO Caleb: How to handle when this shard doesn't exist (splitableData.getSize() <= 0) index = createIndex(); try { ShardIndex.readFromShard(shardData, index); From fbae8ac5e298c94f118fe4965c8a69ed214462b1 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 12 Jun 2025 11:23:21 -0400 Subject: [PATCH 250/423] test: BlockAsShard AbstractN5 Test --- .../saalfeldlab/n5/shard/InMemoryShard.java | 5 -- .../janelia/saalfeldlab/n5/shard/Shard.java | 9 +++- .../saalfeldlab/n5/N5BlockAsShardTest.java | 48 +++++++++++++++++++ .../org/janelia/saalfeldlab/n5/N5FSTest.java | 2 +- 4 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 2bb17b3dd..a2b57d5be 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -35,11 +35,6 @@ private void storeBlock(DataBlock block) { blocks.put(Position.wrap(block.getGridPosition()), block); } - /* - * Returns the {@link DataBlock} given a block grid position. - *

    - * The block grid position is relative to the image, not relative to this shard. - */ @Override public DataBlock getBlock(long... blockGridPosition) { return blocks.get(Position.wrap(blockGridPosition)); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index a48a1cfe1..0b4da5d2c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -102,7 +102,14 @@ default long[] getShardPosition(long... blockPosition) { return shardGridPosition; } - public DataBlock getBlock(long... blockGridPosition); + /** + * Retrieve the DataBlock at {@code blockGridPosition} if it exists and is + * a member of this Shard. + * + * @param blockGridPosition position of the desired block in the block grid + * @return the block if it exists and is part of this shard, otherwise null + */ + DataBlock getBlock(long... blockGridPosition); default Iterator> iterator() { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java new file mode 100644 index 000000000..19ffa7e31 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java @@ -0,0 +1,48 @@ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.GsonBuilder; +import org.janelia.saalfeldlab.n5.shard.InMemoryShard; +import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.ShardIndex; + + +public class N5BlockAsShardTest extends N5FSTest { + + @Override protected N5Writer createN5Writer(String location, GsonBuilder gson) { + + return new N5FSWriter(location, gson) { + + @Override public void writeBlock(String path, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { + + final ShardIndex index = datasetAttributes.getShardingCodec().createIndex(datasetAttributes); + final InMemoryShard shard = new InMemoryShard(datasetAttributes, dataBlock.getGridPosition(), index); + shard.addBlock(dataBlock); + writeShard(path, datasetAttributes, shard); + } + + @Override public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { + + final Shard shard = readShard(pathName, datasetAttributes, gridPosition); + if (shard == null) + return null; + + return shard.getBlock(gridPosition); + } + }; + } + + @Override protected N5Reader createN5Reader(String location, GsonBuilder gson) { + + return new N5FSReader(location, gson) { + + @Override public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { + + final Shard shard = readShard(pathName, datasetAttributes, gridPosition); + if (shard == null) + return null; + + return shard.getBlock(gridPosition); + } + }; + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 9eaeca325..5c3133c6c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -82,7 +82,7 @@ protected String tempN5Location() throws URISyntaxException { @Override protected N5Writer createN5Writer() throws IOException, URISyntaxException { - return new N5FSWriter(tempN5Location(), new GsonBuilder()); + return createN5Writer(tempN5Location(), new GsonBuilder()); } @Override From 5c9a4f008608084bbc392fb9cd9ed04c17ce5190 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 13 Jun 2025 17:11:08 -0400 Subject: [PATCH 251/423] wip: shard merge after staged PR merges --- .../saalfeldlab/n5/DatasetAttributes.java | 10 +- .../n5/FileSystemKeyValueAccess.java | 6 + .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 4 - .../saalfeldlab/n5/HttpKeyValueAccess.java | 24 ++- .../n5/KeyValueAccessSplittableReadData.java | 21 +-- .../org/janelia/saalfeldlab/n5/N5Writer.java | 2 +- .../saalfeldlab/n5/SplitByteBufferedData.java | 151 ------------------ .../n5/SplitKeyValueAccessData.java | 72 --------- .../janelia/saalfeldlab/n5/SplitableData.java | 21 --- .../janelia/saalfeldlab/n5/codec/Codec.java | 9 +- .../saalfeldlab/n5/codec/N5BlockCodec.java | 4 +- .../saalfeldlab/n5/codec/RawBlockCodecs.java | 11 +- .../saalfeldlab/n5/codec/RawBytes.java | 4 +- .../readdata/ReadDataSplittableReadData.java | 117 -------------- .../n5/readdata/SplittableReadData.java | 31 ---- .../n5/shard/BlockAsShardCodec.java | 12 +- .../janelia/saalfeldlab/n5/shard/Shard.java | 4 +- .../saalfeldlab/n5/shard/ShardIndex.java | 15 +- .../saalfeldlab/n5/shard/ShardingCodec.java | 12 +- .../saalfeldlab/n5/AbstractN5Test.java | 2 +- .../saalfeldlab/n5/codec/BytesTests.java | 2 +- .../saalfeldlab/n5/demo/BlockIterators.java | 4 +- .../saalfeldlab/n5/shard/ShardIndexTest.java | 2 +- .../saalfeldlab/n5/shard/ShardTest.java | 14 +- 24 files changed, 66 insertions(+), 488 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/SplitByteBufferedData.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/SplitableData.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadDataSplittableReadData.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 5f2372346..a759b8e9a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -180,7 +180,7 @@ public int[] getBlockSize() { } public boolean isSharded() { - return getArrayCodec() instanceof ShardingCodec; + return getArrayCodec() instanceof ShardingCodec; } /** @@ -189,11 +189,11 @@ public boolean isSharded() { * * @throws N5ShardException if the dataset is not sharded */ - public ShardingCodec getShardingCodec() throws N5ShardException { - if (getArrayCodec() instanceof ShardingCodec) - return (ShardingCodec)getArrayCodec(); + public ShardingCodec getShardingCodec() throws N5ShardException { + if (getArrayCodec() instanceof ShardingCodec) + return (ShardingCodec)getArrayCodec(); else { - return new BlockAsShardCodec<>(getArrayCodec()); + return new BlockAsShardCodec(getArrayCodec()); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 4656048e1..cb161c3c9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -127,6 +127,12 @@ protected FileChannel getFileChannel() { return channel; } + @Override + public long size() throws IOException { + + return channel.size(); + } + private void truncateChannel(int size) { try { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 689a5704b..8d15a2b83 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -134,10 +134,6 @@ default DataBlock readBlock( return datasetAttributes.getArrayCodec().decode(decodeData, gridPosition); } catch (N5Exception.N5NoSuchKeyException e) { return null; - } catch (IOException | UncheckedIOException | N5IOException e) { - throw new N5IOException( - "Failed to read block " + Arrays.toString(gridPosition) + " from dataset " + path, - e); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index 8ebabac26..68b0e6ceb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -230,11 +230,6 @@ private HttpURLConnection httpRequest(String normalPath, String method) throws I return connection; } - @Override - public HttpSplittableReadData createReadData(final String normalPath) { - return new HttpSplittableReadData(this, normalPath, 0, -1); - } - @Override public HttpLazyReadData createReadData(final String normalPath) { return new HttpLazyReadData(this, normalPath, 0, -1); @@ -257,11 +252,6 @@ public LockedChannel lockForWriting(final String normalPath) throws N5IOExceptio throw new N5Exception("HttpKeyValueAccess is read-only"); } - @Override public LockedChannel lockForWriting(String normalPath, long startByte, long size) throws IOException { - - throw new N5Exception("HttpKeyValueAccess is read-only"); - } - /** * List all 'directory'-like children of a path. *

    @@ -373,6 +363,11 @@ protected HttpObjectChannel(final URI uri, long startByte, long size) { this.size = size; } + @Override public long size() { + + return size; + } + private boolean isPartialRead() { return startByte > 0 || (size >= 0 && size != Long.MAX_VALUE); } @@ -393,8 +388,9 @@ public InputStream newInputStream() throws N5IOException { } return conn.getInputStream(); } catch (IOException e) { - throw new N5IOException("Could not open stream for " + uri, e); + throw new N5Exception.N5IOException(e); } + } private String rangeString() { @@ -404,7 +400,7 @@ private String rangeString() { } @Override - public Reader newReader() throws N5IOException { + public Reader newReader() { final InputStreamReader reader = new InputStreamReader(newInputStream(), StandardCharsets.UTF_8); synchronized (resources) { @@ -414,13 +410,13 @@ public Reader newReader() throws N5IOException { } @Override - public OutputStream newOutputStream() throws N5IOException { + public OutputStream newOutputStream() { throw new NonWritableChannelException(); } @Override - public Writer newWriter() throws N5IOException { + public Writer newWriter() { throw new NonWritableChannelException(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java index 7bee73fb1..25e7f2c32 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessSplittableReadData.java @@ -66,31 +66,14 @@ public ReadData materialize() throws N5IOException { abstract void read() throws N5IOException; - abstract KeyValueAccessReadData readOperationSlice(long offset, long length) throws IOException; + abstract KeyValueAccessReadData readOperationSlice(long offset, long length) throws N5IOException; @Override - public ReadData slice(long offset, long length) throws IOException { + public ReadData slice(long offset, long length) throws N5IOException { if (materialized != null) return materialize().slice(offset, length); return readOperationSlice(this.offset + offset, length); } - - @Override - public Pair split(long pivot) throws IOException { - - if (materialized != null) - return materialize().split(pivot); - - final long offsetL = 0; - final long lenL = pivot; - - final long offsetR = offset + pivot; - final long lenR = this.length - pivot; - - return new ImmutablePair( - readOperationSlice(offsetL, lenL), - readOperationSlice(offsetR, lenR)); - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 514a239f0..ee0419a86 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -256,7 +256,7 @@ default void createDataset( final DataType dataType, final Compression compression) throws N5Exception { - createDataset(datasetPath, dimensions, blockSize, dataType, new N5BlockCodec<>(), compression); + createDataset(datasetPath, dimensions, blockSize, dataType, new N5BlockCodec(), compression); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/SplitByteBufferedData.java b/src/main/java/org/janelia/saalfeldlab/n5/SplitByteBufferedData.java deleted file mode 100644 index c4a11afb0..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/SplitByteBufferedData.java +++ /dev/null @@ -1,151 +0,0 @@ -package org.janelia.saalfeldlab.n5; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.function.Consumer; -import java.util.function.Supplier; - -public class SplitByteBufferedData implements SplitableData { - - private byte[] sharedHierarchyData; - private int maxCount = 0; - - private final AccessibleByteArrayOutputStream data; - private long offset; - - public SplitByteBufferedData() { - this(32); - } - - public SplitByteBufferedData(final int initialSize) { - - this(new byte[initialSize]); - } - - public SplitByteBufferedData(final int initialSize, final long offset) { - - this(new byte[initialSize], offset); - } - - public SplitByteBufferedData(final byte[] initialBuffer) { - - this(initialBuffer, 0); - } - - public SplitByteBufferedData(final byte[] initialBuffer, final long offset) { - - this.offset = offset; - this.sharedHierarchyData = initialBuffer; - this.data = new AccessibleByteArrayOutputStream(this::getSharedBuffer, this::setSharedBuffer, this::getMaxCount, this::setMaxCount); - data.setCount((int)offset); - maxCount = initialBuffer.length; - } - - private SplitByteBufferedData(final AccessibleByteArrayOutputStream splitData, final long offset, final long size) { - - this.offset = offset; - this.data = splitData; - data.setCount((int)offset); - } - - private byte[] getSharedBuffer() { - return sharedHierarchyData; - } - - private void setSharedBuffer(byte[] buffer) { - sharedHierarchyData = buffer; - } - - private void setMaxCount(int count) { - if (count > maxCount) - maxCount = count; - } - - private int getMaxCount() { - return maxCount; - } - - @Override - public long getOffset() { - - return offset; - } - - @Override - public long getSize() { - - return getMaxCount() - getOffset(); - } - - @Override - public InputStream newInputStream() { - - final byte[] sharedBuffer = data.getBuf(); - return new ByteArrayInputStream(sharedBuffer, (int)offset, (int)(data.getMaxCount.get() - offset)); - } - - @Override - public OutputStream newOutputStream() { - - return data; - } - - @Override - public SplitableData split(long offset, long size) { - - final AccessibleByteArrayOutputStream newData = new AccessibleByteArrayOutputStream( - data.getSharedBuffer, data.setSharedBuffer, - data.getMaxCount, data.setMaxCount - ); - return new SplitByteBufferedData(newData, offset, size); - } - - private static class AccessibleByteArrayOutputStream extends ByteArrayOutputStream { - - private final Supplier getSharedBuffer; - private final Consumer setSharedBuffer; - private final Supplier getMaxCount; - private final Consumer setMaxCount; - - AccessibleByteArrayOutputStream( - final Supplier getSharedBuffer, - final Consumer setSharedBuffer, - final Supplier getMaxCount, - final Consumer setMaxCount) { - - this.getSharedBuffer = getSharedBuffer; - this.setSharedBuffer = setSharedBuffer; - this.getMaxCount = getMaxCount; - this.setMaxCount = setMaxCount; - } - - byte[] getBuf() { - return getSharedBuffer.get(); - } - - void setCount(int count) { - this.count = count; - } - - @Override public synchronized void write(int b) { - - final byte[] sharedBufBeforeWrite = getSharedBuffer.get(); - - if (buf != sharedBufBeforeWrite) buf = sharedBufBeforeWrite; - super.write(b); - if (buf != sharedBufBeforeWrite) setSharedBuffer.accept(buf); - setMaxCount.accept(count); - } - - @Override public synchronized void write(byte[] b, int off, int len) { - - final byte[] sharedBufBeforeWrite = getSharedBuffer.get(); - if (buf != sharedBufBeforeWrite) buf = sharedBufBeforeWrite; - super.write(b, off, len); - if (buf != sharedBufBeforeWrite) setSharedBuffer.accept(buf); - setMaxCount.accept(count); - } - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java b/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java deleted file mode 100644 index e6f5d40d7..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/SplitKeyValueAccessData.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -public class SplitKeyValueAccessData implements SplitableData { - - private final KeyValueAccess access; - private final String key; - private final long offset; - private final long size; - - public SplitKeyValueAccessData( - final KeyValueAccess access, - final String key) throws IOException { - this.access = access; - this.key = key; - this.offset = 0; - long keySize; - try { - /*If the file exists we need to know the real size, in case we need to read the - * Index N bytes from the end, for example. Potentially we could do this lazily */ - keySize = access.size(key); - } catch (N5Exception.N5NoSuchKeyException e) { - keySize = 0; //TODO Caleb: 0? -1? ??? - } - this.size = keySize; - } - - public SplitKeyValueAccessData( - final KeyValueAccess access, - final String key, - final long offset, - final long size){ - - this.access = access; - this.key = key; - this.offset = offset; - this.size = size; - } - - @Override - public long getOffset() { - - return offset; - } - - @Override - public long getSize() { - - return size; - } - - @Override - public InputStream newInputStream() throws IOException { - //TODO Caleb: Should wrap with BoundedInputStream? - return access.lockForReading(key, offset, size).newInputStream(); - } - - @Override - public OutputStream newOutputStream() throws IOException { - //TODO Caleb: If 0 -> -1? to handle non-existent files - return access.lockForWriting(key, offset, size).newOutputStream(); - } - - @Override - public SplitableData split(long offset, long size) { - - return new SplitKeyValueAccessData(access, key, getOffset() + offset, size); - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/SplitableData.java b/src/main/java/org/janelia/saalfeldlab/n5/SplitableData.java deleted file mode 100644 index 86c152c2a..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/SplitableData.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -public interface SplitableData { - - // TODO Caleb: - // Do we need a getter for this? If it's intended to be relative - // after a split, shouldn't offset always be (locally) 0? - // maybe only need (offset) during #split - long getOffset(); - - long getSize(); - - InputStream newInputStream() throws IOException; - OutputStream newOutputStream() throws IOException; - - SplitableData split(long offset, long size); -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index 114ee72c6..846f06a66 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -2,6 +2,7 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @@ -38,7 +39,7 @@ interface BytesCodec extends Codec { * @return decoded ReadData * @throws IOException if any I/O error occurs */ - ReadData decode(ReadData readData) throws IOException; + ReadData decode(ReadData readData) throws N5Exception.N5IOException; /** * Encode the given {@link ReadData}. @@ -50,7 +51,7 @@ interface BytesCodec extends Codec { * @return encoded ReadData * @throws IOException if any I/O error occurs */ - ReadData encode(ReadData readData) throws IOException; + ReadData encode(ReadData readData) throws N5Exception.N5IOException; } @@ -60,9 +61,9 @@ interface BytesCodec extends Codec { */ interface ArrayCodec extends DeterministicSizeCodec { - DataBlock decode(ReadData readData, long[] gridPosition) throws IOException; + DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException; - ReadData encode(DataBlock dataBlock) throws IOException; + ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException; default long[] getPositionForBlock(final DatasetAttributes attributes, final DataBlock datablock) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java index 7a4c8de82..c414f2890 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java @@ -28,12 +28,12 @@ private DataBlockCodec getDataBlockCodec() { return N5Codecs.createDataBlockCodec(attributes.getDataType(), bytesCodec); } - @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { + @Override public DataBlock decode(ReadData readData, long[] gridPosition) { return this.getDataBlockCodec().decode(readData, gridPosition); } - @Override public ReadData encode(DataBlock dataBlock) throws IOException { + @Override public ReadData encode(DataBlock dataBlock) { return this.getDataBlockCodec().encode(dataBlock); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java index 354c5deeb..f687ff726 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java @@ -106,14 +106,9 @@ private int numElements() { } @Override public DataBlock decode(ReadData readData, long[] gridPosition) { - ReadData decodeData; - try { - decodeData = codec.decode(readData); - final T data = dataCodec.deserialize(decodeData, numElements()); - return dataBlockFactory.createDataBlock(blockSize, gridPosition, data); - } catch (IOException e) { - throw new N5Exception.N5IOException(e); - } + ReadData decodeData = codec.decode(readData); + final T data = dataCodec.deserialize(decodeData, numElements()); + return dataBlockFactory.createDataBlock(blockSize, gridPosition, data); } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java index 4bd34ce8e..56a4d087b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java @@ -58,14 +58,14 @@ private DataBlockCodec< T > getDataBlockCodec() { @SuppressWarnings("unchecked") @Override - public DataBlock decode(ReadData readData, long[] gridPosition) throws IOException { + public DataBlock decode(ReadData readData, long[] gridPosition) { return this.getDataBlockCodec().decode(readData, gridPosition); } @SuppressWarnings( "unchecked" ) @Override - public ReadData encode(DataBlock dataBlock) throws IOException { + public ReadData encode(DataBlock dataBlock) { return this.getDataBlockCodec().encode(dataBlock); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadDataSplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadDataSplittableReadData.java deleted file mode 100644 index 34a7fb097..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadDataSplittableReadData.java +++ /dev/null @@ -1,117 +0,0 @@ -/*- - * #%L - * Not HDF5 - * %% - * Copyright (C) 2017 - 2025 Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.readdata; - -import org.apache.commons.io.input.BoundedInputStream; -import org.apache.commons.io.input.ProxyInputStream; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; - -class ReadDataReadData implements ReadData { - - private final ReadData readData; - private final long offset; - private final long length; - - ReadDataReadData(final ReadData readData, final long offset, final long length) { - - this.readData = readData; - this.offset = offset; - this.length = length; - } - - @Override - public long length() { - - return length; - } - - @Override - public InputStream inputStream() throws N5IOException { - - final InputStream offsetInputStream = new ProxyInputStream(readData.inputStream()) { - - private boolean firstRead = true; - - @Override protected void beforeRead(int n) throws IOException { - - if (firstRead) { - inputStream().skip(offset); - - firstRead = false; - } - - super.beforeRead(n); - } - }; - - try { - return BoundedInputStream.builder() - .setInputStream(offsetInputStream) - .setBufferSize((int)length) - .get(); - } catch (IOException e) { - throw new N5IOException(e); - } - } - - @Override - public byte[] allBytes() throws N5IOException, IllegalStateException { - - final byte[] bytes = readData.allBytes(); - if (offset == 0 && bytes.length == length) { - return bytes; - } else { - return Arrays.copyOfRange(bytes, (int)offset, (int)(offset + length)); - } - } - - @Override - public ReadData materialize() { - - return this; - } - - @Override - public ReadData slice(final long offset, final long length) { - - return new ReadDataReadData(this, offset, length); - } - - @Override - public Pair split(final long pivot) throws IOException { - - return ImmutablePair.of(slice(0, pivot), slice(offset + pivot, length - pivot)); - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java deleted file mode 100644 index 52ca92f91..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SplittableReadData.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.janelia.saalfeldlab.n5.readdata; - -import java.io.IOException; - -import org.apache.commons.lang3.tuple.Pair; - -public interface ReadData extends ReadData { - - default ReadData limit(final long length) throws IOException { - return slice(0, length); - } - - /** - * Returns a new {@link ReadData} representing a slice, or subset - * of this ReadData. - * - * @param offset the offset relative to this - * @param length of the returned ReadData - * @return a slice - * @throws IOException an exception - */ - ReadData slice(final long offset, final long length) throws IOException; - - /* - * TODO do we want this? how should it work? - * this could be useful for infinite data, or data of unknown length? - * - * tail below would be equivalent to slice(pivot, -1) - */ - Pair split(final long pivot) throws IOException; -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java index 1549315fb..0877aab4d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java @@ -10,7 +10,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @NameConfig.Serialize(false) -public class BlockAsShardCodec extends ShardingCodec { +public class BlockAsShardCodec extends ShardingCodec { private static final RawBytes VIRTUAL_SHARD_INDEX_CODEC = new RawBytes() { @@ -29,10 +29,10 @@ public class BlockAsShardCodec extends ShardingCodec { private static final BytesCodec[] EMPTY_SHARD_CODECS = new BytesCodec[0]; private static final DeterministicSizeCodec[] NO_OP_INDEX_CODECS = new DeterministicSizeCodec[]{VIRTUAL_SHARD_INDEX_CODEC}; - final ArrayCodec datasetArrayCodec; + final ArrayCodec datasetArrayCodec; private DatasetAttributes datasetAttributes; - public BlockAsShardCodec(ArrayCodec datasetArrayCodec) { + public BlockAsShardCodec(ArrayCodec datasetArrayCodec) { super(null, EMPTY_SHARD_CODECS, NO_OP_INDEX_CODECS, IndexLocation.START); this.datasetArrayCodec = datasetArrayCodec; @@ -54,7 +54,7 @@ public BlockAsShardCodec(ArrayCodec datasetArrayCodec) { return datasetAttributes.getBlockSize(); } - @Override public ArrayCodec getArrayCodec() { + @Override public ArrayCodec getArrayCodec() { return datasetArrayCodec; } @@ -90,12 +90,12 @@ public BlockAsShardCodec(ArrayCodec datasetArrayCodec) { return datasetArrayCodec.getType(); } - @Override public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { + @Override public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { return datasetArrayCodec.encode(dataBlock); } - @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException { + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException { return datasetArrayCodec.decode(readData, gridPosition); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 0b4da5d2c..6693b8b89 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -161,8 +161,8 @@ default ReadData createReadData() throws N5IOException { final DatasetAttributes datasetAttributes = getDatasetAttributes(); - ShardingCodec shardingCodec = datasetAttributes.getShardingCodec(); - final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); + ShardingCodec shardingCodec = datasetAttributes.getShardingCodec(); + final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); final ShardIndex index = createIndex(); long blocksStartBytes = index.getLocation() == ShardingCodec.IndexLocation.START ? index.numBytes() : 0; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 18e42e4da..615a5307c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -145,12 +145,7 @@ public static void readFromShard(ReadData shardData, ShardIndex index) throws N5 throw new N5IOException("ReadData for shard index must have a valid length, but was " + length); final ShardIndex.IndexByteBounds bounds = ShardIndex.byteBounds(index, length); - final ReadData indexData; - try { - indexData = ((ReadData)shardData).slice(bounds.start, index.numBytes()); - } catch (IOException e) { - throw new N5IOException("Failed to read shard index", e); - } + final ReadData indexData = shardData.slice(bounds.start, index.numBytes()); ShardIndex.read(indexData, index); } @@ -169,19 +164,19 @@ public static boolean read( final ReadData indexData, final ShardIndex index ) { public static void read(InputStream indexIn, final ShardIndex index) throws N5IOException { final ReadData dataIn = ReadData.from(indexIn); - final Codec.ArrayCodec shardIndexCodec = index.indexAttributes.getArrayCodec(); + final Codec.ArrayCodec shardIndexCodec = index.indexAttributes.getArrayCodec(); final DataBlock indexBlock = shardIndexCodec.decode(dataIn, index.gridPosition); System.arraycopy(indexBlock.getData(), 0, index.data, 0, index.data.length); } public static void write( final OutputStream outputStream, final ShardIndex index ) throws N5IOException { - final Codec.ArrayCodec indexCodec = index.indexAttributes.getArrayCodec(); - indexCodec.encode(index).writeTo(outputStream); + final Codec.ArrayCodec indexCodec = index.indexAttributes.getArrayCodec(); + indexCodec.encode(index).writeTo(outputStream); } - public Codec.ArrayCodec getArrayCodec() { + public Codec.ArrayCodec getArrayCodec() { return indexAttributes.getArrayCodec(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 3c0eb48aa..a83de4c84 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -9,12 +9,10 @@ import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.N5Annotations; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @@ -22,7 +20,7 @@ import java.util.Objects; @NameConfig.Name(ShardingCodec.TYPE) -public class ShardingCodec implements Codec.ArrayCodec { +public class ShardingCodec implements Codec.ArrayCodec { private static final long serialVersionUID = -5879797314954717810L; @@ -82,13 +80,13 @@ public IndexLocation getIndexLocation() { return indexLocation; } - public ArrayCodec getArrayCodec() { + public ArrayCodec getArrayCodec() { Objects.requireNonNull(codecs); if (codecs.length == 0) throw new IllegalArgumentException("Sharding Codec requires a single ArrayCodec. None found."); - return (ArrayCodec)codecs[0]; + return (ArrayCodec)codecs[0]; } public BytesCodec[] getCodecs() { @@ -121,12 +119,12 @@ public DeterministicSizeCodec[] getIndexCodecs() { getArrayCodec().initialize(attributes, getCodecs()); } - @Override public ReadData encode(DataBlock dataBlock) throws N5IOException { + @Override public ReadData encode(DataBlock dataBlock) { return getArrayCodec().encode(dataBlock); } - @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException { + @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException { final ReadData splitableReadData = readData.materialize(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 6ea60cc0a..5661446b7 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -282,7 +282,7 @@ public void testWriteReadByteBlockMultipleCodecs() { final String dataset = "8_64_32"; n5.remove(dataset); final Codec[] codecs = { - new N5BlockCodec<>(), + new N5BlockCodec(), new AsTypeCodec(DataType.INT8, DataType.INT32), new AsTypeCodec(DataType.INT32, DataType.INT64) }; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java index 8a0d8abb7..d36f64bbe 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java @@ -37,7 +37,7 @@ public void testSerialization() { new long[]{8, 8}, new int[]{4, 4}, DataType.UINT8, - new N5BlockCodec<>(), + new N5BlockCodec(), new IdentityCodec() ); writer.createGroup("shard"); //Should already exist, but this will ensure. diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java index f876ec85b..d102201ef 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java @@ -36,8 +36,8 @@ public static void shardBlockIterator() { DataType.UINT8, new ShardingCodec<>( new int[] {2, 2}, - new Codec[] { new N5BlockCodec<>() }, - new DeterministicSizeCodec[] { new RawBytes<>() }, + new Codec[] { new N5BlockCodec() }, + new DeterministicSizeCodec[] { new RawBytes() }, IndexLocation.END )); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index 4fc92e713..7ca41f864 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -71,7 +71,7 @@ public void writeReadTest() throws IOException { final int[] shardBlockGridSize = new int[]{6, 5}; final IndexLocation indexLocation = IndexLocation.END; final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[]{ - new N5BlockCodec<>(), + new N5BlockCodec(), new Crc32cChecksumCodec()}; final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "indexTest").toString(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index c8a041002..c90562ce9 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -85,10 +85,10 @@ private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, shardSize, blockSize, DataType.UINT8, - new ShardingCodec<>( + new ShardingCodec( blockSize, - new Codec[]{new N5BlockCodec<>()}, //, new GzipCompression(4)}, - new DeterministicSizeCodec[]{new RawBytes<>(), new Crc32cChecksumCodec()}, + new Codec[]{new N5BlockCodec()}, //, new GzipCompression(4)}, + new DeterministicSizeCodec[]{new RawBytes(), new Crc32cChecksumCodec()}, indexLocation ) ); @@ -100,7 +100,7 @@ private DatasetAttributes getTestAttributes() { } @Test - public void writeReadBlocksTest() throws IOException { + public void writeReadBlocksTest() { final N5Writer writer = tempN5Factory.createTempN5Writer(); final DatasetAttributes datasetAttributes = getTestAttributes( @@ -320,7 +320,7 @@ private DatasetAttributes getNestedShardCodecsAttributes(int[] blockSize) { // probably better to forget about this class - only use DatasetAttributes // and detect shading in another way final ShardingCodec innerShard = new ShardingCodec(innerShardSize, - new Codec[]{new N5BlockCodec<>()}, + new Codec[]{new N5BlockCodec()}, new DeterministicSizeCodec[]{new RawBytes(indexByteOrder), new Crc32cChecksumCodec()}, IndexLocation.START); @@ -345,11 +345,11 @@ private static DatasetAttributes getDatasetAttributes(long[] imageSize, int[] sh blockSize, new Codec[]{ // codecs applied to image data - new RawBytes<>(ByteOrder.BIG_ENDIAN), + new RawBytes(ByteOrder.BIG_ENDIAN), }, new DeterministicSizeCodec[]{ // codecs applied to the shard index, must not be compressors - new RawBytes<>(ByteOrder.LITTLE_ENDIAN), + new RawBytes(ByteOrder.LITTLE_ENDIAN), new Crc32cChecksumCodec() }, IndexLocation.START From b1c785b7ee6c3df62c33843c0dc9367f75b2bfb6 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 24 Jun 2025 15:26:41 -0400 Subject: [PATCH 252/423] doc: update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 09979cbd6..9bc05cb5f 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,6 @@ Custom compression schemes can be implemented using the annotation discovery mec ## Disclaimer -HDF5 is a great format that provides a wealth of conveniences that I do not want to miss. It's inefficiency for parallel writing, however, limit its applicability for handling of very large n-dimensional data. +HDF5 is a great format that provides a wealth of conveniences that I do not want to miss. Its inefficiency for parallel writing, however, limits its applicability for handling of very large n-dimensional data. N5 uses the native filesystem of the target platform and JSON files to specify basic and custom meta-data as attributes. It aims at preserving the convenience of HDF5 where possible but doesn't try too hard to be a full replacement. From d0158c2011b746a1fb242c61e27682632f40de9d Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 27 Jun 2025 10:24:05 -0400 Subject: [PATCH 253/423] chore!: remove AsTypeCodec and related logic --- .../saalfeldlab/n5/codec/AsTypeCodec.java | 409 ------------------ .../n5/codec/FixedScaleOffsetCodec.java | 115 ----- .../saalfeldlab/n5/AbstractN5Test.java | 32 +- .../saalfeldlab/n5/codec/AsTypeTests.java | 69 --- .../n5/serialization/CodecSerialization.java | 31 +- 5 files changed, 7 insertions(+), 649 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java deleted file mode 100644 index 2ea5bda57..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/AsTypeCodec.java +++ /dev/null @@ -1,409 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec; - -import java.nio.ByteBuffer; -import java.util.function.BiConsumer; - -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.serialization.NameConfig; - -@NameConfig.Name(AsTypeCodec.TYPE) -public class AsTypeCodec implements Codec.BytesCodec { - - private static final long serialVersionUID = 1031322606191894484L; - - public static final String TYPE = "astype"; - - protected transient int numBytes; - protected transient int numEncodedBytes; - - protected transient BiConsumer encoder; - protected transient BiConsumer decoder; - - @NameConfig.Parameter - protected final DataType dataType; - - @NameConfig.Parameter - protected final DataType encodedType; - - private AsTypeCodec() { - - this(null, null); - } - - public AsTypeCodec(DataType dataType, DataType encodedType) { - - this.dataType = dataType; - this.encodedType = encodedType; - } - - @Override - public String getType() { - - return TYPE; - } - - public DataType getDataType() { - - return dataType; - } - - public DataType getEncodedDataType() { - - return encodedType; - } - - @Override public ReadData encode(ReadData readData) { - - - numBytes = bytes(dataType); - numEncodedBytes = bytes(encodedType); - - encoder = converter(dataType, encodedType); - decoder = converter(encodedType, dataType); - return readData.encode(out -> { - numBytes = bytes(dataType); - numEncodedBytes = bytes(encodedType); - - encoder = converter(dataType, encodedType); - decoder = converter(encodedType, dataType); - return new FixedLengthConvertedOutputStream(numBytes, numEncodedBytes, encoder, out); - }); - } - - @Override public ReadData decode(ReadData readData) throws N5IOException { - - return ReadData.from(out -> { - numBytes = bytes(dataType); - numEncodedBytes = bytes(encodedType); - - encoder = converter(dataType, encodedType); - decoder = converter(encodedType, dataType); - - final FixedLengthConvertedInputStream convertedIn = new FixedLengthConvertedInputStream(numEncodedBytes, numBytes, decoder, readData.inputStream()); - ReadData.from(convertedIn).writeTo(out); - }); - } - - public static int bytes(DataType type) { - - switch (type) { - case UINT8: - case INT8: - return 1; - case UINT16: - case INT16: - return 2; - case UINT32: - case INT32: - case FLOAT32: - return 4; - case UINT64: - case INT64: - case FLOAT64: - return 8; - default: - return -1; - } - } - - public static BiConsumer converter(final DataType from, final DataType to) { - - // // TODO fill this out - - if (from == to) - return AsTypeCodec::IDENTITY; - else if (from == DataType.INT8) { - - if( to == DataType.INT16 ) - return AsTypeCodec::BYTE_TO_SHORT; - else if( to == DataType.INT32 ) - return AsTypeCodec::BYTE_TO_INT; - else if( to == DataType.INT64 ) - return AsTypeCodec::BYTE_TO_LONG; - else if( to == DataType.FLOAT32 ) - return AsTypeCodec::BYTE_TO_FLOAT; - else if( to == DataType.FLOAT64 ) - return AsTypeCodec::BYTE_TO_DOUBLE; - - } else if (from == DataType.INT16) { - - if (to == DataType.INT8) - return AsTypeCodec::SHORT_TO_BYTE; - else if (to == DataType.INT32) - return AsTypeCodec::SHORT_TO_INT; - else if (to == DataType.INT64) - return AsTypeCodec::SHORT_TO_LONG; - else if (to == DataType.FLOAT32) - return AsTypeCodec::SHORT_TO_FLOAT; - else if (to == DataType.FLOAT64) - return AsTypeCodec::SHORT_TO_DOUBLE; - - } else if (from == DataType.INT32) { - - if (to == DataType.INT8) - return AsTypeCodec::INT_TO_BYTE; - else if (to == DataType.INT16) - return AsTypeCodec::INT_TO_SHORT; - if (to == DataType.INT8) - return AsTypeCodec::INT_TO_BYTE; - else if (to == DataType.INT16) - return AsTypeCodec::INT_TO_SHORT; - else if (to == DataType.INT32) - return AsTypeCodec::IDENTITY; - else if (to == DataType.INT64) - return AsTypeCodec::INT_TO_LONG; - else if (to == DataType.FLOAT32) - return AsTypeCodec::INT_TO_FLOAT; - else if (to == DataType.INT64) - return AsTypeCodec::INT_TO_LONG; - else if (to == DataType.FLOAT32) - return AsTypeCodec::INT_TO_FLOAT; - else if (to == DataType.FLOAT64) - return AsTypeCodec::INT_TO_DOUBLE; - - } else if (from == DataType.INT64) { - - if (to == DataType.INT8) - return AsTypeCodec::LONG_TO_BYTE; - else if (to == DataType.INT16) - return AsTypeCodec::LONG_TO_SHORT; - else if (to == DataType.INT32) - return AsTypeCodec::LONG_TO_INT; - else if (to == DataType.FLOAT32) - return AsTypeCodec::LONG_TO_FLOAT; - else if (to == DataType.FLOAT64) - return AsTypeCodec::LONG_TO_DOUBLE; - - } else if (from == DataType.FLOAT32) { - - if (to == DataType.INT8) - return AsTypeCodec::FLOAT_TO_BYTE; - else if (to == DataType.INT16) - return AsTypeCodec::FLOAT_TO_SHORT; - else if (to == DataType.INT32) - return AsTypeCodec::FLOAT_TO_INT; - else if (to == DataType.INT64) - return AsTypeCodec::FLOAT_TO_LONG; - else if (to == DataType.FLOAT64) - return AsTypeCodec::FLOAT_TO_DOUBLE; - - } else if (from == DataType.FLOAT64) { - - if (to == DataType.INT8) - return AsTypeCodec::DOUBLE_TO_BYTE; - else if (to == DataType.INT16) - return AsTypeCodec::DOUBLE_TO_SHORT; - else if (to == DataType.INT32) - return AsTypeCodec::DOUBLE_TO_INT; - else if (to == DataType.INT64) - return AsTypeCodec::DOUBLE_TO_LONG; - else if (to == DataType.FLOAT32) - return AsTypeCodec::DOUBLE_TO_FLOAT; - } - - return AsTypeCodec::IDENTITY; - } - - public static final void IDENTITY(final ByteBuffer x, final ByteBuffer y) { - - for (int i = 0; i < y.capacity(); i++) - y.put(x.get()); - } - - public static final void IDENTITY_ONE(final ByteBuffer x, final ByteBuffer y) { - - y.put(x.get()); - } - - public static final void BYTE_TO_SHORT(final ByteBuffer b, final ByteBuffer s) { - - final byte zero = 0; - s.put(zero); - s.put(b.get()); - } - - public static final void BYTE_TO_INT(final ByteBuffer b, final ByteBuffer i) { - - final byte zero = 0; - i.put(zero); - i.put(zero); - i.put(zero); - i.put(b.get()); - } - - public static final void BYTE_TO_LONG(final ByteBuffer b, final ByteBuffer l) { - - final byte zero = 0; - l.put(zero); - l.put(zero); - l.put(zero); - l.put(zero); - l.put(zero); - l.put(zero); - l.put(zero); - l.put(b.get()); - } - - public static final void BYTE_TO_FLOAT(final ByteBuffer b, final ByteBuffer f) { - - f.putFloat((float)b.get()); - } - - public static final void BYTE_TO_DOUBLE(final ByteBuffer b, final ByteBuffer d) { - - d.putDouble((double)b.get()); - } - - public static final void SHORT_TO_BYTE(final ByteBuffer s, final ByteBuffer b) { - - final byte zero = 0; - b.put(zero); - b.put(s.get()); - } - - public static final void SHORT_TO_INT(final ByteBuffer s, final ByteBuffer i) { - - final byte zero = 0; - i.put(zero); - i.put(zero); - i.put(s.get()); - i.put(s.get()); - } - - public static final void SHORT_TO_LONG(final ByteBuffer s, final ByteBuffer l) { - - final byte zero = 0; - l.put(zero); - l.put(zero); - l.put(zero); - l.put(zero); - l.put(zero); - l.put(zero); - l.put(s.get()); - l.put(s.get()); - } - - public static final void SHORT_TO_FLOAT(final ByteBuffer s, final ByteBuffer f) { - - f.putFloat((float)s.getShort()); - } - - public static final void SHORT_TO_DOUBLE(final ByteBuffer s, final ByteBuffer d) { - - d.putDouble((double)s.getShort()); - } - - public static final void INT_TO_BYTE(final ByteBuffer i, final ByteBuffer b) { - - b.put(i.get(3)); - } - - public static final void INT_TO_SHORT(final ByteBuffer i, final ByteBuffer s) { - - s.put(i.get(2)); - s.put(i.get(3)); - } - - public static final void INT_TO_LONG(final ByteBuffer i, final ByteBuffer l) { - - final byte zero = 0; - l.put(zero); - l.put(zero); - l.put(zero); - l.put(zero); - l.put(i.get()); - l.put(i.get()); - l.put(i.get()); - l.put(i.get()); - } - - public static final void INT_TO_FLOAT(final ByteBuffer i, final ByteBuffer f) { - - f.putFloat((float)i.getInt()); - } - - public static final void INT_TO_DOUBLE(final ByteBuffer i, final ByteBuffer f) { - - f.putDouble((float)i.getInt()); - } - - public static final void LONG_TO_BYTE(final ByteBuffer l, final ByteBuffer b) { - - b.put((byte)l.getLong()); - } - - public static final void LONG_TO_SHORT(final ByteBuffer l, final ByteBuffer s) { - - s.putShort((short)l.getLong()); - } - - public static final void LONG_TO_INT(final ByteBuffer l, final ByteBuffer i) { - - i.putInt((int)l.getLong()); - } - - public static final void LONG_TO_FLOAT(final ByteBuffer l, final ByteBuffer f) { - - f.putFloat((float)l.getLong()); - } - - public static final void LONG_TO_DOUBLE(final ByteBuffer l, final ByteBuffer f) { - - f.putDouble((float)l.getLong()); - } - - public static final void FLOAT_TO_BYTE(final ByteBuffer f, final ByteBuffer b) { - - b.put((byte)f.getFloat()); - } - - public static final void FLOAT_TO_SHORT(final ByteBuffer f, final ByteBuffer s) { - - s.putShort((short)f.getFloat()); - } - - public static final void FLOAT_TO_INT(final ByteBuffer f, final ByteBuffer i) { - - i.putInt((int)f.getFloat()); - } - - public static final void FLOAT_TO_LONG(final ByteBuffer f, final ByteBuffer l) { - - l.putLong((long)f.getFloat()); - } - - public static final void FLOAT_TO_DOUBLE(final ByteBuffer f, final ByteBuffer d) { - - d.putDouble((double)f.getFloat()); - } - - public static final void DOUBLE_TO_BYTE(final ByteBuffer d, final ByteBuffer b) { - - b.put((byte)d.getDouble()); - } - - public static final void DOUBLE_TO_SHORT(final ByteBuffer d, final ByteBuffer s) { - - s.putShort((short)d.getDouble()); - } - - public static final void DOUBLE_TO_INT(final ByteBuffer d, final ByteBuffer i) { - - i.putInt((int)d.getDouble()); - } - - public static final void DOUBLE_TO_LONG(final ByteBuffer d, final ByteBuffer l) { - - l.putLong((long)d.getDouble()); - } - - public static final void DOUBLE_TO_FLOAT(final ByteBuffer d, final ByteBuffer f) { - - f.putFloat((float)d.getDouble()); - } - - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java deleted file mode 100644 index 33c90f0af..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetCodec.java +++ /dev/null @@ -1,115 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec; - -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.serialization.NameConfig; - -import java.nio.ByteBuffer; -import java.util.function.BiConsumer; - -@NameConfig.Name(FixedScaleOffsetCodec.TYPE) -public class FixedScaleOffsetCodec extends AsTypeCodec { - - private static final long serialVersionUID = 8024945290803548528L; - - public static transient final String TYPE = "fixedscaleoffset"; - - @NameConfig.Parameter - protected final double scale; - - @NameConfig.Parameter - protected final double offset; - - private transient ByteBuffer tmpEncoder; - private transient ByteBuffer tmpDecoder; - - public transient BiConsumer encoder; - public transient BiConsumer encoderPre; - public transient BiConsumer encoderPost; - public transient BiConsumer decoder; - public transient BiConsumer decoderPre; - public transient BiConsumer decoderPost; - - private FixedScaleOffsetCodec() { - - this(1, 0, null, null); - } - - public FixedScaleOffsetCodec(final double scale, final double offset, DataType type, DataType encodedType) { - - super(type, encodedType); - this.scale = scale; - this.offset = offset; - - tmpEncoder = ByteBuffer.wrap(new byte[Double.BYTES]); - tmpDecoder = ByteBuffer.wrap(new byte[Double.BYTES]); - - // encoder goes from type to encoded type - encoderPre = converter(type, DataType.FLOAT64); - encoderPost = converter(DataType.FLOAT64, encodedType); - - // decoder goes from encoded type to type - decoderPre = converter(encodedType, DataType.FLOAT64); - decoderPost = converter(DataType.FLOAT64, type); - - // convert from i type to double, apply scale and offset, then convert to type o - encoder = (i, o) -> { - tmpEncoder.rewind(); - encoderPre.accept(i, tmpEncoder); - tmpEncoder.rewind(); - final double x = tmpEncoder.getDouble(); - tmpEncoder.rewind(); - tmpEncoder.putDouble(scale * x + offset); - tmpEncoder.rewind(); - encoderPost.accept(tmpEncoder, o); - }; - - // convert from i type to double, apply scale and offset, then convert to type o - decoder = (i, o) -> { - tmpDecoder.rewind(); - decoderPre.accept(i, tmpDecoder); - tmpDecoder.rewind(); - final double x = tmpDecoder.getDouble(); - tmpDecoder.rewind(); - tmpDecoder.putDouble((x - offset) / scale); - tmpDecoder.rewind(); - decoderPost.accept(tmpDecoder, o); - }; - } - - public double getScale() { - - return scale; - } - - public double getOffset() { - - return offset; - } - - @Override - public String getType() { - - return TYPE; - } - - @Override public ReadData decode(ReadData readData) throws N5IOException { - - numBytes = bytes(dataType); - numEncodedBytes = bytes(encodedType); - return ReadData.from(new FixedLengthConvertedInputStream(numEncodedBytes, numBytes, this.decoder, readData.inputStream())); - } - - @Override public ReadData encode(ReadData readData) { - - return readData.encode(out -> { - - numBytes = bytes(dataType); - numEncodedBytes = bytes(encodedType); - return new FixedLengthConvertedOutputStream(numBytes, numEncodedBytes, this.encoder, out); - }); - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 5661446b7..96dc92f13 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -57,9 +57,7 @@ import org.janelia.saalfeldlab.n5.N5Reader.Version; import org.janelia.saalfeldlab.n5.shard.InMemoryShard; import org.janelia.saalfeldlab.n5.shard.Shard; -import org.janelia.saalfeldlab.n5.shard.VirtualShard; import org.janelia.saalfeldlab.n5.url.UriAttributeTest; -import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.junit.After; @@ -273,35 +271,7 @@ public void testWriteReadByteBlock() { } @Test - public void testWriteReadByteBlockMultipleCodecs() { - - /*TODO: this tests "passes" in the sense that we get the correct output, but it - * maybe is not the behavior we actually want*/ - - try (final N5Writer n5 = createTempN5Writer()) { - final String dataset = "8_64_32"; - n5.remove(dataset); - final Codec[] codecs = { - new N5BlockCodec(), - new AsTypeCodec(DataType.INT8, DataType.INT32), - new AsTypeCodec(DataType.INT32, DataType.INT64) - }; - final byte[] byteBlock1 = new byte[]{1,2,3,4,5,6,7,8}; - final long[] dimensions1 = new long[]{2,2,2}; - final int[] blockSize1 = new int[]{2,2,2}; - final DatasetAttributes attrs = new DatasetAttributes(dimensions1, blockSize1, DataType.INT8, codecs); - n5.createDataset(dataset, attrs); - final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize1, new long[]{0, 0, 0}, byteBlock1); - n5.writeBlock(dataset, attributes, dataBlock); - - final DataBlock loadedDataBlock = n5.readBlock(dataset, attrs, 0, 0, 0); - assertArrayEquals(byteBlock1, (byte[])loadedDataBlock.getData()); - } - } - - @Test - public void testWriteReadStringBlock() throws IOException, URISyntaxException { + public void testWriteReadStringBlock() { // test dataset; all characters are valid UTF8 but may have different numbers of bytes! final DataType dataType = DataType.STRING; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java deleted file mode 100644 index c1ff93a14..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/AsTypeTests.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; - -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.junit.Test; - -public class AsTypeTests { - - @Test - public void testInt2Byte() throws IOException { - - final int N = 16; - final ByteBuffer intsAsBuffer = ByteBuffer.allocate(Integer.BYTES * N); - final byte[] encodedBytes = new byte[N]; - for (int i = 0; i < N; i++) { - intsAsBuffer.putInt(i); - encodedBytes[i] = (byte)i; - } - - final byte[] decodedInts = intsAsBuffer.array(); - testEncodingAndDecoding(new AsTypeCodec(DataType.INT32, DataType.INT8), encodedBytes, decodedInts); - testEncodingAndDecoding(new AsTypeCodec(DataType.INT8, DataType.INT32), decodedInts, encodedBytes); - } - - @Test - public void testDouble2Byte() throws IOException { - - final int N = 16; - final ByteBuffer doublesAsBuffer = ByteBuffer.allocate(Double.BYTES * N); - final byte[] encodedBytes = new byte[N]; - for (int i = 0; i < N; i++) { - doublesAsBuffer.putDouble(i); - encodedBytes[i] = (byte)i; - } - final byte[] decodedDoubles = doublesAsBuffer.array(); - - testEncodingAndDecoding(new AsTypeCodec(DataType.FLOAT64, DataType.INT8), encodedBytes, decodedDoubles); - testEncodingAndDecoding(new AsTypeCodec(DataType.INT8, DataType.FLOAT64), decodedDoubles, encodedBytes); - } - - public static void testEncodingAndDecoding(Codec.BytesCodec codec, byte[] encodedBytes, byte[] decodedBytes) throws IOException { - - testEncoding(codec, encodedBytes, decodedBytes); - testDecoding(codec, decodedBytes, encodedBytes); - } - - public static void testDecoding(final Codec.BytesCodec codec, final byte[] expected, final byte[] input) throws IOException { - - final ReadData result = codec.decode(ReadData.from(input)); - assertArrayEquals(expected, result.allBytes()); - } - - public static void testEncoding(final Codec.BytesCodec codec, final byte[] expected, final byte[] data) throws IOException { - - final byte[] encodedData = codec.encode(ReadData.from(data)).allBytes(); - assertArrayEquals(expected, encodedData); - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java index 0610c7c50..ff67a9c8a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java @@ -1,16 +1,12 @@ package org.janelia.saalfeldlab.n5.serialization; -import static org.janelia.saalfeldlab.n5.NameConfigAdapter.getJsonAdapter; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.GsonUtils; import org.janelia.saalfeldlab.n5.GzipCompression; -import org.janelia.saalfeldlab.n5.NameConfigAdapter; -import org.janelia.saalfeldlab.n5.codec.AsTypeCodec; import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.FixedScaleOffsetCodec; import org.janelia.saalfeldlab.n5.codec.IdentityCodec; import org.junit.Before; import org.junit.Test; @@ -43,49 +39,34 @@ public void testSerializeIdentity() { assertEquals("identity", expected, jsonId.getAsJsonObject()); } - @Test - public void testSerializeAsType() { - - final AsTypeCodec asTypeCodec = new AsTypeCodec(DataType.FLOAT64, DataType.INT16); - final JsonObject jsonAsType = gson.toJsonTree(asTypeCodec).getAsJsonObject(); - final JsonElement expected = gson.fromJson( - "{\"name\":\"astype\",\"configuration\":{\"dataType\":\"float64\",\"encodedType\":\"int16\"}}", - JsonElement.class); - assertEquals("asType", expected, jsonAsType.getAsJsonObject()); - } - @Test public void testSerializeCodecArray() { Codec[] codecs = new Codec[]{ - new IdentityCodec(), - new AsTypeCodec(DataType.FLOAT64, DataType.INT16) + new IdentityCodec() }; JsonArray jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); JsonElement expected = gson.fromJson( - "[{\"name\":\"id\"},{\"name\":\"astype\",\"configuration\":{\"dataType\":\"float64\",\"encodedType\":\"int16\"}}]", + "[{\"name\":\"id\"}]", JsonElement.class); assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); Codec[] codecsDeserialized = gson.fromJson(expected, Codec[].class); - assertEquals("codecs length not 2", 2, codecsDeserialized.length); + assertEquals("codecs length not 1", 1, codecsDeserialized.length); assertTrue("first codec not identity", codecsDeserialized[0] instanceof IdentityCodec); - assertTrue("second codec not asType", codecsDeserialized[1] instanceof AsTypeCodec); codecs = new Codec[]{ - new AsTypeCodec(DataType.FLOAT64, DataType.INT16), new GzipCompression() }; jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); expected = gson.fromJson( - "[{\"name\":\"astype\",\"configuration\":{\"dataType\":\"float64\",\"encodedType\":\"int16\"}},{\"name\":\"gzip\",\"configuration\":{\"level\":-1,\"useZlib\":false}}]", + "[{\"name\":\"gzip\",\"configuration\":{\"level\":-1,\"useZlib\":false}}]", JsonElement.class); assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); codecsDeserialized = gson.fromJson(expected, Codec[].class); - assertEquals("codecs length not 2", 2, codecsDeserialized.length); - assertTrue("first codec not asType", codecsDeserialized[0] instanceof AsTypeCodec); - assertTrue("second codec not gzip", codecsDeserialized[1] instanceof GzipCompression); + assertEquals("codecs length not 1", 1, codecsDeserialized.length); + assertTrue("second codec not gzip", codecsDeserialized[0] instanceof GzipCompression); } } From a541c2f03bec381b0cb41bf02f40460cebce0ea5 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 27 Jun 2025 13:08:02 -0400 Subject: [PATCH 254/423] chore: cleanup for PR --- .../saalfeldlab/n5/AbstractDataBlock.java | 4 +- .../saalfeldlab/n5/ByteArrayDataBlock.java | 4 +- .../n5/CachedGsonKeyValueN5Reader.java | 2 +- .../saalfeldlab/n5/CompressionAdapter.java | 4 +- .../org/janelia/saalfeldlab/n5/DataBlock.java | 4 +- .../org/janelia/saalfeldlab/n5/DataType.java | 4 +- .../saalfeldlab/n5/DatasetAttributes.java | 54 +++++----- .../saalfeldlab/n5/DoubleArrayDataBlock.java | 4 +- .../n5/FileSystemKeyValueAccess.java | 6 -- .../saalfeldlab/n5/FloatArrayDataBlock.java | 4 +- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 32 +++--- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 46 +++++---- .../janelia/saalfeldlab/n5/GsonN5Reader.java | 3 +- .../saalfeldlab/n5/HttpKeyValueAccess.java | 16 ++- .../saalfeldlab/n5/IntArrayDataBlock.java | 4 +- .../saalfeldlab/n5/KeyValueAccess.java | 4 +- .../janelia/saalfeldlab/n5/LockedChannel.java | 11 +- .../saalfeldlab/n5/LongArrayDataBlock.java | 4 +- .../saalfeldlab/n5/Lz4Compression.java | 3 +- .../janelia/saalfeldlab/n5/N5Exception.java | 27 ----- .../org/janelia/saalfeldlab/n5/N5Reader.java | 7 +- .../org/janelia/saalfeldlab/n5/N5URI.java | 2 +- .../org/janelia/saalfeldlab/n5/N5Writer.java | 37 +------ .../saalfeldlab/n5/NameConfigAdapter.java | 1 - .../saalfeldlab/n5/RawCompression.java | 6 +- .../saalfeldlab/n5/ReflectionUtils.java | 4 +- .../saalfeldlab/n5/ShortArrayDataBlock.java | 4 +- .../saalfeldlab/n5/StringDataBlock.java | 4 +- .../janelia/saalfeldlab/n5/codec/Codec.java | 18 ++-- .../n5/codec/ConcatenatedBytesCodec.java | 10 +- .../saalfeldlab/n5/codec/DataBlockCodec.java | 5 +- .../saalfeldlab/n5/codec/DataCodec.java | 5 +- .../FixedLengthConvertedInputStream.java | 78 -------------- .../FixedLengthConvertedOutputStream.java | 64 ------------ .../saalfeldlab/n5/codec/IdentityCodec.java | 7 +- .../saalfeldlab/n5/codec/N5BlockCodec.java | 10 +- .../saalfeldlab/n5/codec/RawBlockCodecs.java | 6 +- .../saalfeldlab/n5/codec/RawBytes.java | 6 +- .../n5/readdata/InputStreamReadData.java | 4 +- .../saalfeldlab/n5/readdata/ReadData.java | 1 - .../n5/shard/BlockAsShardCodec.java | 39 ++++--- .../saalfeldlab/n5/shard/InMemoryShard.java | 4 +- .../janelia/saalfeldlab/n5/shard/Shard.java | 6 +- .../saalfeldlab/n5/shard/ShardException.java | 14 --- .../saalfeldlab/n5/shard/ShardIndex.java | 1 - .../saalfeldlab/n5/shard/ShardParameters.java | 6 +- .../saalfeldlab/n5/shard/ShardingCodec.java | 28 +++-- .../saalfeldlab/n5/shard/VirtualShard.java | 3 +- .../saalfeldlab/n5/util/GridIterator.java | 77 +++----------- .../janelia/saalfeldlab/n5/util/Position.java | 10 +- .../saalfeldlab/n5/AbstractN5Test.java | 13 +-- .../saalfeldlab/n5/N5BlockAsShardTest.java | 15 ++- .../saalfeldlab/n5/N5ReadBenchmark.java | 4 +- .../saalfeldlab/n5/codec/BytesTests.java | 6 ++ .../codec/FixedConvertedInputStreamTest.java | 86 ---------------- .../codec/FixedConvertedOutputStreamTest.java | 96 ------------------ .../n5/codec/FixedScaleOffsetTests.java | 60 ----------- .../n5/http/HttpReaderFsWriter.java | 5 - .../n5/serialization/CodecSerialization.java | 1 - .../saalfeldlab/n5/shard/ShardIndexTest.java | 2 - .../n5/shard/ShardPropertiesTests.java | 8 +- .../saalfeldlab/n5/shard/ShardTest.java | 25 +---- .../shardExamples/test.n5/attributes.json | 3 - .../test.n5/mid_sharded/attributes.json | 30 ------ .../test.zarr/mid_sharded/attributes.json | 54 ---------- .../shardExamples/test.zarr/mid_sharded/c/0/0 | Bin 88 -> 0 bytes .../test.zarr/mid_sharded/zarr.json | 54 ---------- .../writeReadBlockTest.n5/attributes.json | 1 - .../writeReadBlockTest.n5/shard/0/0 | Bin 336 -> 0 bytes .../shard/attributes.json | 1 - 70 files changed, 244 insertions(+), 927 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedOutputStream.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardException.java delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedInputStreamTest.java delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java delete mode 100644 src/test/resources/shardExamples/test.n5/attributes.json delete mode 100644 src/test/resources/shardExamples/test.n5/mid_sharded/attributes.json delete mode 100644 src/test/resources/shardExamples/test.zarr/mid_sharded/attributes.json delete mode 100644 src/test/resources/shardExamples/test.zarr/mid_sharded/c/0/0 delete mode 100644 src/test/resources/shardExamples/test.zarr/mid_sharded/zarr.json delete mode 100644 src/test/resources/shardExamples/writeReadBlockTest.n5/attributes.json delete mode 100644 src/test/resources/shardExamples/writeReadBlockTest.n5/shard/0/0 delete mode 100644 src/test/resources/shardExamples/writeReadBlockTest.n5/shard/attributes.json diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index f1cf6809f..d69648d49 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index ca9664be0..da2658113 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index 8be7376cb..3f0db47bf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -30,6 +30,7 @@ import java.lang.reflect.Type; +import com.google.gson.JsonSyntaxException; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; import org.janelia.saalfeldlab.n5.cache.N5JsonCacheableContainer; @@ -37,7 +38,6 @@ import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import com.google.gson.JsonSyntaxException; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java index 96d636b66..022f4b78a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 0324add04..5d3124cf3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 843c52521..08bebf1e6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index a759b8e9a..d9312b1af 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -34,7 +34,6 @@ import java.util.HashMap; import java.util.stream.Stream; -import org.janelia.saalfeldlab.n5.N5Exception.N5ShardException; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; @@ -110,6 +109,7 @@ public DatasetAttributes( this.blockSize = blockSize; this.dataType = dataType; + //TODO: refactor? final Codec[] filteredCodecs = Arrays.stream(codecs).filter(it -> !(it instanceof RawCompression)).toArray(Codec[]::new); if (filteredCodecs.length == 0) { byteCodecs = new BytesCodec[]{}; @@ -131,6 +131,7 @@ public DatasetAttributes( .filter(c -> c instanceof BytesCodec) .toArray(BytesCodec[]::new); } + arrayCodec.initialize(this, byteCodecs); } @@ -184,12 +185,14 @@ public boolean isSharded() { } /** - * If this dataset is sharded, return the ArrayCodec as a ShardingCodec + * If this dataset is sharded, return the ArrayCodec as a ShardingCodec. + * If the dataset is NOT sharded, the returned ShardingCodec will allow you + * to read and write blocks as if they were shards. + * * @return the ShardingCodec * - * @throws N5ShardException if the dataset is not sharded */ - public ShardingCodec getShardingCodec() throws N5ShardException { + public ShardingCodec getShardingCodec() { if (getArrayCodec() instanceof ShardingCodec) return (ShardingCodec)getArrayCodec(); else { @@ -237,7 +240,7 @@ public BytesCodec[] getCodecs() { /** * TODO do we keep this? Deprecated in favor of {@link DatasetAttributesAdapter} for serialization * - * @return serilizable properties of {@link DatasetAttributes} + * @return serializable properties of {@link DatasetAttributes} */ @Deprecated public HashMap asMap() { @@ -250,15 +253,6 @@ public HashMap asMap() { return map; } - protected Codec[] concatenateCodecs() { - - final Codec[] allCodecs = new Codec[byteCodecs.length + 1]; - allCodecs[0] = arrayCodec; - for (int i = 0; i < byteCodecs.length; i++) - allCodecs[i + 1] = byteCodecs[i]; - - return allCodecs; - } private static DatasetAttributesAdapter adapter = null; public static DatasetAttributesAdapter getJsonAdapter() { @@ -268,20 +262,6 @@ public static DatasetAttributesAdapter getJsonAdapter() { return adapter; } - public static class InvalidN5DatasetException extends N5Exception { - - public InvalidN5DatasetException(String dataset, String reason, Throwable cause) { - - this(String.format("Invalid dataset %s: %s", dataset, reason), cause); - } - - public InvalidN5DatasetException(String message, Throwable cause) { - - super(message, cause); - } - } - - public static class DatasetAttributesAdapter implements JsonSerializer, JsonDeserializer { @Override public DatasetAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { @@ -338,10 +318,22 @@ public static class DatasetAttributesAdapter implements JsonSerializer Shard readShard( final String keyPath, final DatasetAttributes datasetAttributes, @@ -114,7 +106,7 @@ default Shard readShard( final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(keyPath), shardGridPosition); try { final ReadData readData = getKeyValueAccess().createReadData(path).materialize(); - return new VirtualShard( datasetAttributes, shardGridPosition, readData); + return new VirtualShard<>( datasetAttributes, shardGridPosition, readData); } catch (N5Exception.N5NoSuchKeyException e) { return null; } @@ -130,8 +122,9 @@ default DataBlock readBlock( final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), keyPos); try { - final ReadData decodeData = getKeyValueAccess().createReadData(path); - return datasetAttributes.getArrayCodec().decode(decodeData, gridPosition); + final ReadData readData = getKeyValueAccess().createReadData(path); + final ArrayCodec arrayCodec = datasetAttributes.getArrayCodec(); + return arrayCodec.decode(readData, gridPosition); } catch (N5Exception.N5NoSuchKeyException e) { return null; } @@ -144,10 +137,9 @@ default List> readBlocks( final DatasetAttributes datasetAttributes, final List blockPositions) throws N5Exception { - // TODO which interface should have this implementation? if (datasetAttributes.isSharded()) { - /* Group by shard position */ + //TODO: make static? final Map> shardBlockMap = datasetAttributes.groupBlockPositions(blockPositions); final ArrayList> blocks = new ArrayList<>(); for( Map.Entry> e : shardBlockMap.entrySet()) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 163c8e9b7..7c9e6c13d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -28,20 +28,7 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.Gson; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.JsonSyntaxException; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.shard.InMemoryShard; -import org.janelia.saalfeldlab.n5.shard.Shard; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec; -import org.janelia.saalfeldlab.n5.util.Position; - import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.Arrays; @@ -50,6 +37,18 @@ import java.util.Map.Entry; import java.util.stream.Collectors; +import com.google.gson.JsonSyntaxException; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import org.janelia.saalfeldlab.n5.shard.InMemoryShard; +import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.util.Position; + /** * Default implementation of {@link N5Writer} with JSON attributes parsed with * {@link Gson}. @@ -217,17 +216,18 @@ default boolean removeAttributes(final String pathName, final List attri return removed; } - @Override default void writeBlocks( + @Override + default void writeBlocks( final String datasetPath, final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { if (datasetAttributes.isSharded()) { - /* Group blocks by shard index */ final Map>> shardBlockMap = datasetAttributes.groupBlocks( Arrays.stream(dataBlocks).collect(Collectors.toList())); + //TODO: extract static method? for (final Entry>> e : shardBlockMap.entrySet()) { final long[] shardPosition = e.getKey().get(); @@ -250,6 +250,8 @@ default boolean removeAttributes(final String pathName, final List attri } } + + @Override default void writeBlock( final String path, @@ -267,10 +269,12 @@ default void writeBlock( final LockedChannel channel = getKeyValueAccess().lockForWriting(keyPath); final OutputStream out = channel.newOutputStream() ) { - final Codec.ArrayCodec codec = datasetAttributes.getArrayCodec(); + final ArrayCodec codec = datasetAttributes.getArrayCodec(); codec.encode(dataBlock).writeTo(out); - } catch (IOException e) { - throw new N5IOException(e); + } catch (final IOException | UncheckedIOException e) { + throw new N5IOException( + "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, + e); } } @@ -319,6 +323,8 @@ default boolean deleteBlock( getKeyValueAccess().delete(blockPath); + //TODO: should we try/catch and return false if exceptions are thrown? + /* an IOException should have occurred if anything had failed midway */ return true; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index c4ae61495..15447da04 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -33,7 +33,6 @@ import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonParseException; -import org.janelia.saalfeldlab.n5.codec.Codec; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -47,7 +46,7 @@ public interface GsonN5Reader extends N5Reader { Gson getGson(); - public String getAttributesKey(); + String getAttributesKey(); @Override default Map> listAttributes(final String pathName) throws N5Exception { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index f938cbaab..2d50cb9ad 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -36,6 +36,7 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import java.io.Closeable; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -304,7 +305,7 @@ private static N5Exception validExistsResponse(int code, String responseMsg, Str if (code >= 200 && code < (allowRedirect ? 400 : 300)) return null; final RuntimeException cause = new RuntimeException(message + "( "+ responseMsg + ")(" + code + ")"); - if (code == 404) + if (code == 404 | code == 410) return new N5Exception.N5NoSuchKeyException(message, cause); return new N5Exception(message, cause); @@ -363,11 +364,6 @@ protected HttpObjectChannel(final URI uri, long startByte, long size) { this.size = size; } - @Override public long size() { - - return size; - } - private boolean isPartialRead() { return startByte > 0 || (size >= 0 && size != Long.MAX_VALUE); } @@ -387,10 +383,12 @@ public InputStream newInputStream() throws N5IOException { } } return conn.getInputStream(); + } catch (FileNotFoundException e) { + /*default HttpURLConnection throws FileNotFoundException on 404 or 410 */ + throw new N5Exception.N5NoSuchKeyException("Could not open stream for " + uri, e); } catch (IOException e) { - throw new N5Exception.N5IOException(e); + throw new N5IOException("Could not open stream for " + uri, e); } - } private String rangeString() { @@ -400,7 +398,7 @@ private String rangeString() { } @Override - public Reader newReader() { + public Reader newReader() { final InputStreamReader reader = new InputStreamReader(newInputStream(), StandardCharsets.UTF_8); synchronized (resources) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 9bc5b831e..915c9d65c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 9a73446b4..832e5368a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -344,7 +344,7 @@ default URI uri(final String uriString) throws URISyntaxException { *

    * Implementations of this interface handle the specifics of accessing data from * their respective sources. - * + * * @see ReadData * @see KeyValueAccessReadData */ @@ -372,7 +372,7 @@ interface LazyRead { /** * Returns the total size of the data source in bytes. - * + * * @return the size of the data source in bytes * @throws N5IOException * if an I/O error occurs while trying to get the length diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java index af4657453..f956937e4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java @@ -31,7 +31,6 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import java.io.Closeable; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Reader; @@ -45,8 +44,6 @@ */ public interface LockedChannel extends Closeable { - public long size() throws IOException; - /** * Create a UTF-8 {@link Reader}. * @@ -54,7 +51,7 @@ public interface LockedChannel extends Closeable { * @throws N5IOException * if the reader could not be created */ - public Reader newReader() throws N5IOException; + Reader newReader() throws N5IOException; /** * Create a new {@link InputStream}. @@ -63,7 +60,7 @@ public interface LockedChannel extends Closeable { * @throws N5IOException * if an input stream could not be created */ - public InputStream newInputStream() throws N5IOException; + InputStream newInputStream() throws N5IOException; /** * Create a new UTF-8 {@link Writer}. @@ -72,7 +69,7 @@ public interface LockedChannel extends Closeable { * @throws N5IOException * if a writer could not be created */ - public Writer newWriter() throws N5IOException; + Writer newWriter() throws N5IOException; /** * Create a new {@link OutputStream}. @@ -81,5 +78,5 @@ public interface LockedChannel extends Closeable { * @throws N5IOException * if an output stream could not be created */ - public OutputStream newOutputStream() throws N5IOException; + OutputStream newOutputStream() throws N5IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index 4780e1299..be0d8883d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index 3770c6b7b..ea01879df 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -31,7 +31,6 @@ import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @@ -65,7 +64,7 @@ public boolean equals(final Object other) { } @Override - public ReadData decode(final ReadData readData) throws N5IOException { + public ReadData decode(final ReadData readData) { return ReadData.from(new LZ4BlockInputStream(readData.inputStream())); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java index 2cc4fd2fe..49922748a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java @@ -178,31 +178,4 @@ protected N5JsonParseException( super(message, cause, enableSuppression, writableStackTrace); } } - - public static class N5ShardException extends N5IOException { - - public N5ShardException(final String message) { - - super(message); - } - - public N5ShardException(final String message, final Throwable cause) { - - super(message, cause); - } - - public N5ShardException(final Throwable cause) { - - super(cause); - } - - protected N5ShardException( - final String message, - final Throwable cause, - final boolean enableSuppression, - final boolean writableStackTrace) { - - super(message, cause, enableSuppression, writableStackTrace); - } - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 4c5063a5c..586986811 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -229,8 +229,6 @@ default Version getVersion() throws N5Exception { * * @return the base path URI */ - // TODO: should this throw URISyntaxException or can we assume that this is - // never possible if we were able to instantiate this N5Reader? URI getURI(); /** @@ -361,7 +359,6 @@ default List> readBlocks( * @throws ClassNotFoundException * the class not found exception */ - @SuppressWarnings("unchecked") default T readSerializedBlock( final String dataset, final DatasetAttributes attributes, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java index 945c2d1df..c086acc05 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java @@ -533,7 +533,7 @@ static String removeRelativePathParts(final String path) { /** * Normalize the {@link String attributePath}. *

    - * Attribute paths have a few of special characters: + * Attribute paths have a few special characters: *

      *
    • "." which represents the current element
      • *
    • ".." which represent the previous elemnt
      • diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index ee0419a86..06f79e4cd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -29,7 +29,6 @@ package org.janelia.saalfeldlab.n5; import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.shard.Shard; import java.io.ByteArrayOutputStream; @@ -234,31 +233,6 @@ default void createDataset( createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, codecs)); } - /** - * DEPRECATED. {@link Compression}s are {@link Codec}s. - * Use {@link #createDataset(String, long[], int[], DataType, Codec...)} - *

      - * Creates a dataset. This does not create any data but the path and - * mandatory attributes only. - * - * @param datasetPath dataset path - * @param dimensions the dataset dimensions - * @param blockSize the block size - * @param dataType the data type - * @param compression the compression - * @throws N5Exception the exception - */ - @Deprecated - default void createDataset( - final String datasetPath, - final long[] dimensions, - final int[] blockSize, - final DataType dataType, - final Compression compression) throws N5Exception { - - createDataset(datasetPath, dimensions, blockSize, dataType, new N5BlockCodec(), compression); - } - /** * Writes a {@link DataBlock}. * @@ -274,7 +248,7 @@ void writeBlock( final DataBlock dataBlock) throws N5Exception; /** - * Write multiple data blocks, useful for request aggregation . + * Write multiple data blocks, useful for request aggregation. * * @param datasetPath dataset path * @param datasetAttributes the dataset attributes @@ -307,18 +281,13 @@ void writeShard( final Shard shard) throws N5Exception; /** - * Deletes the block at {@code gridPosition} + * Deletes the block at {@code gridPosition}. * * @param datasetPath dataset path * @param gridPosition position of block to be deleted * @throws N5Exception the exception * - * @return {@code true} if the block at {@code gridPosition} is "empty" - * after - * deletion. The meaning of "empty" is implementation dependent. For - * example "empty" means that no file exists on the file system for - * the - * deleted block in case of the file system implementation. + * @return {@code true} if the block at {@code gridPosition} was deleted. * */ boolean deleteBlock( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java index 3bfff5216..2c7fa9b39 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java @@ -38,7 +38,6 @@ import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.serialization.N5Annotations; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import org.janelia.saalfeldlab.n5.shard.BlockAsShardCodec; import org.scijava.annotations.Index; import org.scijava.annotations.IndexItem; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index a64a4fd9c..6e252e790 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -28,7 +28,6 @@ */ package org.janelia.saalfeldlab.n5; - import org.janelia.saalfeldlab.n5.Compression.CompressionType; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -42,6 +41,7 @@ public boolean equals(final Object other) { return other != null && other.getClass() == RawCompression.class; } + @Override public ReadData encode(final ReadData readData) { return readData; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java index 52d598d69..3a7fed420 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 753ad5e50..e247f76a6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index f26d6cf23..0e628161b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java index 846f06a66..25aedbb13 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -2,15 +2,13 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.io.Serializable; +import static org.janelia.saalfeldlab.n5.N5Exception.*; + /** * {@code Codec}s can encode and decode {@link ReadData} objects. *

      @@ -37,9 +35,9 @@ interface BytesCodec extends Codec { * * @param readData data to decode * @return decoded ReadData - * @throws IOException if any I/O error occurs + * @throws N5IOException if any I/O error occurs */ - ReadData decode(ReadData readData) throws N5Exception.N5IOException; + ReadData decode(ReadData readData) throws N5IOException; /** * Encode the given {@link ReadData}. @@ -49,9 +47,9 @@ interface BytesCodec extends Codec { * * @param readData data to encode * @return encoded ReadData - * @throws IOException if any I/O error occurs + * @throws N5IOException if any I/O error occurs */ - ReadData encode(ReadData readData) throws N5Exception.N5IOException; + ReadData encode(ReadData readData) throws N5IOException; } @@ -61,9 +59,9 @@ interface BytesCodec extends Codec { */ interface ArrayCodec extends DeterministicSizeCodec { - DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException; + DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException; - ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException; + ReadData encode(DataBlock dataBlock) throws N5IOException; default long[] getPositionForBlock(final DatasetAttributes attributes, final DataBlock datablock) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java index 13851691a..9f21ec462 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java @@ -1,6 +1,5 @@ package org.janelia.saalfeldlab.n5.codec; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; public class ConcatenatedBytesCodec implements Codec.BytesCodec { @@ -11,7 +10,8 @@ public ConcatenatedBytesCodec(final BytesCodec... codecs) { this.codecs = codecs; } - @Override public ReadData encode(ReadData readData) throws N5IOException { + @Override + public ReadData encode(ReadData readData) { ReadData encodeData = readData; if (codecs != null) { @@ -22,7 +22,8 @@ public ConcatenatedBytesCodec(final BytesCodec... codecs) { return encodeData; } - @Override public ReadData decode(ReadData readData) throws N5IOException { + @Override + public ReadData decode(ReadData readData) { ReadData decodeData = readData; if (codecs != null) { for (int i = codecs.length - 1; i >= 0; i--) { @@ -33,7 +34,8 @@ public ConcatenatedBytesCodec(final BytesCodec... codecs) { return decodeData; } - @Override public String getType() { + @Override + public String getType() { return "internal-concatenated-codecs"; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java index fa4ff1fc9..a8ef5b582 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -39,7 +39,6 @@ * @param * type of the data contained in the DataBlock */ -//TODO Caleb: replace with ArrayCodec? Or use for Factory naming? public interface DataBlockCodec { ReadData encode(DataBlock dataBlock) throws N5IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java index 94bb14c45..f50f058df 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -38,7 +38,6 @@ import java.util.Arrays; import java.util.function.IntFunction; import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java deleted file mode 100644 index 0b786d041..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedInputStream.java +++ /dev/null @@ -1,78 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.util.function.BiConsumer; - -/** - * An {@link InputStream} that converts between two fixed-length types. - */ -public class FixedLengthConvertedInputStream extends InputStream { - - private final int numBytes; - private final int numBytesAfterDecoding; - - private final byte[] raw; - private final byte[] decoded; - - private final ByteBuffer rawBuffer; - private final ByteBuffer decodedBuffer; - - private final InputStream src; - - private BiConsumer converter; - - private int incrememntalBytesRead; - - public FixedLengthConvertedInputStream( - final int numBytes, - final int numBytesAfterDecoding, - BiConsumer converter, - final InputStream src ) { - - this.numBytes = numBytes; - this.numBytesAfterDecoding = numBytesAfterDecoding; - this.converter = converter; - - raw = new byte[numBytes]; - decoded = new byte[numBytesAfterDecoding]; - incrememntalBytesRead = 0; - - rawBuffer = ByteBuffer.wrap(raw); - decodedBuffer = ByteBuffer.wrap(decoded); - - this.src = src; - } - - @Override - public int read() throws IOException { - - // TODO not sure if this always reads enough bytes - // int n = src.read(toEncode); - if (incrememntalBytesRead == 0) { - - rawBuffer.rewind(); - decodedBuffer.rewind(); - - for (int i = 0; i < numBytes; i++) { - final int retval = src.read(); - if (retval == -1 && i == 0) - return retval; - else if (retval == -1) - throw new IOException("Unexpected end of stream"); - - raw[i] = (byte)retval; - } - - converter.accept(rawBuffer, decodedBuffer); - } - - final int out = decoded[incrememntalBytesRead++]; - if (incrememntalBytesRead == numBytesAfterDecoding) - incrememntalBytesRead = 0; - - return out; - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedOutputStream.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedOutputStream.java deleted file mode 100644 index 87544fc79..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FixedLengthConvertedOutputStream.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.function.BiConsumer; - -/* - * An {@link OutputStream} that converts between two fixed-length types. - */ -public class FixedLengthConvertedOutputStream extends OutputStream { - - private final int numBytes; - - private final byte[] raw; - private final byte[] encoded; - - private final ByteBuffer rawBuffer; - private final ByteBuffer encodedBuffer; - - private final OutputStream src; - - private BiConsumer converter; - - private int incrememntalBytesWritten; - - public FixedLengthConvertedOutputStream( - final int numBytes, - final int numBytesAfterEncoding, - final BiConsumer converter, - final OutputStream src ) { - - this.numBytes = numBytes; - this.converter = converter; - - raw = new byte[numBytes]; - encoded = new byte[numBytesAfterEncoding]; - - rawBuffer = ByteBuffer.wrap(raw); - encodedBuffer = ByteBuffer.wrap(encoded); - - incrememntalBytesWritten = 0; - - this.src = src; - } - - @Override - public void write(int b) throws IOException { - - raw[incrememntalBytesWritten++] = (byte)b; - - // write out the encoded bytes after writing numBytes bytes - if (incrememntalBytesWritten == numBytes) { - - rawBuffer.rewind(); - encodedBuffer.rewind(); - - converter.accept(rawBuffer, encodedBuffer); - src.write(encoded); - incrememntalBytesWritten = 0; - } - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java index fcc368e15..68d035b49 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -1,6 +1,5 @@ package org.janelia.saalfeldlab.n5.codec; -import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @@ -17,12 +16,14 @@ public String getType() { return TYPE; } - @Override public ReadData decode(ReadData readData) { + @Override + public ReadData decode(ReadData readData) { return readData; } - @Override public ReadData encode(ReadData readData) { + @Override + public ReadData encode(ReadData readData) { return readData; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java index c414f2890..c60e78425 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java @@ -5,8 +5,6 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import java.io.IOException; - @NameConfig.Name(value = N5BlockCodec.TYPE) public class N5BlockCodec implements Codec.ArrayCodec { @@ -24,16 +22,18 @@ public class N5BlockCodec implements Codec.ArrayCodec { } private DataBlockCodec getDataBlockCodec() { - /*TODO: Consider an attributes.createDataBlockCodec() without parameters? */ + return N5Codecs.createDataBlockCodec(attributes.getDataType(), bytesCodec); } - @Override public DataBlock decode(ReadData readData, long[] gridPosition) { + @Override + public DataBlock decode(ReadData readData, long[] gridPosition) { return this.getDataBlockCodec().decode(readData, gridPosition); } - @Override public ReadData encode(DataBlock dataBlock) { + @Override + public ReadData encode(DataBlock dataBlock) { return this.getDataBlockCodec().encode(dataBlock); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java index f687ff726..5447f1f3c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java @@ -97,7 +97,8 @@ private int numElements() { return elements; } - @Override public ReadData encode(DataBlock dataBlock) { + @Override + public ReadData encode(DataBlock dataBlock) { return ReadData.from(out -> { final ReadData blockData = dataCodec.serialize(dataBlock.getData()); @@ -105,7 +106,8 @@ private int numElements() { }); } - @Override public DataBlock decode(ReadData readData, long[] gridPosition) { + @Override + public DataBlock decode(ReadData readData, long[] gridPosition) { ReadData decodeData = codec.decode(readData); final T data = dataCodec.deserialize(decodeData, numElements()); return dataBlockFactory.createDataBlock(blockSize, gridPosition, data); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java index 56a4d087b..1f66c8899 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java @@ -13,7 +13,6 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import java.io.IOException; import java.nio.ByteOrder; @@ -51,19 +50,18 @@ public ByteOrder getByteOrder() { this.bytesCodec = new ConcatenatedBytesCodec(byteCodecs); } - private DataBlockCodec< T > getDataBlockCodec() { + private DataBlockCodec getDataBlockCodec() { return RawBlockCodecs.createDataBlockCodec( attributes.getDataType(), getByteOrder(), attributes.getBlockSize(), bytesCodec ); } - @SuppressWarnings("unchecked") @Override public DataBlock decode(ReadData readData, long[] gridPosition) { return this.getDataBlockCodec().decode(readData, gridPosition); } - @SuppressWarnings( "unchecked" ) + @Override public ReadData encode(DataBlock dataBlock) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java index 1646b26cf..0c1f35128 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index b40a2a518..a298897a1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -32,7 +32,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; -import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java index 0877aab4d..872a73078 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java @@ -14,12 +14,14 @@ public class BlockAsShardCodec extends ShardingCodec { private static final RawBytes VIRTUAL_SHARD_INDEX_CODEC = new RawBytes() { - @Override public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { + @Override + public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { return ReadData.from(new byte[0]); } - @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException { + @Override + public DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException { final long[] data = new long[]{ 0, -1}; return new LongArrayDataBlock(new int[0], new long[0], data); @@ -38,7 +40,8 @@ public BlockAsShardCodec(ArrayCodec datasetArrayCodec) { this.datasetArrayCodec = datasetArrayCodec; } - @Override public ShardIndex createIndex(DatasetAttributes attributes) { + @Override + public ShardIndex createIndex(DatasetAttributes attributes) { return new ShardIndex(attributes.getBlocksPerShard(), getIndexLocation(), NO_OP_INDEX_CODECS) { @@ -49,53 +52,63 @@ public BlockAsShardCodec(ArrayCodec datasetArrayCodec) { }; } - @Override public int[] getBlockSize() { + @Override + public int[] getBlockSize() { return datasetAttributes.getBlockSize(); } - @Override public ArrayCodec getArrayCodec() { + @Override + public ArrayCodec getArrayCodec() { return datasetArrayCodec; } - @Override public long[] getPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { + @Override + public long[] getPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { return datasetArrayCodec.getPositionForBlock(attributes, datablock); } - @Override public long[] getPositionForBlock(DatasetAttributes attributes, long... blockPosition) { + @Override + public long[] getPositionForBlock(DatasetAttributes attributes, long... blockPosition) { return datasetArrayCodec.getPositionForBlock(attributes, blockPosition); } - @Override public void initialize(DatasetAttributes attributes, BytesCodec... codecs) { + @Override + public void initialize(DatasetAttributes attributes, BytesCodec... codecs) { this.datasetAttributes = attributes; datasetArrayCodec.initialize(attributes, codecs); } - @Override public long encodedSize(long size) { + @Override + public long encodedSize(long size) { return datasetArrayCodec.encodedSize(size); } - @Override public long decodedSize(long size) { + @Override + public long decodedSize(long size) { return datasetArrayCodec.decodedSize(size); } - @Override public String getType() { + @Override + public String getType() { //TODO Caleb: can we ensure this is never called? return datasetArrayCodec.getType(); } - @Override public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { + @Override + public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { return datasetArrayCodec.encode(dataBlock); } - @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException { + @Override + public DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException { return datasetArrayCodec.decode(readData, gridPosition); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index a2b57d5be..ce8842281 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -2,7 +2,6 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.Position; import java.util.ArrayList; @@ -35,7 +34,8 @@ private void storeBlock(DataBlock block) { blocks.put(Position.wrap(block.getGridPosition()), block); } - @Override public DataBlock getBlock(long... blockGridPosition) { + @Override + public DataBlock getBlock(long... blockGridPosition) { return blocks.get(Position.wrap(blockGridPosition)); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 6693b8b89..5a348eaea 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -1,6 +1,5 @@ package org.janelia.saalfeldlab.n5.shard; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -58,7 +57,7 @@ default int[] getBlockSize() { * * @return position on the shard grid */ - public long[] getGridPosition(); + long[] getGridPosition(); /** * Returns of the block at the given position relative to this shard, or null if this shard does not contain the given block. @@ -158,8 +157,6 @@ default ShardIndex createIndex() { default ReadData createReadData() throws N5IOException { - - final DatasetAttributes datasetAttributes = getDatasetAttributes(); ShardingCodec shardingCodec = datasetAttributes.getShardingCodec(); final Codec.ArrayCodec arrayCodec = shardingCodec.getArrayCodec(); @@ -211,7 +208,6 @@ class DataBlockIterator implements Iterator> { private final GridIterator it; private final Shard shard; private final ShardIndex index; - // TODO ShardParameters is deprecated? private final DatasetAttributes attributes; private int blockIndex = 0; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardException.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardException.java deleted file mode 100644 index d208c62ed..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardException.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import org.janelia.saalfeldlab.n5.N5Exception; - -public class ShardException extends N5Exception { - - private static final long serialVersionUID = -77907634621557855L; - - public static class IndexException extends ShardException { - - private static final long serialVersionUID = 3924426352575114063L; - - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 615a5307c..f1583d9d2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -9,7 +9,6 @@ import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import java.io.IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java index f2f8d734d..a6ff3d25d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java @@ -20,11 +20,11 @@ @Deprecated public interface ShardParameters { - public long[] getDimensions(); + long[] getDimensions(); - public int getNumDimensions(); + int getNumDimensions(); - public int[] getBlockSize(); + int[] getBlockSize(); /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index a83de4c84..3f93eab26 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -33,7 +33,7 @@ public class ShardingCodec implements Codec.ArrayCodec { private DatasetAttributes attributes = null; public enum IndexLocation { - START, END; + START, END } @N5Annotations.ReverseArray // TODO need to reverse for zarr, not for n5 @@ -49,6 +49,9 @@ public enum IndexLocation { @NameConfig.Parameter(value = INDEX_LOCATION_KEY, optional = true) private final IndexLocation indexLocation; + /** + * Used via reflections by the NameConfig serializer. + */ @SuppressWarnings("unused") private ShardingCodec() { @@ -103,31 +106,36 @@ public DeterministicSizeCodec[] getIndexCodecs() { return indexCodecs; } - @Override public long[] getPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { + @Override + public long[] getPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { final long[] blockPosition = datablock.getGridPosition(); return attributes.getShardPositionForBlock(blockPosition); } - @Override public long[] getPositionForBlock(DatasetAttributes attributes, final long... blockPosition) { + @Override + public long[] getPositionForBlock(DatasetAttributes attributes, final long... blockPosition) { return attributes.getShardPositionForBlock(blockPosition); } - @Override public void initialize(DatasetAttributes attributes, final BytesCodec[] codecs) { + @Override + public void initialize(DatasetAttributes attributes, final BytesCodec[] codecs) { + this.attributes = attributes; getArrayCodec().initialize(attributes, getCodecs()); } - @Override public ReadData encode(DataBlock dataBlock) { + @Override + public ReadData encode(DataBlock dataBlock) { return getArrayCodec().encode(dataBlock); } - @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException { + @Override + public DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException { final ReadData splitableReadData = readData.materialize(); - final VirtualShard shard = new VirtualShard<>(attributes, gridPosition, splitableReadData); return shard.getBlock(gridPosition); } @@ -147,7 +155,8 @@ public String getType() { public static class IndexLocationAdapter implements JsonSerializer, JsonDeserializer { - @Override public IndexLocation deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { + @Override + public IndexLocation deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { if (!json.isJsonPrimitive()) return null; @@ -155,7 +164,8 @@ public static class IndexLocationAdapter implements JsonSerializer extends AbstractShard { private final ReadData shardData; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java index 9880e7b30..b201015bb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java @@ -29,7 +29,7 @@ public GridIterator(final long[] dimensions, final long[] min) { final int m = n - 1; long k = steps[0] = 1; - for (int d = 0; d < m;) { + for (int d = 0; d < m; ) { final long dimd = dimensions[d]; this.dimensions[d] = dimd; k *= dimd; @@ -51,113 +51,64 @@ public GridIterator(final int[] dimensions) { } public void fwd() { + ++index; } public void reset() { + index = -1; } @Override public boolean hasNext() { + return index < lastIndex; } @Override public long[] next() { + fwd(); indexToPosition(index, dimensions, min, position); return position; } - public int[] nextAsInt() { - return long2int(next()); - } - public int getIndex() { + return index; } - final static public void indexToPosition(long index, final long[] dimensions, final long[] offset, + public static void indexToPosition(long index, final long[] dimensions, final long[] offset, final long[] position) { - for (int dim = 0; dim < dimensions.length; dim++) { - position[dim] = (index % dimensions[dim]) + offset[dim]; - index /= dimensions[dim]; - } - } - final static public void indexToPosition(long index, final long[] dimensions, final long[] position) { for (int dim = 0; dim < dimensions.length; dim++) { - position[dim] = index % dimensions[dim]; + position[dim] = (index % dimensions[dim]) + offset[dim]; index /= dimensions[dim]; } } - final static public void indexToPosition(long index, final int[] dimensions, final long[] offset, + public static void indexToPosition(long index, final int[] dimensions, final long[] offset, final long[] position) { - for (int dim = 0; dim < dimensions.length; dim++) { - position[dim] = (index % dimensions[dim]) + offset[dim]; - index /= dimensions[dim]; - } - } - final static public void indexToPosition(long index, final int[] dimensions, final long[] position) { for (int dim = 0; dim < dimensions.length; dim++) { - position[dim] = index % dimensions[dim]; + position[dim] = (index % dimensions[dim]) + offset[dim]; index /= dimensions[dim]; } } - final static public long positionToIndex(final long[] dimensions, final long[] position) { - long idx = position[0]; - int cumulativeSize = 1; - for (int i = 0; i < position.length; i++) { - idx += position[i] * cumulativeSize; - cumulativeSize *= dimensions[i]; - } - return idx; - } - - final static public long positionToIndex(final long[] dimensions, final int[] position) { - long idx = position[0]; - int cumulativeSize = 1; - for (int i = 0; i < position.length; i++) { - idx += position[i] * cumulativeSize; - cumulativeSize *= dimensions[i]; - } - return idx; - } + public static int[] long2int(final long[] a) { - final static public long positionToIndex(final int[] dimensions, final int[] position) { - long idx = position[0]; - int cumulativeSize = 1; - for (int i = 0; i < position.length; i++) { - idx += position[i] * cumulativeSize; - cumulativeSize *= dimensions[i]; - } - return idx; - } - - final static public long positionToIndex(final int[] dimensions, final long[] position) { - long idx = position[0]; - int cumulativeSize = 1; - for (int i = 0; i < position.length; i++) { - idx += position[i] * cumulativeSize; - cumulativeSize *= dimensions[i]; - } - return idx; - } - - final static public int[] long2int(final long[] a) { final int[] i = new int[a.length]; for (int d = 0; d < a.length; ++d) - i[d] = (int) a[d]; + i[d] = (int)a[d]; return i; } - final static public long[] int2long(final int[] i) { + public static long[] int2long(final int[] i) { + final long[] l = new long[i.length]; for (int d = 0; d < l.length; ++d) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java index 5ddb8cf0c..eb1a9078f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java @@ -8,9 +8,9 @@ */ public interface Position extends Comparable { - public long[] get(); + long[] get(); - public long get(int i); + long get(int i); default int numDimensions() { return get().length; @@ -29,7 +29,7 @@ default int compareTo(Position other) { return 0; } - public static boolean equals(final Position a, final Object b) { + static boolean equals(final Position a, final Object b) { if (a == null && b == null) return true; @@ -51,11 +51,11 @@ public static boolean equals(final Position a, final Object b) { return true; } - public static String toString(Position p) { + static String toString(Position p) { return "Position: " + Arrays.toString(p.get()); } - public static Position wrap(final long[] p) { + static Position wrap(final long[] p) { return new FinalPosition(p); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 96dc92f13..20e990313 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -59,7 +59,6 @@ import org.janelia.saalfeldlab.n5.shard.Shard; import org.janelia.saalfeldlab.n5.url.UriAttributeTest; import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -323,7 +322,7 @@ public void testWriteReadIntBlock() { DataType.INT32}) { try (final N5Writer n5 = createTempN5Writer()) { - n5.createDataset(datasetName, dimensions, blockSize, dataType, (Codec)compression); + n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final IntArrayDataBlock dataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); n5.writeBlock(datasetName, attributes, dataBlock); @@ -481,7 +480,6 @@ public void testWriteInvalidBlock() { n5.writeBlock(datasetName, attributes, smallDataBlock); final DataBlock loadedSmallDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); - System.out.println(((byte[])loadedSmallDataBlock.getData()).length); assertArrayEquals(smallerData, (byte[])loadedSmallDataBlock.getData()); // write a block of the wrong type @@ -1517,8 +1515,7 @@ private String jsonKeyVal(final String key, final String val) { } @Test - public void - testRootLeaves() { + public void testRootLeaves() { /* Test retrieving non-JsonObject root leaves */ try (final N5Writer n5 = createTempN5Writer()) { @@ -1658,7 +1655,7 @@ public void testPathsWithIllegalUriCharacters() throws IOException, URISyntaxExc } @Test - public void testWriteReadShardOnUnshardedDataset() throws Exception { + public void testWriteReadShardOnUnshardedDataset() { try (N5Writer writer = createTempN5Writer()) { final String datasetName = "testWriteShardOnUnshardedDataset"; final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64, new RawCompression()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java index 19ffa7e31..04022cb89 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java @@ -8,11 +8,13 @@ public class N5BlockAsShardTest extends N5FSTest { - @Override protected N5Writer createN5Writer(String location, GsonBuilder gson) { + @Override + protected N5Writer createN5Writer(String location, GsonBuilder gson) { return new N5FSWriter(location, gson) { - @Override public void writeBlock(String path, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { + @Override + public void writeBlock(String path, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { final ShardIndex index = datasetAttributes.getShardingCodec().createIndex(datasetAttributes); final InMemoryShard shard = new InMemoryShard(datasetAttributes, dataBlock.getGridPosition(), index); @@ -20,7 +22,8 @@ public class N5BlockAsShardTest extends N5FSTest { writeShard(path, datasetAttributes, shard); } - @Override public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { + @Override + public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { final Shard shard = readShard(pathName, datasetAttributes, gridPosition); if (shard == null) @@ -31,11 +34,13 @@ public class N5BlockAsShardTest extends N5FSTest { }; } - @Override protected N5Reader createN5Reader(String location, GsonBuilder gson) { + @Override + protected N5Reader createN5Reader(String location, GsonBuilder gson) { return new N5FSReader(location, gson) { - @Override public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { + @Override + public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { final Shard shard = readShard(pathName, datasetAttributes, gridPosition); if (shard == null) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java index 3e245c676..de771041b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java index d36f64bbe..6c7df18ce 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java @@ -1,7 +1,9 @@ package org.janelia.saalfeldlab.n5.codec; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.nio.ByteOrder; @@ -20,6 +22,10 @@ public class BytesTests { @Test public void testSerialization() { + //TODO: we don't intend to support codec serialization for N5. + // Rewrite this to directly test NameConfigAdapter bit Gson + fail("Rewrite this to directly test NameConfigAdapter with Gson"); + final N5Factory factory = new N5Factory(); factory.cacheAttributes(false); final GsonBuilder gsonBuilder = new GsonBuilder(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedInputStreamTest.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedInputStreamTest.java deleted file mode 100644 index 27c744fa4..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedInputStreamTest.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec; - -import static org.junit.Assert.assertEquals; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.stream.IntStream; - -import org.junit.Test; - -public class FixedConvertedInputStreamTest { - - @Test - public void testLengthOne() throws IOException - { - - final byte expected = 5; - final byte[] data = new byte[32]; - Arrays.fill(data, expected); - - final FixedLengthConvertedInputStream convertedId = new FixedLengthConvertedInputStream(1, 1, - AsTypeCodec::IDENTITY_ONE, - new ByteArrayInputStream(data)); - - final FixedLengthConvertedInputStream convertedPlusOne = new FixedLengthConvertedInputStream(1, 1, - (x, y) -> { - y.put((byte)(x.get() + 1)); - }, - new ByteArrayInputStream(data)); - - for (int i = 0; i < 32; i++) { - assertEquals(expected, convertedId.read()); - assertEquals(expected + 1, convertedPlusOne.read()); - } - - convertedId.close(); - convertedPlusOne.close(); - } - - @Test - public void testIntToByte() throws IOException - { - - final int N = 16; - final ByteBuffer buf = ByteBuffer.allocate(Integer.BYTES * N); - IntStream.range(0, N).forEach( x -> { - buf.putInt(x); - }); - - final byte[] data = buf.array(); - final FixedLengthConvertedInputStream intToByte = new FixedLengthConvertedInputStream( - 4, 1, - AsTypeCodec::INT_TO_BYTE, - new ByteArrayInputStream(data)); - - for( int i = 0; i < N; i++ ) - assertEquals((byte)i, intToByte.read()); - - intToByte.close(); - } - - @Test - public void testByteToInt() throws IOException - { - - final int N = 16; - final byte[] data = new byte[16]; - for( int i = 0; i < N; i++ ) - data[i] = (byte)i; - - final FixedLengthConvertedInputStream byteToInt = new FixedLengthConvertedInputStream( - 1, 4, AsTypeCodec::BYTE_TO_INT, - new ByteArrayInputStream(data)); - - final DataInputStream dataStream = new DataInputStream(byteToInt); - for( int i = 0; i < N; i++ ) - assertEquals(i, dataStream.readInt()); - - dataStream.close(); - byteToInt.close(); - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java deleted file mode 100644 index f8cf5215c..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedConvertedOutputStreamTest.java +++ /dev/null @@ -1,96 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.stream.IntStream; - -import org.junit.Test; - -public class FixedConvertedOutputStreamTest { - - @Test - public void testLengthOne() throws IOException - { - final int N = 2; - final byte expected = 5; - final byte expectedPlusOne = 6; - final byte[] expectedData = new byte[N]; - Arrays.fill(expectedData, expected); - - final byte[] expectedPlusOneData = new byte[N]; - Arrays.fill(expectedPlusOneData, expectedPlusOne); - - final ByteArrayOutputStream outId = new ByteArrayOutputStream(N); - final FixedLengthConvertedOutputStream convertedId = new FixedLengthConvertedOutputStream(1, 1, - AsTypeCodec::IDENTITY_ONE, - outId); - - convertedId.write(expectedData); - convertedId.flush(); - convertedId.close(); - - assertArrayEquals(expectedData, outId.toByteArray()); - - - final ByteArrayOutputStream outPlusOne = new ByteArrayOutputStream(N); - final FixedLengthConvertedOutputStream convertedPlusOne = new FixedLengthConvertedOutputStream(1, 1, - (x, y) -> y.put((byte)(x.get() + 1)), - outPlusOne); - - convertedPlusOne.write(expectedData); - convertedPlusOne.close(); - assertArrayEquals(expectedPlusOneData, outPlusOne.toByteArray()); - } - - @Test - public void testIntToByte() throws IOException - { - - final int N = 16; - final ByteBuffer buf = ByteBuffer.allocate(Integer.BYTES * N); - IntStream.range(0, N).forEach(buf::putInt); - - final ByteBuffer expected = ByteBuffer.allocate(N); - IntStream.range(0, N).forEach( x -> expected.put((byte)x)); - - final ByteArrayOutputStream outStream = new ByteArrayOutputStream(N); - final FixedLengthConvertedOutputStream intToByte = new FixedLengthConvertedOutputStream( - 4, 1, - AsTypeCodec::INT_TO_BYTE, - outStream); - - intToByte.write(buf.array()); - intToByte.close(); - - assertArrayEquals(expected.array(), outStream.toByteArray()); - } - @Test - public void testByteToInt() throws IOException - { - - final int N = 16; - final byte[] data = new byte[16]; - for( int i = 0; i < N; i++ ) - data[i] = (byte)i; - - FixedLengthConvertedInputStream byteToInt = new FixedLengthConvertedInputStream( - 1, 4, - (input, output) -> output.putInt(input.get()), - new ByteArrayInputStream(data)); - - final DataInputStream dataStream = new DataInputStream(byteToInt); - for( int i = 0; i < N; i++ ) - assertEquals(i, dataStream.readInt()); - - dataStream.close(); - byteToInt.close(); - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java deleted file mode 100644 index 135a7f2ba..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/FixedScaleOffsetTests.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.stream.DoubleStream; - -import org.janelia.saalfeldlab.n5.DataType; -import org.junit.Test; - -public class FixedScaleOffsetTests { - - @Test - public void testDouble2Byte() throws IOException { - - final int N = 16; - final double[] doubles = DoubleStream.iterate(0.0, x -> x + 1).limit(N).toArray(); - final ByteBuffer encodedDoubles = ByteBuffer.allocate(Double.BYTES * N); - final byte[] bytes = new byte[N]; - - final double scale = 2; - final double offset = 1; - - for (int i = 0; i < N; i++) { - final double val = (scale * doubles[i] + offset); - bytes[i] = (byte)val; - encodedDoubles.putDouble(i); - } - - final FixedScaleOffsetCodec double2Byte = new FixedScaleOffsetCodec(scale, offset, DataType.FLOAT64, DataType.INT8); - AsTypeTests.testEncoding(double2Byte, bytes, encodedDoubles.array()); - AsTypeTests.testDecoding(double2Byte, encodedDoubles.array(), bytes); - } - - @Test - public void testLong2Short() throws IOException { - - final int N = 16; - final ByteBuffer encodedLongs = ByteBuffer.allocate(Long.BYTES * N); - final ByteBuffer encodedShorts = ByteBuffer.allocate(Short.BYTES * N); - - final long scale = 2; - final long offset = 1; - - for (int i = 0; i < N; i++) { - final long val = (scale * i + offset); - encodedShorts.putShort((short)val); - encodedLongs.putLong(i); - } - - final byte[] shortBytes = encodedShorts.array(); - final byte[] longBytes = encodedLongs.array(); - - final FixedScaleOffsetCodec long2short = new FixedScaleOffsetCodec(scale, offset, DataType.INT64, DataType.INT16); - AsTypeTests.testEncoding(long2short, shortBytes, longBytes); - AsTypeTests.testDecoding(long2short, longBytes, shortBytes); - } - - - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index 6556bfcaa..fc4f17bac 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -265,11 +265,6 @@ public HttpRead writer.createDataset(datasetPath, datasetAttributes); } - @Override public void createDataset(String datasetPath, long[] dimensions, int[] blockSize, DataType dataType, Compression compression) throws N5Exception { - - writer.createDataset(datasetPath, dimensions, blockSize, dataType, compression); - } - @Override public void writeBlock(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { writer.writeBlock(datasetPath, datasetAttributes, dataBlock); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java index ff67a9c8a..8b21fd1ac 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java @@ -3,7 +3,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.GsonUtils; import org.janelia.saalfeldlab.n5.GzipCompression; import org.janelia.saalfeldlab.n5.codec.Codec; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index d0bc2c64a..068a82b64 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -81,8 +81,6 @@ public void writeReadTest() throws IOException { index.set(93, 111, new int[]{3, 0}); index.set(143, 1, new int[]{1, 2}); - final long indexSize = index.getArrayCodec().encodedSize(index.numBytes()); - long currentSize; try { currentSize = kva.size(path); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java index 50ce36a13..97aff4dc6 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java @@ -21,7 +21,7 @@ public class ShardPropertiesTests { @Test - public void testShardProperties() throws Exception { + public void testShardProperties() { final long[] arraySize = new long[]{16, 16}; final int[] shardSize = new int[]{16, 16}; @@ -41,7 +41,7 @@ public void testShardProperties() throws Exception { ) ); - @SuppressWarnings({"rawtypes"}) final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); + final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); assertArrayEquals(new int[]{4, 4}, shard.getBlockGridSize()); @@ -60,7 +60,7 @@ public void testShardProperties() throws Exception { } @Test - public void testShardBlockPositionIterator() throws Exception { + public void testShardBlockPositionIterator() { final long[] arraySize = new long[]{16, 16}; final int[] shardSize = new int[]{16, 16}; @@ -80,7 +80,7 @@ public void testShardBlockPositionIterator() throws Exception { ) ); - @SuppressWarnings({"rawtypes", "unchecked"}) final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); + final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); int i = 0; Iterator it = shard.blockPositionIterator(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index f0ccb1315..850c3d2ae 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -23,7 +23,6 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import java.io.IOException; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Arrays; @@ -208,7 +207,7 @@ public void writeReadBlockTest() { final String dataset = "writeReadBlock"; writer.remove(dataset); writer.createDataset(dataset, datasetAttributes); - writer.deleteBlock(dataset, 0, 0); //TODO Caleb: We are abusing this here. It shouldn't delete the entire shard.. + writer.deleteBlock(dataset, 0, 0); //FIXME Caleb: We are abusing this here. It shouldn't delete the entire shard.. final int[] blockSize = datasetAttributes.getBlockSize(); final DataType dataType = datasetAttributes.getDataType(); @@ -334,26 +333,4 @@ private DatasetAttributes getNestedShardCodecsAttributes(int[] blockSize) { ); } - private static DatasetAttributes getDatasetAttributes(long[] imageSize, int[] shardSize, int[] blockSize) { - - return new DatasetAttributes( - imageSize, - shardSize, - blockSize, - DataType.INT32, - new ShardingCodec( - blockSize, - new Codec[]{ - // codecs applied to image data - new RawBytes(ByteOrder.BIG_ENDIAN), - }, - new DeterministicSizeCodec[]{ - // codecs applied to the shard index, must not be compressors - new RawBytes(ByteOrder.LITTLE_ENDIAN), - new Crc32cChecksumCodec() - }, - IndexLocation.START - ) - ); - } } diff --git a/src/test/resources/shardExamples/test.n5/attributes.json b/src/test/resources/shardExamples/test.n5/attributes.json deleted file mode 100644 index 573b01888..000000000 --- a/src/test/resources/shardExamples/test.n5/attributes.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "n5": "4.0.0" -} \ No newline at end of file diff --git a/src/test/resources/shardExamples/test.n5/mid_sharded/attributes.json b/src/test/resources/shardExamples/test.n5/mid_sharded/attributes.json deleted file mode 100644 index b9e575b2a..000000000 --- a/src/test/resources/shardExamples/test.n5/mid_sharded/attributes.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "codecs": [ - { - "name": "sharding_indexed", - "configuration": { - "chunk_shape": [ - 2, - 3 - ], - "codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - } - ], - "index_codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - } - ], - "index_location": "end" - } - } - ] -} diff --git a/src/test/resources/shardExamples/test.zarr/mid_sharded/attributes.json b/src/test/resources/shardExamples/test.zarr/mid_sharded/attributes.json deleted file mode 100644 index 920dff928..000000000 --- a/src/test/resources/shardExamples/test.zarr/mid_sharded/attributes.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "shape": [ - 4, - 6 - ], - "fill_value": 0, - "chunk_grid": { - "name": "regular", - "configuration": { - "chunk_shape": [ - 4, - 6 - ] - } - }, - "attributes": {}, - "zarr_format": 3, - "data_type": "uint8", - "chunk_key_encoding": { - "name": "default", - "configuration": { - "separator": "/" - } - }, - "codecs": [ - { - "name": "sharding_indexed", - "configuration": { - "chunk_shape": [ - 2, - 3 - ], - "codecs": [ - { - "name": "n5bytes", - "configuration": { - "endian": "little" - } - } - ], - "index_codecs": [ - { - "name": "bytes", - "configuration": { - "endian": "little" - } - } - ], - "index_location": "end" - } - } - ], - "node_type": "array" -} diff --git a/src/test/resources/shardExamples/test.zarr/mid_sharded/c/0/0 b/src/test/resources/shardExamples/test.zarr/mid_sharded/c/0/0 deleted file mode 100644 index 19ad91e1627a7c0e559a2b3e7fb030cbb62f9934..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 88 rcmb=f1_2&kJ|STdW)@aXE^dASK~XVr1}I>I(mZHKXpCM00M$cpU;qFB diff --git a/src/test/resources/shardExamples/writeReadBlockTest.n5/shard/attributes.json b/src/test/resources/shardExamples/writeReadBlockTest.n5/shard/attributes.json deleted file mode 100644 index 7b444a415..000000000 --- a/src/test/resources/shardExamples/writeReadBlockTest.n5/shard/attributes.json +++ /dev/null @@ -1 +0,0 @@ -{"dimensions":[8,8],"blockSize":[2,2],"shardSize":[4,4],"dataType":"uint8","codecs":[{"name":"sharding_indexed","configuration":{"index_codecs":[{"name":"bytes","configuration":{"endian":"big"}},{"name":"crc32c"}],"codecs":[{"name":"n5bytes","configuration":{"endian":"big"}}],"index_location":"end","chunk_shape":[2,2]}}]} \ No newline at end of file From 9e115342652a14198ce75315e3862295d03d72ac Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Sat, 28 Jun 2025 18:04:51 -0400 Subject: [PATCH 255/423] feat: add Shard.blockExists --- .../org/janelia/saalfeldlab/n5/shard/Shard.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 5a348eaea..74f75c2d5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -101,6 +101,19 @@ default long[] getShardPosition(long... blockPosition) { return shardGridPosition; } + /** + * Tests whether the block at the {@code blockGridPosition} exists. + *

      + * Avoids reading the block data, if possible. + * + * @return true of the block exists + */ + default boolean blockExists(long... blockGridPosition) { + + final int[] relativePosition = getBlockPosition(blockGridPosition); + return getIndex().exists(relativePosition); + } + /** * Retrieve the DataBlock at {@code blockGridPosition} if it exists and is * a member of this Shard. From 77e86e42afaeb39dccf311361137cb9e2311a4c9 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Sat, 28 Jun 2025 18:08:04 -0400 Subject: [PATCH 256/423] feat!: add N5Writer.deleteShard * add tests for sharded and unsharded datasets * change return value behavior for delete methods --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 11 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 94 +++++++- .../org/janelia/saalfeldlab/n5/N5Writer.java | 18 +- .../saalfeldlab/n5/AbstractN5Test.java | 205 ++++++++++++++++-- .../saalfeldlab/n5/N5BlockAsShardTest.java | 9 + .../saalfeldlab/n5/http/N5HttpTest.java | 10 + 6 files changed, 313 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 2ba4dff5f..3417cf8f9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -164,11 +164,11 @@ default String[] list(final String pathName) throws N5Exception { } /** - * Constructs the path for a data block in a dataset at a given grid - * position. + * Constructs the path for a shard or data block in a dataset at a given + * grid position. *
      - * If the gridPosition passed in refers to shard position - * in a sharded dataset, this will return the path to the shard key + * If the gridPosition passed in refers to shard position in a sharded + * dataset, this will return the path to the shard key. *

      * The returned path is * @@ -180,7 +180,8 @@ default String[] list(final String pathName) throws N5Exception { * * @param normalPath * normalized dataset path - * @param gridPosition to the target data block + * @param gridPosition + * to the target data block * @return the absolute path to the data block ad gridPosition */ default String absoluteDataBlockPath( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 7c9e6c13d..26f7b8cdc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -47,6 +48,7 @@ import com.google.gson.JsonObject; import org.janelia.saalfeldlab.n5.shard.InMemoryShard; import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.ShardIndex; import org.janelia.saalfeldlab.n5.util.Position; /** @@ -308,24 +310,94 @@ default boolean remove(final String path) throws N5Exception { return true; } - //TODO: Add deleteShard? + /** + * Delete a shard at the specified position. + * + * @param path + * the dataset path + * @param shardPosition + * the position of the shard to delete + * @return true if the shard existed was successfully deleted. + * @throws N5Exception + * if an error occurs during deletion + */ + default boolean deleteShard( + final String path, + final long... shardPosition) throws N5Exception { + + final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shardPosition); + if (getKeyValueAccess().isFile(shardPath)) { + try { + getKeyValueAccess().delete(shardPath); + return true; + } catch (final Exception e) { + throw new N5Exception("The shard at " + + Arrays.toString(shardPosition) + + " could not be deleted.", e); + } + } + return false; + } @Override default boolean deleteBlock( final String path, final long... gridPosition) throws N5Exception { - final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), gridPosition); - //TODO: how do we want to handle sharded datasets? - // - Delete block from shard? - // - Delete entire shard for block - if (getKeyValueAccess().isFile(blockPath)) - getKeyValueAccess().delete(blockPath); + final String normalPath = N5URI.normalizeGroupPath(path); + final DatasetAttributes datasetAttributes = getDatasetAttributes(normalPath); + + if (datasetAttributes == null) { + return false; // Dataset doesn't exist - return true for consistency + } + if (datasetAttributes.isSharded()) { + // For sharded datasets, we need to: + // 1. Find which shard contains this block + // 2. Read the shard + // 3. Remove the block from the shard + // 4. Write the shard back (or delete if empty) + + final long[] shardPosition = datasetAttributes.getShardPositionForBlock(gridPosition); + final Shard shard = readShard(normalPath, datasetAttributes, shardPosition); + + if (shard == null) + return false; // Shard doesn't exist, so block doesn't exist - + // return false for consistency + + if (!shard.blockExists(gridPosition)) + return false; + + // Convert to InMemoryShard to manipulate blocks + final InMemoryShard inMemoryShard = InMemoryShard.fromShard(shard); + + // Get all blocks except the one to remove + final List> remainingBlocks = new ArrayList<>(); + for (DataBlock block : inMemoryShard.getBlocks()) { + if (!Arrays.equals(block.getGridPosition(), gridPosition)) { + remainingBlocks.add(block); + } + } - //TODO: should we try/catch and return false if exceptions are thrown? + if (remainingBlocks.isEmpty()) { + // If no blocks remain, delete the entire shard + return deleteShard(normalPath, shardPosition); + } else { + // Create new shard with remaining blocks + final InMemoryShard newShard = new InMemoryShard<>(datasetAttributes, shardPosition); + for (DataBlock block : remainingBlocks) { + newShard.addBlock(block); + } - /* an IOException should have occurred if anything had failed midway */ - return true; + // Write the updated shard + writeShard(normalPath, datasetAttributes, newShard); + return true; + } + + } else { + // For non-sharded datasets, deleting the key deletes the block + // and deleteShard deletes the key for gridPosition + return deleteShard(path, gridPosition); + } } -} +} \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 06f79e4cd..c78f7e356 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -285,15 +285,27 @@ void writeShard( * * @param datasetPath dataset path * @param gridPosition position of block to be deleted - * @throws N5Exception the exception - * - * @return {@code true} if the block at {@code gridPosition} was deleted. + * @throws N5Exception if the block exists but could not be deleted * + * @return {@code true} if the block at {@code gridPosition} existed and was deleted. */ boolean deleteBlock( final String datasetPath, final long... gridPosition) throws N5Exception; + /** + * Deletes the shard at {@code gridPosition}. + * + * @param datasetPath dataset path + * @param gridPosition position of shard to be deleted + * @throws N5Exception if the block exists but could not be deleted + * + * @return {@code true} if the shard at {@code gridPosition} existed and was deleted. + */ + boolean deleteShard( + final String datasetPath, + final long... gridPosition) throws N5Exception; + /** * Save a {@link Serializable} as an N5 {@link DataBlock} at a given offset. * The diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 20e990313..260957589 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -57,8 +57,12 @@ import org.janelia.saalfeldlab.n5.N5Reader.Version; import org.janelia.saalfeldlab.n5.shard.InMemoryShard; import org.janelia.saalfeldlab.n5.shard.Shard; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.url.UriAttributeTest; import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; +import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -1173,6 +1177,186 @@ public void testReaderCreation() throws IOException, URISyntaxException { } } + @SuppressWarnings("unchecked") + @Test + public void testShardDelete() { + try (GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)createTempN5Writer()) { + final String datasetName = "testShardDelete"; + + // Create a sharded dataset + final int[] shardSize = new int[]{blockSize[0] * 2, blockSize[1] * 2, blockSize[2] * 2}; + final DatasetAttributes datasetAttributes = new DatasetAttributes( + dimensions, + shardSize, + blockSize, + DataType.UINT8, + new ShardingCodec( + blockSize, + new Codec[]{new N5BlockCodec()}, + new DeterministicSizeCodec[]{new RawBytes()}, + ShardingCodec.IndexLocation.END + ) + ); + writer.createDataset(datasetName, datasetAttributes); + + // Create blocks in different shards + final long[] shardPosition1 = {0, 0, 0}; + final long[] shardPosition2 = {0, 0, 1}; // Different shard in z dimension + final long[] shardPositionNonExistent = {0, 0, 2}; // Different shard in z dimension + + // Create blocks within the first shard + final long[] blockPosition1 = {0, 0, 0}; + final long[] blockPosition2 = {1, 0, 0}; + final long[] blockPosition3 = {0, 1, 0}; + + // Create a block in the second shard + final long[] blockPosition4 = {0, 0, 2}; // This will be in + // shardPosition2 + + final ByteArrayDataBlock block1 = new ByteArrayDataBlock(blockSize, blockPosition1, byteBlock); + final ByteArrayDataBlock block2 = new ByteArrayDataBlock(blockSize, blockPosition2, byteBlock); + final ByteArrayDataBlock block3 = new ByteArrayDataBlock(blockSize, blockPosition3, byteBlock); + final ByteArrayDataBlock block4 = new ByteArrayDataBlock(blockSize, blockPosition4, byteBlock); + + // Write blocks to create shards + writer.writeBlocks(datasetName, datasetAttributes, block1, block2, block3, block4); + + // Verify shards exist + final Shard shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); + final Shard shard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); + final Shard shardDNE = writer.readShard(datasetName, datasetAttributes, shardPositionNonExistent); + assertNotNull("Shard 1 should exist", shard1); + assertNotNull("Shard 2 should exist", shard2); + assertNull("Shard 3 should not exist", shardDNE); + assertEquals("Shard 1 should contain 3 blocks", 3, shard1.getBlocks().size()); + assertEquals("Shard 2 should contain 1 block", 1, shard2.getBlocks().size()); + + // Test deleteShard + boolean deleted1 = writer.deleteShard(datasetName, shardPosition1); + assertTrue("deleteShard should return true when shard exists", deleted1); + + // Verify shard is deleted + final Shard deletedShard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); + assertNull("Shard 1 should be deleted", deletedShard1); + + // Verify blocks in the deleted shard are gone + assertNull("Block 1 should be deleted", writer.readBlock(datasetName, datasetAttributes, blockPosition1)); + assertNull("Block 2 should be deleted", writer.readBlock(datasetName, datasetAttributes, blockPosition2)); + assertNull("Block 3 should be deleted", writer.readBlock(datasetName, datasetAttributes, blockPosition3)); + + // Verify other shard is unaffected + final Shard stillExistingShard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); + assertNotNull("Shard 2 should still exist", stillExistingShard2); + assertNotNull("Block 4 should still exist", writer.readBlock(datasetName, datasetAttributes, blockPosition4)); + + // Test deleting non-existent shard + boolean deletedAgain = writer.deleteShard(datasetName, shardPosition1); + assertFalse("deleteShard should return false when shard doesn't exist", deletedAgain); + + // Test deleting shard at invalid position + final long[] invalidShardPosition = {100, 100, 100}; + boolean deletedInvalid = writer.deleteShard(datasetName, invalidShardPosition); + assertFalse("deleteShard should return false for non-existent shard position", deletedInvalid); + } + } + + @SuppressWarnings("unchecked") + @Test + public void testShardedBlockDelete() { + try (GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)createTempN5Writer()) { + final String datasetName = "testShardBlockDelete"; + + // Create a sharded dataset + final int[] shardSize = new int[]{blockSize[0] * 2, blockSize[1] * 2, blockSize[2] * 2}; + final DatasetAttributes datasetAttributes = new DatasetAttributes( + dimensions, + shardSize, + blockSize, + DataType.UINT8, + new ShardingCodec( + blockSize, + new Codec[]{new N5BlockCodec()}, + new DeterministicSizeCodec[]{new RawBytes()}, + ShardingCodec.IndexLocation.END + ) + ); + writer.createDataset(datasetName, datasetAttributes); + + // Create blocks in different shards + final long[] shardPosition1 = {0, 0, 0}; + final long[] shardPosition2 = {0, 0, 1}; // Different shard in z + // dimension + final long[] shardPositionNonExistent = {0, 0, 2}; // Different + // shard in z + // dimension + + // Create blocks within the first shard + final long[] blockPosition1 = {0, 0, 0}; + final long[] blockPosition2 = {1, 0, 0}; + final long[] blockPosition3 = {0, 1, 0}; + + // Create a block in the second shard + final long[] blockPosition4 = {0, 0, 2}; // This will be in + // shardPosition2 + + final ByteArrayDataBlock block1 = new ByteArrayDataBlock(blockSize, blockPosition1, byteBlock); + final ByteArrayDataBlock block2 = new ByteArrayDataBlock(blockSize, blockPosition2, byteBlock); + final ByteArrayDataBlock block3 = new ByteArrayDataBlock(blockSize, blockPosition3, byteBlock); + final ByteArrayDataBlock block4 = new ByteArrayDataBlock(blockSize, blockPosition4, byteBlock); + + // Write blocks to create shards + writer.writeBlocks(datasetName, datasetAttributes, block1, block2, block3, block4); + + // Verify shards exist + Shard shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); + Shard shard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); + Shard shardDNE = writer.readShard(datasetName, datasetAttributes, shardPositionNonExistent); + assertNotNull("Shard 1 should exist", shard1); + assertNotNull("Shard 2 should exist", shard2); + assertNull("Shard 3 should not exist", shardDNE); + assertEquals("Shard 1 should contain 3 blocks", 3, shard1.getBlocks().size()); + assertEquals("Shard 2 should contain 1 block", 1, shard2.getBlocks().size()); + + // Test delete one block from a multi-block shard + boolean deleted1 = writer.deleteBlock(datasetName, blockPosition1); + assertTrue("deleteBlock1", deleted1); + shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); + assertNotNull("Shard 1 should still exist", shard1); + DataBlock blk1Read = writer.readBlock(datasetName, datasetAttributes, blockPosition1); + DataBlock blk2Read = writer.readBlock(datasetName, datasetAttributes, blockPosition2); + DataBlock blk3Read = writer.readBlock(datasetName, datasetAttributes, blockPosition3); + assertNull("Block 1 should not exist", blk1Read); + assertNotNull("Block 2 should exist", blk2Read); + assertNotNull("Block 3 should exist", blk3Read); + + // Test delete one block from a multi-block shard + boolean deleted2 = writer.deleteBlock(datasetName, blockPosition2); + assertTrue("deleteBlock2", deleted2); + shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); + assertNotNull("Shard 1 should still exist", shard1); + blk2Read = writer.readBlock(datasetName, datasetAttributes, blockPosition2); + blk3Read = writer.readBlock(datasetName, datasetAttributes, blockPosition3); + assertNull("Block 2 should not exist", blk2Read); + assertNotNull("Block 3 should exist", blk3Read); + + // Test delete last block from a multi-block shard + boolean deleted3 = writer.deleteBlock(datasetName, blockPosition3); + assertTrue("deleteBlock3", deleted3); + shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); + assertNull("Shard 1 should not exist", shard1); + blk3Read = writer.readBlock(datasetName, datasetAttributes, blockPosition3); + assertNull("Block 3 should not exist", blk3Read); + + // Test delete last block from a multi-block shard + boolean deleted4 = writer.deleteBlock(datasetName, blockPosition4); + assertTrue("deleteBlock4", deleted4); + shard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); + assertNull("Shard 2 should not exist", shard2); + DataBlock blk4Read = writer.readBlock(datasetName, datasetAttributes, blockPosition4); + assertNull("Block 2 should not exist", blk4Read); + } + } + @Test public void testDelete() { @@ -1184,8 +1368,7 @@ public void testDelete() { final long[] position2 = {0, 1, 2}; // no blocks should exist to begin with - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1))); - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); + assertNull(n5.readBlock(datasetName, attributes, position1)); final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, position1, byteBlock); n5.writeBlock(datasetName, attributes, dataBlock); @@ -1195,24 +1378,17 @@ public void testDelete() { assertNotNull(readBlock); assertTrue(readBlock instanceof ByteArrayDataBlock); assertArrayEquals(byteBlock, ((ByteArrayDataBlock)readBlock).getData()); - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); - // deletion should report true in all cases - assertTrue(n5.deleteBlock(datasetName, position1)); - assertTrue(n5.deleteBlock(datasetName, position1)); - assertTrue(n5.deleteBlock(datasetName, position2)); + assertTrue("deleting existing block should return true", n5.deleteBlock(datasetName, position1)); + assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position1)); + assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position2)); // no block should exist anymore - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1))); - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); + assertNull(n5.readBlock(datasetName, attributes, position1)); + assertNull(n5.readBlock(datasetName, attributes, position2)); } } - protected boolean testDeleteIsBlockDeleted(final DataBlock dataBlock) { - - return dataBlock == null; - } - public static class TestData { public String groupPath; @@ -1693,7 +1869,6 @@ public void testWriteReadShardOnUnshardedDataset() { assertArrayEquals("block written through shard should be identical", (long[])readBlock.getData(), (long[])block0.getData()); } - } protected void assertDatasetAttributesEquals(final DatasetAttributes expected, final DatasetAttributes actual) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java index 04022cb89..db3fc2b06 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java @@ -4,6 +4,7 @@ import org.janelia.saalfeldlab.n5.shard.InMemoryShard; import org.janelia.saalfeldlab.n5.shard.Shard; import org.janelia.saalfeldlab.n5.shard.ShardIndex; +import org.junit.Ignore; public class N5BlockAsShardTest extends N5FSTest { @@ -50,4 +51,12 @@ public DataBlock readBlock(String pathName, DatasetAttributes datasetAttr } }; } + + @Override + @Ignore("irrelevant when creating a sharded dataset") + public void testShardDelete() { } + + @Override + @Ignore("irrelevant when creating a sharded dataset") + public void testShardedBlockDelete() { } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/N5HttpTest.java b/src/test/java/org/janelia/saalfeldlab/n5/http/N5HttpTest.java index 672e3e0b6..a564f5425 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/N5HttpTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/N5HttpTest.java @@ -160,6 +160,16 @@ public void testVersion() throws NumberFormatException { } + @Ignore("N5Writer not supported for HTTP") + @Override public void testShardDelete() { + + } + + @Ignore("N5Writer not supported for HTTP") + @Override public void testShardedBlockDelete() { + + } + @Ignore("N5Writer not supported for HTTP") @Override public void testWriterSeparation() { From e1468f194b5d0da17ded1fa452b574f76bf96440 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 30 Jun 2025 16:10:28 -0400 Subject: [PATCH 257/423] fix/test: fix N5Codecs.getSize * split BytesTests to ArrayCodecTests and BytesCodecTests * better test for CodecSerialization --- .../saalfeldlab/n5/codec/N5Codecs.java | 9 +- .../saalfeldlab/n5/codec/ArrayCodecTests.java | 272 ++++++++++++++++++ .../{CodecTests.java => BytesCodecTests.java} | 106 ++++++- .../saalfeldlab/n5/codec/BytesTests.java | 57 ---- .../n5/serialization/CodecSerialization.java | 18 +- 5 files changed, 397 insertions(+), 65 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/ArrayCodecTests.java rename src/test/java/org/janelia/saalfeldlab/n5/codec/{CodecTests.java => BytesCodecTests.java} (50%) delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java index 021ab05ff..41d3011f8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java @@ -290,13 +290,16 @@ static class BlockHeader { public int getSize() { + // two bytes for mode, two bytes for num dimensions + // four bytes per dimension. + // four more bytes for varlength switch (mode) { case MODE_DEFAULT: - return 2 + 4 * blockSize.length; + return 4 + 4 * blockSize.length; case MODE_VARLENGTH: - return 2 + 4 * blockSize.length + 4; + return 4 + 4 * blockSize.length + 4; case MODE_OBJECT: - return 2 + 4; + return 4 + 4; default: throw new IllegalArgumentException("Unexpected mode: " + mode); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/ArrayCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/ArrayCodecTests.java new file mode 100644 index 000000000..52ebdffa0 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/ArrayCodecTests.java @@ -0,0 +1,272 @@ +package org.janelia.saalfeldlab.n5.codec; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.Random; + +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock; +import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; +import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.saalfeldlab.n5.IntArrayDataBlock; +import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; +import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; +import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.junit.Test; + +public class ArrayCodecTests { + + static Random random = new Random(12345); + + final int[] blockSize = {11, 7, 5}; + private final BitShiftBytesCodec shiftCodec = new BitShiftBytesCodec(3); + private final GzipCompression compressor = new GzipCompression(); + private final BytesCodec[][] bytesCodecs = new BytesCodec[][]{ + {}, // empty: "raw" compression + {compressor}, + {shiftCodec}, + {shiftCodec, compressor} + }; + + private final DataType[] dataTypes = { + DataType.INT8, DataType.UINT8, + DataType.INT16, DataType.UINT16, + DataType.INT32, DataType.UINT32, + DataType.INT64, DataType.UINT64, + DataType.FLOAT32, DataType.FLOAT64 + }; + + @Test + public void testN5BlockCodec() throws Exception { + for (DataType dataType : dataTypes) { + for (BytesCodec[] codecs : bytesCodecs) { + + final DatasetAttributes attributes = new DatasetAttributes( + new long[]{32, 32, 32}, + blockSize, + dataType); + + final N5BlockCodec codec = new N5BlockCodec(); + codec.initialize(attributes, codecs); + testArrayCodecHelper(codec, attributes); + } + } + } + + @Test + public void testRawBytesCodec() throws Exception { + // Test RawBytes codec with different byte orders and DataTypes + final ByteOrder[] byteOrders = {ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}; + for (DataType dataType : dataTypes) { + for (ByteOrder byteOrder : byteOrders) { + for (BytesCodec[] codecs : bytesCodecs) { + + final DatasetAttributes attributes = new DatasetAttributes( + new long[]{32, 32, 32}, + blockSize, + dataType); + + final RawBytes codec = new RawBytes(byteOrder); + codec.initialize(attributes, codecs); + testArrayCodecHelper(codec, attributes); + } + } + } + } + + private void testArrayCodecHelper(ArrayCodec codec, DatasetAttributes attributes) throws Exception { + final int[] blockSize = attributes.getBlockSize(); + final DataType dataType = attributes.getDataType(); + final long[] gridPosition = {3, 2, 1}; + + // Create appropriate data block based on type + DataBlock originalBlock = createRandomDataBlock(dataType, blockSize, gridPosition); + + // Test encode/decode roundtrip + final ReadData encoded = codec.encode(originalBlock); + assertNotNull(encoded); + + final DataBlock decoded = codec.decode(encoded, gridPosition); + assertNotNull(decoded); + + assertArrayEquals("Block size should match", blockSize, decoded.getSize()); + assertArrayEquals("Grid position should match", gridPosition, decoded.getGridPosition()); + assertDataEquals(originalBlock, decoded); + verifyCompatibleDataType(dataType, decoded); + } + + @Test + public void testEmptyBlock() throws Exception { + // Test handling of empty blocks + final int[] blockSize = {0, 0}; + final long[] gridPosition = {0, 0}; + final DatasetAttributes attributes = new DatasetAttributes( + new long[]{64, 64}, + new int[]{8, 8}, + DataType.UINT8); + + final N5BlockCodec codec = new N5BlockCodec(); + codec.initialize(attributes, new Codec.BytesCodec[0]); + + // Test encode/decode + final ByteArrayDataBlock emptyBlock = new ByteArrayDataBlock(blockSize, gridPosition, new byte[0]); + final ReadData encoded = codec.encode(emptyBlock); + final DataBlock decoded = codec.decode(encoded, gridPosition); + + assertEquals("Empty block should have 0 elements", 0, decoded.getNumElements()); + } + + @Test + public void testEncodedSizeCalculation() throws Exception { + // Test that encoded size calculations are correct + final int[] blockSize = {64, 64}; + final DatasetAttributes attributes = new DatasetAttributes( + new long[]{512, 512}, + blockSize, + DataType.INT16); + + final N5BlockCodec n5Codec = new N5BlockCodec(); + n5Codec.initialize(attributes, new Codec.BytesCodec[0]); + + final RawBytes rawCodec = new RawBytes(); + rawCodec.initialize(attributes, new Codec.BytesCodec[0]); + + // Calculate expected sizes + final long rawDataSize = blockSize[0] * blockSize[1] * 2; // INT16 has 2 bytes per element + + // N5BlockCodec adds a header + // the estimate of the encoded size + final long n5EncodedSize = n5Codec.encodedSize(rawDataSize); + assertTrue("N5 encoded size should be larger than raw size", n5EncodedSize > rawDataSize); + + DataBlock dataBlock = createRandomDataBlock(attributes.getDataType(), blockSize, new long[] {0,0}); + ReadData n5EncodedDataBlock = n5Codec.encode(dataBlock); + assertEquals("N5 actual encoded size should equal estimated size", n5EncodedSize, n5EncodedDataBlock.length()); + + // RawBytes should not change size + final long rawEncodedSize = rawCodec.encodedSize(rawDataSize); + assertEquals("Raw encoded size should equal input size", rawDataSize, rawEncodedSize); + + ReadData rawEncodedDataBlock = rawCodec.encode(dataBlock); + assertEquals("Raw actual encoded size should equal estimated size", rawEncodedSize, rawEncodedDataBlock.length()); + } + + private static DataBlock createRandomDataBlock(DataType dataType, int[] blockSize, long[] gridPosition) { + final int numElements = Arrays.stream(blockSize).reduce(1, (a, b) -> a * b); + + switch (dataType) { + case INT8: + case UINT8: + byte[] uint8Data = new byte[numElements]; + for (int i = 0; i < numElements; i++) { + uint8Data[i] = (byte) random.nextInt(256); + } + return new ByteArrayDataBlock(blockSize, gridPosition, uint8Data); + + case INT16: + case UINT16: + short[] uint16Data = new short[numElements]; + for (int i = 0; i < numElements; i++) { + uint16Data[i] = (short) random.nextInt(65536); + } + return new ShortArrayDataBlock(blockSize, gridPosition, uint16Data); + + case INT32: + case UINT32: + int[] uint32Data = new int[numElements]; + for (int i = 0; i < numElements; i++) { + uint32Data[i] = random.nextInt(); + } + return new IntArrayDataBlock(blockSize, gridPosition, uint32Data); + + case INT64: + case UINT64: + long[] uint64Data = new long[numElements]; + for (int i = 0; i < numElements; i++) { + uint64Data[i] = random.nextLong(); + } + return new LongArrayDataBlock(blockSize, gridPosition, uint64Data); + + case FLOAT32: + float[] floatData = new float[numElements]; + for (int i = 0; i < numElements; i++) { + floatData[i] = random.nextFloat(); + } + return new FloatArrayDataBlock(blockSize, gridPosition, floatData); + + case FLOAT64: + double[] doubleData = new double[numElements]; + for (int i = 0; i < numElements; i++) { + doubleData[i] = random.nextDouble(); + } + return new DoubleArrayDataBlock(blockSize, gridPosition, doubleData); + + default: + throw new IllegalArgumentException("Unsupported data type: " + dataType); + } + } + + private static void verifyCompatibleDataType(DataType expectedType, DataBlock block) { + + Object data = block.getData(); + switch (expectedType) { + case INT8: + case UINT8: + assertTrue("Expected byte array for " + expectedType, data instanceof byte[]); + break; + case INT16: + case UINT16: + assertTrue("Expected short array for " + expectedType, data instanceof short[]); + break; + case INT32: + case UINT32: + assertTrue("Expected int array for " + expectedType, data instanceof int[]); + break; + case INT64: + case UINT64: + assertTrue("Expected long array for " + expectedType, data instanceof long[]); + break; + case FLOAT32: + assertTrue("Expected float array for " + expectedType, data instanceof float[]); + break; + case FLOAT64: + assertTrue("Expected double array for " + expectedType, data instanceof double[]); + break; + default: + throw new IllegalArgumentException("Unsupported data type: " + expectedType); + } + } + + private static void assertDataEquals(DataBlock expected, DataBlock actual) { + + Object expectedData = expected.getData(); + Object actualData = actual.getData(); + + if (expectedData instanceof byte[]) { + assertArrayEquals((byte[]) expectedData, (byte[]) actualData); + } else if (expectedData instanceof short[]) { + assertArrayEquals((short[]) expectedData, (short[]) actualData); + } else if (expectedData instanceof int[]) { + assertArrayEquals((int[]) expectedData, (int[]) actualData); + } else if (expectedData instanceof long[]) { + assertArrayEquals((long[]) expectedData, (long[]) actualData); + } else if (expectedData instanceof float[]) { + assertArrayEquals((float[]) expectedData, (float[]) actualData, 0.0f); + } else if (expectedData instanceof double[]) { + assertArrayEquals((double[]) expectedData, (double[]) actualData, 0.0); + } else { + throw new IllegalArgumentException("Unknown data type"); + } + } +} \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/CodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java similarity index 50% rename from src/test/java/org/janelia/saalfeldlab/n5/codec/CodecTests.java rename to src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java index 804fb29ea..f6195af06 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/CodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java @@ -1,6 +1,7 @@ package org.janelia.saalfeldlab.n5.codec; import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; import java.io.IOException; import java.io.InputStream; @@ -12,11 +13,11 @@ import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.ReadData.OutputStreamOperator; -import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; import org.junit.BeforeClass; import org.junit.Test; -public class CodecTests { +public class BytesCodecTests { static Random random; @@ -25,6 +26,25 @@ public static void setup() { random = new Random(7777); } + @Test + public void testEncodeDecodeBytes() { + + // Create a BitShiftBytesCodec with shift value + final BitShiftBytesCodec originalCodec = new BitShiftBytesCodec(3); + + // Test encode/decode roundtrip + final byte[] testData = new byte[12]; + random.nextBytes(testData); + + final ReadData original = ReadData.from(testData); + final ReadData encoded = originalCodec.encode(original); + final ReadData decoded = originalCodec.decode(encoded); + + final byte[] result = decoded.allBytes(); + assertEquals("Length should match", testData.length, result.length); + assertArrayEquals("encoded-decoded bytes should match original", testData, result); + } + @Test public void concatenatedBytesCodecTest() throws IOException { @@ -95,4 +115,86 @@ public void write(int b) throws IOException { } } + @NameConfig.Name(BitShiftBytesCodec.TYPE) + public static class BitShiftBytesCodec implements Codec.BytesCodec { + + private static final String TYPE = "bitshift"; + + @NameConfig.Parameter + private int shift; + + public BitShiftBytesCodec() { + + shift = 0; + } + + public BitShiftBytesCodec(int shift) { + + this.shift = shift; + } + + @Override + public String getType() { + + return TYPE; + } + + @Override + public ReadData decode(ReadData readData) throws N5IOException { + + if (shift == 0) { + return readData; + } + + final byte[] data = readData.allBytes(); + final byte[] decoded = new byte[data.length]; + + // Apply inverse bit shift (right rotate) to decode + for (int i = 0; i < data.length; i++) { + int b = data[i] & 0xFF; + decoded[i] = (byte)((b >>> shift) | (b << (8 - shift))); + } + + return ReadData.from(decoded); + } + + @Override + public ReadData encode(ReadData readData) throws N5IOException { + + if (shift == 0) { + return readData; + } + + byte[] data = readData.allBytes(); + byte[] encoded = new byte[data.length]; + + // Apply bit shift (left rotate) to encode + for (int i = 0; i < data.length; i++) { + int b = data[i] & 0xFF; + encoded[i] = (byte)((b << shift) | (b >>> (8 - shift))); + } + return ReadData.from(encoded); + } + + @Override + public boolean equals(Object obj) { + + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + BitShiftBytesCodec other = (BitShiftBytesCodec)obj; + return shift == other.shift; + } + + @Override + public int hashCode() { + + return Integer.hashCode(shift); + } + + } + } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java deleted file mode 100644 index 6c7df18ce..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesTests.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.nio.ByteOrder; - -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.NameConfigAdapter; -import org.janelia.saalfeldlab.n5.RawCompression; -import org.janelia.saalfeldlab.n5.universe.N5Factory; -import org.junit.Test; - -import com.google.gson.GsonBuilder; - -public class BytesTests { - - @Test - public void testSerialization() { - - //TODO: we don't intend to support codec serialization for N5. - // Rewrite this to directly test NameConfigAdapter bit Gson - fail("Rewrite this to directly test NameConfigAdapter with Gson"); - - final N5Factory factory = new N5Factory(); - factory.cacheAttributes(false); - final GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); - gsonBuilder.registerTypeAdapter(ByteOrder.class, RawBytes.byteOrderAdapter); - factory.gsonBuilder(gsonBuilder); - - final N5Writer reader = factory.openWriter("n5:src/test/resources/shardExamples/test.n5"); - final Codec bytes = reader.getAttribute("mid_sharded", "codecs[0]/configuration/codecs[0]", Codec.class); - assertTrue("as RawBytes", bytes instanceof RawBytes); - - final N5Writer writer = factory.openWriter("n5:src/test/resources/shardExamples/test.n5"); - - final DatasetAttributes datasetAttributes = new DatasetAttributes( - new long[]{8, 8}, - new int[]{4, 4}, - DataType.UINT8, - new N5BlockCodec(), - new IdentityCodec() - ); - writer.createGroup("shard"); //Should already exist, but this will ensure. - writer.setAttribute("shard", "/", datasetAttributes); - final DatasetAttributes deserialized = writer.getAttribute("shard", "/", DatasetAttributes.class); - - assertEquals("1 codecs", 1, deserialized.getCodecs().length); - assertTrue("Identity", deserialized.getCodecs()[0] instanceof IdentityCodec); - assertTrue("Bytes", deserialized.getArrayCodec() instanceof N5BlockCodec); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java index 8b21fd1ac..8137ab2c2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java @@ -5,6 +5,8 @@ import org.janelia.saalfeldlab.n5.GsonUtils; import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.IdentityCodec; import org.junit.Before; @@ -23,19 +25,29 @@ public class CodecSerialization { @Before public void before() { - final GsonBuilder gsonBuilder = new GsonBuilder(); GsonUtils.registerGson(gsonBuilder); gson = gsonBuilder.create(); } @Test - public void testSerializeIdentity() { + public void testCodecSerialization() { final IdentityCodec id = new IdentityCodec(); final JsonObject jsonId = gson.toJsonTree(id).getAsJsonObject(); final JsonElement expected = gson.fromJson("{\"name\":\"id\"}", JsonElement.class); - assertEquals("identity", expected, jsonId.getAsJsonObject()); + assertEquals("identity json", expected, jsonId.getAsJsonObject()); + + final BitShiftBytesCodec codec = new BitShiftBytesCodec(3); + final JsonObject bitShiftJson = gson.toJsonTree(codec).getAsJsonObject(); + final JsonElement expectedBitShift = gson.fromJson( + "{\"name\":\"bitshift\",\"configuration\":{\"shift\":3}}", + JsonElement.class); + assertEquals("bitshift json", expectedBitShift, bitShiftJson); + + final BytesCodec deserializedCodec = gson.fromJson(bitShiftJson, Codec.BytesCodec.class); + // Verify deserialized codec + assertEquals("Deserialized codec should equal original", codec, deserializedCodec); } @Test From 936091defc91ecbd7201192383e3b0e3f56665e2 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 1 Jul 2025 13:59:36 -0400 Subject: [PATCH 258/423] refactor/test: remove ShardProperties * move methods to DatasetAttributes * rm ShardPropertiesTests, add DatasetAttributesTest --- .../saalfeldlab/n5/DatasetAttributes.java | 198 ++++++++++++++- .../janelia/saalfeldlab/n5/shard/Shard.java | 11 - .../saalfeldlab/n5/shard/ShardParameters.java | 227 ----------------- .../saalfeldlab/n5/DatasetAttributesTest.java | 232 ++++++++++++++++++ .../n5/shard/ShardPropertiesTests.java | 130 ---------- 5 files changed, 422 insertions(+), 376 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index d9312b1af..496167d08 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -30,8 +30,13 @@ import java.io.Serializable; import java.lang.reflect.Type; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.IntStream; import java.util.stream.Stream; import org.janelia.saalfeldlab.n5.codec.Codec; @@ -39,7 +44,6 @@ import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.shard.BlockAsShardCodec; -import org.janelia.saalfeldlab.n5.shard.ShardParameters; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; @@ -49,6 +53,7 @@ import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; +import org.janelia.saalfeldlab.n5.util.Position; /** * Mandatory dataset attributes: @@ -62,8 +67,7 @@ * * @author Stephan Saalfeld */ -//TODO : inline ShardParameters and delete interface -public class DatasetAttributes implements ShardParameters, Serializable { +public class DatasetAttributes implements Serializable { private static final long serialVersionUID = -4521467080388947553L; @@ -82,11 +86,16 @@ public class DatasetAttributes implements ShardParameters, Serializable { protected static final String compressionTypeKey = "compressionType"; private final long[] dimensions; + + // number of samples per block per dimension private final int[] blockSize; + + // number of samples per shard per dimension + private final int[] shardSize; + private final DataType dataType; private final ArrayCodec arrayCodec; private final BytesCodec[] byteCodecs; - private final int[] shardSize; /** * Constructs a DatasetAttributes instance with specified dimensions, block size, data type, @@ -104,6 +113,8 @@ public DatasetAttributes( final DataType dataType, final Codec... codecs) { + validateBlockShardSizes(dimensions, shardSize, blockSize); + this.dimensions = dimensions; this.shardSize = shardSize; this.blockSize = blockSize; @@ -135,6 +146,33 @@ public DatasetAttributes( arrayCodec.initialize(this, byteCodecs); } + private void validateBlockShardSizes(long[] dimensions, int[] shardSize, int[] blockSize) { + + final int nd = dimensions.length; + + if (blockSize.length != nd) + throw new IllegalArgumentException(String.format("Number of block dimensions (%d) must equal number of dimensions (%d).", + blockSize.length, nd)); + + if (shardSize.length != nd) + throw new IllegalArgumentException(String.format("Number of shard dimensions (%d) must equal number of dimensions (%d).", + shardSize.length, nd)); + + for (int i = 0; i < blockSize.length; i++) { + + if (blockSize[i] <= 0) + throw new IllegalArgumentException(String.format("Block size in dimension %d (%d) is <= 0", + i, blockSize[i])); + + if (shardSize[i] < blockSize[i]) + throw new IllegalArgumentException(String.format("Shard size in dimension %d (%d) is larger than the block size (%d)", + i, shardSize[i], blockSize[i])); + else if (shardSize[i] % blockSize[i] != 0) + throw new IllegalArgumentException(String.format("Shard size in dimension %d (%d) not a multiple of the block size (%d)", + i, shardSize[i], blockSize[i])); + } + } + protected Codec.ArrayCodec defaultArrayCodec() { return new N5BlockCodec(); } @@ -156,30 +194,174 @@ public DatasetAttributes( this( dimensions, blockSize, blockSize, dataType, codecs ); } - @Override public long[] getDimensions() { return dimensions; } - @Override public int getNumDimensions() { return dimensions.length; } - @Override public int[] getShardSize() { return shardSize; } - @Override public int[] getBlockSize() { return blockSize; } + /** + * Returns the number of blocks per dimension for a shard. + * + * @return the size of the block grid of a shard + */ + public int[] getBlocksPerShard() { + + final int[] shardSize = getShardSize(); + final int nd = getNumDimensions(); + final int[] blocksPerShard = new int[nd]; + final int[] blockSize = getBlockSize(); + for (int i = 0; i < nd; i++) + blocksPerShard[i] = shardSize[i] / blockSize[i]; + + return blocksPerShard; + } + + /** + * Returns the number of blocks per dimension that tile the image. + * + * @return blocks per image + */ + public long[] blocksPerImage() { + return IntStream.range(0, getNumDimensions()) + .mapToLong(i -> (long) Math.ceil((double)getDimensions()[i] / getBlockSize()[i])) + .toArray(); + } + + /** + * Returns the number of shards per dimension that tile the image. + * + * @return shards per image + */ + public long[] shardsPerImage() { + return IntStream.range(0, getNumDimensions()) + .mapToLong(i -> (long)Math.ceil((double)getDimensions()[i] / getShardSize()[i])) + .toArray(); + } + + /** + * @return the number of blocks per shard + */ + public long getNumBlocksPerShard() { + + return Arrays.stream(getBlocksPerShard()).reduce(1, (x, y) -> x * y); + } + + /** + * Given a block's position relative to the array, returns the position of the shard containing that block relative to the shard grid. + * + * @param blockGridPosition + * position of a block relative to the array + * @return the position of the containing shard in the shard grid + */ + public long[] getShardPositionForBlock(final long... blockGridPosition) { + + final int[] blocksPerShard = getBlocksPerShard(); + final long[] shardGridPosition = new long[blockGridPosition.length]; + for (int i = 0; i < shardGridPosition.length; i++) { + shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / blocksPerShard[i]); + } + + return shardGridPosition; + } + + /** + * Returns the block at the given position relative to this shard, or null if this shard does not contain the given block. + * + * @return the block position + */ + public int[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { + + final long[] shardPos = getShardPositionForBlock(blockPosition); + if (!Arrays.equals(shardPosition, shardPos)) + return null; + + final int[] shardSize = getBlocksPerShard(); + final int[] blockShardPos = new int[shardSize.length]; + for (int i = 0; i < shardSize.length; i++) { + blockShardPos[i] = (int)(blockPosition[i] % shardSize[i]); + } + return blockShardPos; + } + + /** + * Given a block's position relative to a shard, returns its position relative + * to the image. + * + * @param shardPosition shard position in the shard grid + * @param blockPosition block position relative to the shard + * @return the block position in the block grid + */ + public long[] getBlockPositionFromShardPosition(final long[] shardPosition, final long[] blockPosition) { + + final int[] shardBlockSize = getBlocksPerShard(); + final long[] blockImagePos = new long[getNumDimensions()]; + for (int i = 0; i < getNumDimensions(); i++) { + blockImagePos[i] = (shardPosition[i] * shardBlockSize[i]) + (blockPosition[i]); + } + + return blockImagePos; + } + + /** + * Returns the number of shards per dimension for the dataset. + * + * @return the size of the shard grid of a dataset + */ + public int[] getShardBlockGridSize() { + + final int nd = getNumDimensions(); + final int[] shardBlockGridSize = new int[nd]; + final int[] blockSize = getBlockSize(); + for (int i = 0; i < nd; i++) + shardBlockGridSize[i] = (int)(Math.ceil((double)getDimensions()[i] / blockSize[i])); + + return shardBlockGridSize; + } + + public Map> groupBlockPositions(final List blockPositions) { + + final TreeMap> map = new TreeMap<>(); + for( final long[] blockPos : blockPositions ) { + Position shardPos = Position.wrap(getShardPositionForBlock(blockPos)); + if( !map.containsKey(shardPos)) { + map.put(shardPos, new ArrayList<>()); + } + map.get(shardPos).add(blockPos); + } + + return map; + } + + public Map>> groupBlocks(final List> blocks) { + + // figure out how to re-use groupBlockPositions here? + final TreeMap>> map = new TreeMap<>(); + for (final DataBlock block : blocks) { + Position shardPos = Position.wrap(getShardPositionForBlock(block.getGridPosition())); + if (!map.containsKey(shardPos)) { + map.put(shardPos, new ArrayList<>()); + } + map.get(shardPos).add(block); + } + + return map; + } + public boolean isSharded() { return getArrayCodec() instanceof ShardingCodec; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 74f75c2d5..1158fbedf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -142,17 +142,6 @@ default List> getBlocks() { return blocks; } - /** - * Returns an {@link Iterator} over block positions contained in this shard. - * - * @return - */ - default Iterator blockPositionIterator() { - - final int nd = getSize().length; - long[] min = getDatasetAttributes().getBlockPositionFromShardPosition( getGridPosition(), new long[nd]); - return new GridIterator(GridIterator.int2long(getBlockGridSize()), min); - } /** * @return the ShardIndex for this shard, or a new ShardIndex if the Shard is non-existent diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java deleted file mode 100644 index a6ff3d25d..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardParameters.java +++ /dev/null @@ -1,227 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.TreeMap; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.util.GridIterator; -import org.janelia.saalfeldlab.n5.util.Position; - -@Deprecated -public interface ShardParameters { - - long[] getDimensions(); - - int getNumDimensions(); - - int[] getBlockSize(); - - - /** - * The size of the blocks in pixel units. - * - * @return the number of pixels per dimension for this shard. - */ - int[] getShardSize(); - - - /** - * Returns the number of blocks per dimension for a shard. - * - * @return the size of the block grid of a shard - */ - default int[] getBlocksPerShard() { - - final int[] shardSize = getShardSize(); - final int nd = getNumDimensions(); - final int[] blocksPerShard = new int[nd]; - final int[] blockSize = getBlockSize(); - for (int i = 0; i < nd; i++) - blocksPerShard[i] = shardSize[i] / blockSize[i]; - - return blocksPerShard; - } - - /** - * Returns the number of blocks per dimension that tile the image. - * - * @return blocks per image - */ - default long[] blocksPerImage() { - return IntStream.range(0, getNumDimensions()) - .mapToLong(i -> (long) Math.ceil((double)getDimensions()[i] / getBlockSize()[i])) - .toArray(); - } - - /** - * Returns the number of shards per dimension that tile the image. - * - * @return shards per image - */ - default long[] shardsPerImage() { - return IntStream.range(0, getNumDimensions()) - .mapToLong(i -> (long)Math.ceil((double)getDimensions()[i] / getShardSize()[i])) - .toArray(); - } - - /** - * Given a block's position relative to the array, returns the position of the shard containing that block relative to the shard grid. - * - * @param blockGridPosition - * position of a block relative to the array - * @return the position of the containing shard in the shard grid - */ - default long[] getShardPositionForBlock(final long... blockGridPosition) { - - final int[] blocksPerShard = getBlocksPerShard(); - final long[] shardGridPosition = new long[blockGridPosition.length]; - for (int i = 0; i < shardGridPosition.length; i++) { - shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / blocksPerShard[i]); - } - - return shardGridPosition; - } - - /** - * Returns the number of shards per dimension for the dataset. - * - * @return the size of the shard grid of a dataset - */ - default int[] getShardBlockGridSize() { - - final int nd = getNumDimensions(); - final int[] shardBlockGridSize = new int[nd]; - final int[] blockSize = getBlockSize(); - for (int i = 0; i < nd; i++) - shardBlockGridSize[i] = (int)(Math.ceil((double)getDimensions()[i] / blockSize[i])); - - return shardBlockGridSize; - } - - /** - * Returns the block at the given position relative to this shard, or null if this shard does not contain the given block. - * - * @return the block position - */ - default int[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { - - // TODO check correctness - final long[] shardPos = getShardPositionForBlock(blockPosition); - if (!Arrays.equals(shardPosition, shardPos)) - return null; - - final int[] shardSize = getBlocksPerShard(); - final int[] blockShardPos = new int[shardSize.length]; - for (int i = 0; i < shardSize.length; i++) { - blockShardPos[i] = (int)(blockPosition[i] % shardSize[i]); - } - - return blockShardPos; - } - - /** - * Given a block's position relative to a shard, returns its position in pixels - * relative to the image. - * - * @param shardPosition shard position in the shard grid - * @param blockPosition block position the - * @return the block's min pixel coordinate - */ - default long[] getBlockMinFromShardPosition(final long[] shardPosition, final long[] blockPosition) { - - // is this useful? - final int[] blockSize = getBlockSize(); - final int[] shardSize = getShardSize(); - Objects.requireNonNull(shardSize, "getShardSize() must not be null"); - final long[] blockImagePos = new long[shardSize.length]; - for (int i = 0; i < shardSize.length; i++) { - blockImagePos[i] = (shardPosition[i] * shardSize[i]) + (blockPosition[i] * blockSize[i]); - } - - return blockImagePos; - } - - /** - * Given a block's position relative to a shard, returns its position relative - * to the image. - * - * @param shardPosition shard position in the shard grid - * @param blockPosition block position relative to the shard - * @return the block position in the block grid - */ - default long[] getBlockPositionFromShardPosition(final long[] shardPosition, final long[] blockPosition) { - - // is this useful? - final int[] shardBlockSize = getBlocksPerShard(); - final long[] blockImagePos = new long[getNumDimensions()]; - for (int i = 0; i < getNumDimensions(); i++) { - blockImagePos[i] = (shardPosition[i] * shardBlockSize[i]) + (blockPosition[i]); - } - - return blockImagePos; - } - - default Map> groupBlockPositions(final List blockPositions) { - - final TreeMap> map = new TreeMap<>(); - for( final long[] blockPos : blockPositions ) { - Position shardPos = Position.wrap(getShardPositionForBlock(blockPos)); - if( !map.containsKey(shardPos)) { - map.put(shardPos, new ArrayList<>()); - } - map.get(shardPos).add(blockPos); - } - - return map; - } - - default Map>> groupBlocks(final List> blocks) { - - // figure out how to re-use groupBlockPositions here? - final TreeMap>> map = new TreeMap<>(); - for (final DataBlock block : blocks) { - Position shardPos = Position.wrap(getShardPositionForBlock(block.getGridPosition())); - if (!map.containsKey(shardPos)) { - map.put(shardPos, new ArrayList<>()); - } - map.get(shardPos).add(block); - } - - return map; - } - - /** - * @return the number of blocks per shard - */ - default long getNumBlocksPerShard() { - - return Arrays.stream(getBlocksPerShard()).reduce(1, (x, y) -> x * y); - } - - default Stream blockPositions() { - - final int[] blocksPerShard = getBlocksPerShard(); - return toStream( new GridIterator(shardsPerImage())) - .flatMap( shardPosition -> { - final int nd = getNumDimensions(); - final long[] min = getBlockPositionFromShardPosition(shardPosition, new long[nd]); - return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); - }); - } - - static Stream toStream( final Iterator it ) { - return StreamSupport.stream( Spliterators.spliteratorUnknownSize( - it, Spliterator.ORDERED), - false); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java new file mode 100644 index 000000000..97b0d446d --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java @@ -0,0 +1,232 @@ +/*- + * #%L + * Not HDF5 + * %% + * Copyright (C) 2017 - 2025 Stephan Saalfeld + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.janelia.saalfeldlab.n5; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; +import org.janelia.saalfeldlab.n5.codec.RawBytes; +import org.janelia.saalfeldlab.n5.shard.InMemoryShard; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec; +import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; +import org.janelia.saalfeldlab.n5.util.GridIterator; +import org.janelia.saalfeldlab.n5.util.Position; +import org.junit.Test; + +/** + * Unit tests for DatasetAttributes class. + */ +public class DatasetAttributesTest { + + /** + * Test that validateBlockShardSizes method accepts valid shard and block size combinations. + */ + @Test + public void testValidateBlockShardSizesValid() { + + // Test case 1: shard size equals block size + long[] dimensions = new long[]{100, 200, 300}; + int[] shardSize = new int[]{64, 64, 64}; + int[] blockSize = new int[]{64, 64, 64}; + DataType dataType = DataType.UINT8; + + // This should not throw any exception + DatasetAttributes attrs = new DatasetAttributes(dimensions, shardSize, blockSize, dataType); + assertEquals(shardSize, attrs.getShardSize()); + assertEquals(blockSize, attrs.getBlockSize()); + assertArrayEquals(new int[]{1, 1, 1}, attrs.getBlocksPerShard()); + + // Test case 2: shard size is a multiple of block size + shardSize = new int[]{128}; + blockSize = new int[]{64}; + attrs = new DatasetAttributes(new long[]{128}, shardSize, blockSize, dataType); + assertEquals(shardSize, attrs.getShardSize()); + assertEquals(blockSize, attrs.getBlockSize()); + assertArrayEquals(new int[]{2}, attrs.getBlocksPerShard()); + + // Test case 3: different multiples per dimension + shardSize = new int[]{128, 256, 32, 2}; + blockSize = new int[]{32, 64, 32, 1}; + attrs = new DatasetAttributes(new long[]{128, 128, 128, 128}, shardSize, blockSize, dataType); + assertEquals(shardSize, attrs.getShardSize()); + assertEquals(blockSize, attrs.getBlockSize()); + assertArrayEquals(new int[]{4, 4, 1, 2}, attrs.getBlocksPerShard()); + + // Test case 4: large multiples + shardSize = new int[]{1024, 2048, 512}; + blockSize = new int[]{32, 64, 16}; + attrs = new DatasetAttributes(dimensions, shardSize, blockSize, dataType); + assertEquals(shardSize, attrs.getShardSize()); + assertEquals(blockSize, attrs.getBlockSize()); + assertArrayEquals(new int[]{32, 32, 32}, attrs.getBlocksPerShard()); + } + + /** + * Test that validateBlockShardSizes method rejects invalid shard and block size combinations. + */ + @Test + public void testValidateBlockShardSizesInvalid() { + + final long[] dimensions = new long[]{100, 200, 300}; + final DataType dataType = DataType.UINT8; + + // Block size too small + IllegalArgumentException ex0 = assertThrows( + IllegalArgumentException.class, + () -> new DatasetAttributes(dimensions, new int[]{1, 1, 1}, new int[]{1, 0, -1}, dataType)); + assertTrue(ex0.getMessage().contains("<= 0")); + + // Different number of dimensions + IllegalArgumentException ex1 = assertThrows( + IllegalArgumentException.class, + () -> new DatasetAttributes(dimensions, new int[]{64, 64}, new int[]{32, 32, 32}, dataType)); + assertTrue(ex1.getMessage().contains("must equal number of dimensions")); + + // Shard size smaller than block size + IllegalArgumentException ex2 = assertThrows( + IllegalArgumentException.class, + () -> new DatasetAttributes(dimensions, new int[]{32, 64, 64}, new int[]{64, 64, 64}, dataType)); + assertTrue(ex2.getMessage().contains("is larger than the block size")); + + // Shard size not a multiple of block size + IllegalArgumentException ex3 = assertThrows( + IllegalArgumentException.class, + () -> new DatasetAttributes(dimensions, new int[]{100, 100, 100}, new int[]{64, 64, 64}, dataType)); + assertTrue(ex3.getMessage().contains("not a multiple of the block size")); + + // Multiple violations - shard smaller than block in one dimension + IllegalArgumentException ex4 = assertThrows( + IllegalArgumentException.class, + () -> new DatasetAttributes(dimensions, new int[]{128, 32, 128}, new int[]{64, 64, 64}, dataType)); + assertTrue(ex4.getMessage().contains("is larger than the block size")); + + // Edge case - shard size of 0 + assertThrows( + IllegalArgumentException.class, + () -> new DatasetAttributes(dimensions, new int[]{0, 64, 64}, new int[]{64, 64, 64}, dataType)); + } + + @Test + public void testShardProperties() { + + final long[] arraySize = new long[]{16, 16}; + final int[] shardSize = new int[]{16, 16}; + final long[] shardPosition = new long[]{1, 1}; + final int[] blkSize = new int[]{4, 4}; + + final DatasetAttributes dsetAttrs = new DatasetAttributes( + arraySize, + shardSize, + blkSize, + DataType.UINT8, + new ShardingCodec( + blkSize, + new Codec[]{new N5BlockCodec()}, + new DeterministicSizeCodec[]{new RawBytes()}, + IndexLocation.END + ) + ); + + final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); + + assertArrayEquals(new int[]{4, 4}, shard.getBlockGridSize()); + + assertArrayEquals(new long[]{0, 0}, shard.getShardPosition(0, 0)); + assertArrayEquals(new long[]{1, 1}, shard.getShardPosition(5, 5)); + assertArrayEquals(new long[]{1, 0}, shard.getShardPosition(5, 0)); + assertArrayEquals(new long[]{0, 1}, shard.getShardPosition(0, 5)); + + assertArrayEquals(new int[]{0, 0}, shard.getBlockPosition(4, 4)); + assertArrayEquals(new int[]{1, 1}, shard.getBlockPosition(5, 5)); + assertArrayEquals(new int[]{2, 2}, shard.getBlockPosition(6, 6)); + assertArrayEquals(new int[]{3, 3}, shard.getBlockPosition(7, 7)); + } + + @Test + public void testShardGrouping() { + + final long[] arraySize = new long[]{8, 12}; + final int[] shardSize = new int[]{4, 6}; + final int[] blkSize = new int[]{2, 3}; + + final DatasetAttributes attrs = new DatasetAttributes( + arraySize, + shardSize, + blkSize, + DataType.UINT8, + new ShardingCodec( + blkSize, + new Codec[]{ new N5BlockCodec() }, + new DeterministicSizeCodec[]{new RawBytes()}, + IndexLocation.END + ) + ); + + List blockPositions = blockPositions(attrs).collect(Collectors.toList()); + final Map> result = attrs.groupBlockPositions(blockPositions); + + // there are four shards in this image + assertEquals(4, result.size()); + + // there are four blocks per shard in this image + result.values().forEach(x -> assertEquals(4, x.size())); + } + + private static Stream blockPositions( final DatasetAttributes attrs ) { + + final int nd = attrs.getNumDimensions(); + final int[] blocksPerShard = attrs.getBlocksPerShard(); + return toStream( new GridIterator(attrs.shardsPerImage())) + .flatMap( shardPosition -> { + final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new long[nd]); + return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); + }); + } + + private static Stream toStream( final Iterator it ) { + return StreamSupport.stream( Spliterators.spliteratorUnknownSize( + it, Spliterator.ORDERED), + false); + } + +} \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java deleted file mode 100644 index 97aff4dc6..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardPropertiesTests.java +++ /dev/null @@ -1,130 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; -import org.janelia.saalfeldlab.n5.codec.RawBytes; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; -import org.janelia.saalfeldlab.n5.util.Position; -import org.junit.Test; - -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -public class ShardPropertiesTests { - - @Test - public void testShardProperties() { - - final long[] arraySize = new long[]{16, 16}; - final int[] shardSize = new int[]{16, 16}; - final long[] shardPosition = new long[]{1, 1}; - final int[] blkSize = new int[]{4, 4}; - - final DatasetAttributes dsetAttrs = new DatasetAttributes( - arraySize, - shardSize, - blkSize, - DataType.UINT8, - new ShardingCodec( - blkSize, - new Codec[]{new N5BlockCodec()}, - new DeterministicSizeCodec[]{new RawBytes()}, - IndexLocation.END - ) - ); - - final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); - - assertArrayEquals(new int[]{4, 4}, shard.getBlockGridSize()); - - assertArrayEquals(new long[]{0, 0}, shard.getShardPosition(0, 0)); - assertArrayEquals(new long[]{1, 1}, shard.getShardPosition(5, 5)); - assertArrayEquals(new long[]{1, 0}, shard.getShardPosition(5, 0)); - assertArrayEquals(new long[]{0, 1}, shard.getShardPosition(0, 5)); - - // assertNull(shard.getBlockPosition(0, 0)); - // assertNull(shard.getBlockPosition(3, 3)); - - assertArrayEquals(new int[]{0, 0}, shard.getBlockPosition(4, 4)); - assertArrayEquals(new int[]{1, 1}, shard.getBlockPosition(5, 5)); - assertArrayEquals(new int[]{2, 2}, shard.getBlockPosition(6, 6)); - assertArrayEquals(new int[]{3, 3}, shard.getBlockPosition(7, 7)); - } - - @Test - public void testShardBlockPositionIterator() { - - final long[] arraySize = new long[]{16, 16}; - final int[] shardSize = new int[]{16, 16}; - final long[] shardPosition = new long[]{1, 1}; - final int[] blkSize = new int[]{4, 4}; - - final DatasetAttributes dsetAttrs = new DatasetAttributes( - arraySize, - shardSize, - blkSize, - DataType.UINT8, - new ShardingCodec( - blkSize, - new Codec[]{ new N5BlockCodec() }, - new DeterministicSizeCodec[]{new RawBytes()}, - IndexLocation.END - ) - ); - - final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); - - int i = 0; - Iterator it = shard.blockPositionIterator(); - long[] p = null; - while (it.hasNext()) { - - p = it.next(); - if (i == 0) - assertArrayEquals(new long[]{4, 4}, p); - - i++; - } - assertEquals(16, i); - assertArrayEquals(new long[]{7, 7}, p); - } - - @Test - public void testShardGrouping() { - - final long[] arraySize = new long[]{8, 12}; - final int[] shardSize = new int[]{4, 6}; - final int[] blkSize = new int[]{2, 3}; - - final DatasetAttributes attrs = new DatasetAttributes( - arraySize, - shardSize, - blkSize, - DataType.UINT8, - new ShardingCodec( - blkSize, - new Codec[]{ new N5BlockCodec() }, - new DeterministicSizeCodec[]{new RawBytes()}, - IndexLocation.END - ) - ); - - List blockPositions = attrs.blockPositions().collect(Collectors.toList()); - final Map> result = attrs.groupBlockPositions(blockPositions); - - // there are four shards in this image - assertEquals(4, result.size()); - - // there are four blocks per shard in this image - result.values().forEach(x -> assertEquals(4, x.size())); - } - -} From e22e50515e5aae9b91d6207bb66d3651cc5002fe Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 2 Jul 2025 14:49:46 -0400 Subject: [PATCH 259/423] refactor!: blockPositions for shard methods are shardRelative * rename and better document methods taking blockPositions * refactoring * update tests --- .../org/janelia/saalfeldlab/n5/DataBlock.java | 2 +- .../saalfeldlab/n5/DatasetAttributes.java | 69 +++++++++++------- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 3 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 3 +- .../saalfeldlab/n5/shard/InMemoryShard.java | 2 +- .../janelia/saalfeldlab/n5/shard/Shard.java | 73 +++++++------------ .../saalfeldlab/n5/shard/ShardingCodec.java | 6 +- .../saalfeldlab/n5/shard/VirtualShard.java | 12 +-- .../janelia/saalfeldlab/n5/util/Position.java | 4 + .../saalfeldlab/n5/DatasetAttributesTest.java | 22 +++--- .../saalfeldlab/n5/N5BlockAsShardTest.java | 4 +- .../saalfeldlab/n5/demo/BlockIterators.java | 2 +- 12 files changed, 97 insertions(+), 105 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 5d3124cf3..b2e2cf1d6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -50,7 +50,7 @@ public interface DataBlock { int[] getSize(); /** - * Returns the position of this data block on the block grid. + * Returns the position of this data block on the block grid relative to dataset. *

      * The dimensionality of the grid position is expected to be equal to the * dimensionality of the dataset. Consistency is not enforced. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 496167d08..d92819862 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -215,9 +215,9 @@ public int[] getBlockSize() { } /** - * Returns the number of blocks per dimension for a shard. + * Returns the number of blocks per dimension for each shard. * - * @return the size of the block grid of a shard + * @return the blocks per shard */ public int[] getBlocksPerShard() { @@ -232,29 +232,31 @@ public int[] getBlocksPerShard() { } /** - * Returns the number of blocks per dimension that tile the image. + * Returns the number of blocks per dimension for this dataset. * - * @return blocks per image + * @return blocks per dataset */ - public long[] blocksPerImage() { + public long[] blocksPerDataset() { return IntStream.range(0, getNumDimensions()) .mapToLong(i -> (long) Math.ceil((double)getDimensions()[i] / getBlockSize()[i])) .toArray(); } /** - * Returns the number of shards per dimension that tile the image. + * Returns the number of shards per dimension for this dataset. * - * @return shards per image + * @return shards per dataset */ - public long[] shardsPerImage() { + public long[] shardsPerDataset() { return IntStream.range(0, getNumDimensions()) .mapToLong(i -> (long)Math.ceil((double)getDimensions()[i] / getShardSize()[i])) .toArray(); } /** - * @return the number of blocks per shard + * Returns the total number of blocks in each shard. + * + * @return number of blocks in a shard */ public long getNumBlocksPerShard() { @@ -262,10 +264,11 @@ public long getNumBlocksPerShard() { } /** - * Given a block's position relative to the array, returns the position of the shard containing that block relative to the shard grid. + * Given a block's position relative to the dataset, returns the position of + * the shard containing that block. * * @param blockGridPosition - * position of a block relative to the array + * position of a block relative to the dataset * @return the position of the containing shard in the shard grid */ public long[] getShardPositionForBlock(final long... blockGridPosition) { @@ -280,41 +283,51 @@ public long[] getShardPositionForBlock(final long... blockGridPosition) { } /** - * Returns the block at the given position relative to this shard, or null if this shard does not contain the given block. - * - * @return the block position + * Given a {@code datasetRelativeBlockPosition} returns the position + * relative the the shard at position {@code shardPosition}. + * + * @param shardPosition + * position of the shard + * @param datasetRelativeBlockPosition + * position of the block relative to the dataset + * @return position of the block relative to the shard + * @see {@link #getBlockPositionFromShardPosition(long[], int[])} */ - public int[] getBlockPositionInShard(final long[] shardPosition, final long[] blockPosition) { + public int[] getShardRelativeBlockPosition(final long[] shardPosition, final long[] datasetRelativeBlockPosition) { - final long[] shardPos = getShardPositionForBlock(blockPosition); + final long[] shardPos = getShardPositionForBlock(datasetRelativeBlockPosition); if (!Arrays.equals(shardPosition, shardPos)) return null; final int[] shardSize = getBlocksPerShard(); - final int[] blockShardPos = new int[shardSize.length]; + final int[] shardRelativeBlockPosition = new int[shardSize.length]; for (int i = 0; i < shardSize.length; i++) { - blockShardPos[i] = (int)(blockPosition[i] % shardSize[i]); + shardRelativeBlockPosition[i] = (int)(datasetRelativeBlockPosition[i] % shardSize[i]); } - return blockShardPos; + return shardRelativeBlockPosition; } /** - * Given a block's position relative to a shard, returns its position relative - * to the image. + * Given a {@code shardRelativeBlockPosition} relative to the shard at + * position {@code shardPosition}, returns the block' position relative the + * dataset. * - * @param shardPosition shard position in the shard grid - * @param blockPosition block position relative to the shard - * @return the block position in the block grid + * @param shardPosition + * position of the shard + * @param shardRelativeBlockPosition + * position of the block relative to the shard + * @return position of the block relative to the dataset + * @see {@link #getShardRelativeBlockPosition(long[], int[])} */ - public long[] getBlockPositionFromShardPosition(final long[] shardPosition, final long[] blockPosition) { + public long[] getBlockPositionFromShardPosition(final long[] shardPosition, final int[] shardRelativeBlockPosition) { final int[] shardBlockSize = getBlocksPerShard(); - final long[] blockImagePos = new long[getNumDimensions()]; + final long[] datasetRelativeBlockPosition = new long[getNumDimensions()]; for (int i = 0; i < getNumDimensions(); i++) { - blockImagePos[i] = (shardPosition[i] * shardBlockSize[i]) + (blockPosition[i]); + datasetRelativeBlockPosition[i] = (shardPosition[i] * shardBlockSize[i]) + (shardRelativeBlockPosition[i]); } - return blockImagePos; + return datasetRelativeBlockPosition; } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 3417cf8f9..bde34a3a8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -37,7 +37,6 @@ import java.util.Map; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -149,7 +148,7 @@ default List> readBlocks( continue; for (final long[] blkPosition : e.getValue()) { - blocks.add(currentShard.getBlock(blkPosition)); + blocks.add(currentShard.getBlock(currentShard.getRelativeBlockPosition(blkPosition))); } } return blocks; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 26f7b8cdc..650e2d76b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -365,7 +365,8 @@ default boolean deleteBlock( return false; // Shard doesn't exist, so block doesn't exist - // return false for consistency - if (!shard.blockExists(gridPosition)) + final int[] relativePosition = shard.getRelativeBlockPosition(gridPosition); + if (!shard.blockExists(relativePosition)) return false; // Convert to InMemoryShard to manipulate blocks diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index ce8842281..10f6adb84 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -35,7 +35,7 @@ private void storeBlock(DataBlock block) { } @Override - public DataBlock getBlock(long... blockGridPosition) { + public DataBlock getBlock(int... blockGridPosition) { return blocks.get(Position.wrap(blockGridPosition)); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 1158fbedf..5dad16e3d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -60,68 +60,45 @@ default int[] getBlockSize() { long[] getGridPosition(); /** - * Returns of the block at the given position relative to this shard, or null if this shard does not contain the given block. + * Given a {@code blockPosition} relative to the dataset, return its position relative to this + * shard. * - * @return the shard position + * @param blockPositionInDataset dataset-relative block position + * @return the shard-relative block positiorelativeBlockPositionn + * @see {@link DatasetAttributes#getBlockPositionInShard(long[], long[])} + * @see {@link #getBlockPositionFromShardPosition(long[], long[])} */ - default int[] getBlockPosition(long... blockPosition) { + default int[] getRelativeBlockPosition(long... datasetBlockPosition) { - final long[] shardPos = getDatasetAttributes().getShardPositionForBlock(blockPosition); - return getDatasetAttributes().getBlockPositionInShard(shardPos, blockPosition); - } - - /** - * Returns the position in pixels of the - * - * @return the min - */ - default long[] getShardMinPosition(long... shardPosition) { - - final int[] shardSize = getSize(); - final long[] shardMin = new long[shardSize.length]; - for (int i = 0; i < shardSize.length; i++) { - shardMin[i] = shardPosition[i] * shardSize[i]; - } - return shardMin; - } - - /** - * Returns the position of the shard containing the block with the given block position. - * - * @return the shard position - */ - default long[] getShardPosition(long... blockPosition) { - - final int[] shardBlockDimensions = getBlockGridSize(); - final long[] shardGridPosition = new long[shardBlockDimensions.length]; - for (int i = 0; i < shardGridPosition.length; i++) { - shardGridPosition[i] = (long)Math.floor((double)(blockPosition[i]) / shardBlockDimensions[i]); - } - - return shardGridPosition; + return getDatasetAttributes().getShardRelativeBlockPosition( + getGridPosition(), + datasetBlockPosition); } /** - * Tests whether the block at the {@code blockGridPosition} exists. + * Tests whether the block at the {@code relativeBlockPosition} (relative to + * this shard) exists. *

      * Avoids reading the block data, if possible. * - * @return true of the block exists + * @return true of the block exists in this shard */ - default boolean blockExists(long... blockGridPosition) { + default boolean blockExists(int... relativeBlockPosition) { - final int[] relativePosition = getBlockPosition(blockGridPosition); - return getIndex().exists(relativePosition); + return getIndex().exists(relativeBlockPosition); } /** - * Retrieve the DataBlock at {@code blockGridPosition} if it exists and is - * a member of this Shard. + * Retrieve the DataBlock at {@code blockGridPosition} relative to this shard if it exists and is + * a member of this shard. + *

      + * If needed, use {@code getRelativeBlockPosition} to convert block positions relative to the dataset into + * positions relative to this shard. * - * @param blockGridPosition position of the desired block in the block grid + * @param blockGridPosition position of the desired block relative to this shard * @return the block if it exists and is part of this shard, otherwise null */ - DataBlock getBlock(long... blockGridPosition); + DataBlock getBlock(int... blockGridPosition); default Iterator> iterator() { @@ -173,7 +150,7 @@ default ReadData createReadData() throws N5IOException { long prevCount = 0; for (DataBlock block : getBlocks()) { arrayCodec.encode(block).writeTo(countOut); - final int[] blockPosition = getBlockPosition(block.getGridPosition()); + final int[] blockPosition = getRelativeBlockPosition(block.getGridPosition()); final long curCount = countOut.getByteCount(); final long blockWrittenSize = curCount - prevCount; prevCount = curCount; @@ -193,7 +170,7 @@ default ReadData createReadData() throws N5IOException { blocksData.add(readDataBlock); final long length = readDataBlock.length(); synchronized (index) { - index.set(blockOffset.getAndAdd(length), length, getBlockPosition(dataBlock.getGridPosition())); + index.set(blockOffset.getAndAdd(length), length, getRelativeBlockPosition(dataBlock.getGridPosition())); } } return ReadData.from(out -> { @@ -237,7 +214,7 @@ public DataBlock next() { while (!index.exists(blockIndex++)) it.fwd(); - return shard.getBlock(it.next()); + return shard.getBlock(GridIterator.long2int(it.next())); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 3f93eab26..f6a5f12b3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -136,8 +136,10 @@ public ReadData encode(DataBlock dataBlock) { public DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException { final ReadData splitableReadData = readData.materialize(); - final VirtualShard shard = new VirtualShard<>(attributes, gridPosition, splitableReadData); - return shard.getBlock(gridPosition); + final long[] shardPosition = getPositionForBlock(attributes, gridPosition); + final VirtualShard shard = new VirtualShard<>(attributes, shardPosition, splitableReadData); + final int[] relativeBlockPosition = shard.getRelativeBlockPosition(gridPosition); + return shard.getBlock(relativeBlockPosition); } public ShardIndex createIndex(final DatasetAttributes attributes) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index a7d513127..b657fe562 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -92,11 +92,7 @@ public List> getBlocks(final int[] blockIndexes) { } @Override - public DataBlock getBlock(long... blockGridPosition) { - - final int[] relativePosition = getBlockPosition(blockGridPosition); - if (relativePosition == null) - throw new N5IOException("Attempted to read a block from the wrong shard."); + public DataBlock getBlock(int... relativePosition) { final ShardIndex idx = getIndex(); if (!idx.exists(relativePosition)) @@ -105,14 +101,14 @@ public DataBlock getBlock(long... blockGridPosition) { final long blockOffset = idx.getOffset(relativePosition); final long blockSize = idx.getNumBytes(relativePosition); - final long[] blockPosInImg = getDatasetAttributes().getBlockPositionFromShardPosition(getGridPosition(), blockGridPosition); + final long[] blockPosInDataset = getDatasetAttributes().getBlockPositionFromShardPosition(getGridPosition(), relativePosition); try { final ReadData blockData = shardData.slice(blockOffset, blockSize); - return getBlock(blockData, blockPosInImg); + return getBlock(blockData, blockPosInDataset); } catch (final N5Exception.N5NoSuchKeyException e) { return null; } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to read block from " + Arrays.toString(blockGridPosition), e); + throw new N5IOException("Failed to read block from " + Arrays.toString(blockPosInDataset), e); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java index eb1a9078f..2403835b9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java @@ -59,4 +59,8 @@ static Position wrap(final long[] p) { return new FinalPosition(p); } + static Position wrap(final int[] p) { + return new FinalPosition(GridIterator.int2long(p)); + } + } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java index 97b0d446d..2ea84671a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java @@ -171,15 +171,15 @@ public void testShardProperties() { assertArrayEquals(new int[]{4, 4}, shard.getBlockGridSize()); - assertArrayEquals(new long[]{0, 0}, shard.getShardPosition(0, 0)); - assertArrayEquals(new long[]{1, 1}, shard.getShardPosition(5, 5)); - assertArrayEquals(new long[]{1, 0}, shard.getShardPosition(5, 0)); - assertArrayEquals(new long[]{0, 1}, shard.getShardPosition(0, 5)); - - assertArrayEquals(new int[]{0, 0}, shard.getBlockPosition(4, 4)); - assertArrayEquals(new int[]{1, 1}, shard.getBlockPosition(5, 5)); - assertArrayEquals(new int[]{2, 2}, shard.getBlockPosition(6, 6)); - assertArrayEquals(new int[]{3, 3}, shard.getBlockPosition(7, 7)); + assertArrayEquals(new long[]{0, 0}, dsetAttrs.getShardPositionForBlock(0, 0)); + assertArrayEquals(new long[]{1, 1}, dsetAttrs.getShardPositionForBlock(5, 5)); + assertArrayEquals(new long[]{1, 0}, dsetAttrs.getShardPositionForBlock(5, 0)); + assertArrayEquals(new long[]{0, 1}, dsetAttrs.getShardPositionForBlock(0, 5)); + + assertArrayEquals(new int[]{0, 0}, shard.getRelativeBlockPosition(4, 4)); + assertArrayEquals(new int[]{1, 1}, shard.getRelativeBlockPosition(5, 5)); + assertArrayEquals(new int[]{2, 2}, shard.getRelativeBlockPosition(6, 6)); + assertArrayEquals(new int[]{3, 3}, shard.getRelativeBlockPosition(7, 7)); } @Test @@ -216,9 +216,9 @@ private static Stream blockPositions( final DatasetAttributes attrs ) { final int nd = attrs.getNumDimensions(); final int[] blocksPerShard = attrs.getBlocksPerShard(); - return toStream( new GridIterator(attrs.shardsPerImage())) + return toStream( new GridIterator(attrs.shardsPerDataset())) .flatMap( shardPosition -> { - final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new long[nd]); + final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]); return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); }); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java index db3fc2b06..b93ee0e45 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java @@ -30,7 +30,7 @@ public DataBlock readBlock(String pathName, DatasetAttributes datasetAttr if (shard == null) return null; - return shard.getBlock(gridPosition); + return shard.getBlock(shard.getRelativeBlockPosition(gridPosition)); } }; } @@ -47,7 +47,7 @@ public DataBlock readBlock(String pathName, DatasetAttributes datasetAttr if (shard == null) return null; - return shard.getBlock(gridPosition); + return shard.getBlock(shard.getRelativeBlockPosition(gridPosition)); } }; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java index f1511c41f..67bf5cf3d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java @@ -81,7 +81,7 @@ public static Stream shardPositions( DatasetAttributes attrs ) { .flatMap( shardPosition -> { final int nd = attrs.getNumDimensions(); - final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new long[nd]); + final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]); return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); }); } From 3ea57cc0f825e8ff77246646c639dd1e423e8571 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 2 Jul 2025 16:27:36 -0400 Subject: [PATCH 260/423] feat: add GridIterator.nextInt * shards index their blocks with int[] * avoids creating many int[]s --- .../janelia/saalfeldlab/n5/shard/Shard.java | 2 +- .../saalfeldlab/n5/util/GridIterator.java | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 5dad16e3d..e19046b9b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -214,7 +214,7 @@ public DataBlock next() { while (!index.exists(blockIndex++)) it.fwd(); - return shard.getBlock(GridIterator.long2int(it.next())); + return shard.getBlock(it.nextInt()); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java index b201015bb..283cfc24e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java @@ -13,6 +13,8 @@ public class GridIterator implements Iterator { final protected long[] position; + final protected int[] intPosition; + final protected long[] min; final protected int lastIndex; @@ -24,6 +26,7 @@ public GridIterator(final long[] dimensions, final long[] min) { final int n = dimensions.length; this.dimensions = new long[n]; this.position = new long[n]; + this.intPosition = new int[n]; this.min = min; steps = new long[n]; @@ -74,6 +77,13 @@ public long[] next() { return position; } + public int[] nextInt() { + + next(); + long2int(position, intPosition); + return intPosition; + } + public int getIndex() { return index; @@ -97,6 +107,14 @@ public static void indexToPosition(long index, final int[] dimensions, final lon } } + public static int[] long2int(final long[] src, final int[] tgt) { + + for (int d = 0; d < tgt.length; ++d) + tgt[d] = (int)src[d]; + + return tgt; + } + public static int[] long2int(final long[] a) { final int[] i = new int[a.length]; From d31ce223d63b0955e5567895b57b912dc0ed4f26 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 3 Jul 2025 09:34:00 -0400 Subject: [PATCH 261/423] feat/doc: add ShardIndex.setEmpty * add documentation --- .../saalfeldlab/n5/shard/ShardIndex.java | 221 +++++++++++++++++- 1 file changed, 216 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index f1583d9d2..f6d0f6e76 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -18,8 +18,31 @@ import java.util.Arrays; import java.util.stream.IntStream; +/** + * The ShardIndex tracks the offset and length of blocks contained within a + * shard. + *

      + * Blocks in a shard are arrayed in an n-dimensional grid, referred to as the + * {@code shardBlockGrid}. The ShardIndex is implemented as an (n+1)-dimensional + * {@link LongArrayDataBlock}, where the 0th dimensions is length 2 and contains + * the block offsets and lengths. The grid position of the index iteself is meaningless, + * and as a result, {@link #getGridPosition()} will return {@code null}. + *

      + * The index stores two values for each block: offset and number of bytes. Blocks + * that don't exist are marked with the special value {@link #EMPTY_INDEX_NBYTES}. + *

      + * Block grid positions in this class are relative to the shard. + * + * @see The + * Zarr V3 specification for the binary shard format + */ public class ShardIndex extends LongArrayDataBlock { + /** + * Special value indicating an empty block entry in the index. + * Used for both offset and length when a block doesn't exist. + */ public static final long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; private static final int BYTES_PER_LONG = 8; private static final int LONGS_PER_BLOCK = 2; @@ -30,36 +53,75 @@ public class ShardIndex extends LongArrayDataBlock { private final DeterministicSizeCodec[] codecs; private final ShardIndexAttributes indexAttributes; + /** + * Creates a ShardIndex with specified data. + * + * @param shardBlockGridSize the dimensions of the block grid within the shard + * @param data the raw index data containing offsets and lengths + * @param location where the index is stored (START or END of shard) + * @param codecs compression codecs applied to the index + */ public ShardIndex(int[] shardBlockGridSize, long[] data, IndexLocation location, final DeterministicSizeCodec... codecs) { + // prepend the number of longs per block to the shard block grid size super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, data); this.codecs = codecs; this.location = location; this.indexAttributes = new ShardIndexAttributes(this); } + /** + * Creates an empty ShardIndex with specified location. + * + * @param shardBlockGridSize the dimensions of the block grid within the shard + * @param location where the index is stored (START or END of shard) + * @param codecs compression codecs applied to the index + */ public ShardIndex(int[] shardBlockGridSize, IndexLocation location, DeterministicSizeCodec... codecs) { this(shardBlockGridSize, emptyIndexData(shardBlockGridSize), location, codecs); } + /** + * Creates an empty ShardIndex with default location (END). + * + * @param shardBlockGridSize the dimensions of the block grid within the shard + * @param codecs compression codecs applied to the index + */ public ShardIndex(int[] shardBlockGridSize, DeterministicSizeCodec... codecs) { this(shardBlockGridSize, emptyIndexData(shardBlockGridSize), IndexLocation.END, codecs); } + /** + * Checks existence of the block at a given grid position. + * + * @param gridPosition the n-dimensional position of the block in the shard grid + * @return true if the block exists, false otherwise + */ public boolean exists(int[] gridPosition) { return getOffset(gridPosition) != EMPTY_INDEX_NBYTES || getNumBytes(gridPosition) != EMPTY_INDEX_NBYTES; } - public boolean exists(int blockNum) { + /** + * Checks existence of the block at a given flat index. + * + * @param index the flattened index of the block + * @return true if the block exists, false otherwise + */ + public boolean exists(int index) { - return data[blockNum * 2] != EMPTY_INDEX_NBYTES || - data[blockNum * 2 + 1] != EMPTY_INDEX_NBYTES; + return data[index * 2] != EMPTY_INDEX_NBYTES || + data[index * 2 + 1] != EMPTY_INDEX_NBYTES; } + /** + * Gets the total number of blocks that can be stored in this index. + * + * @return the total number of blocks in the shard grid + */ public int getNumBlocks() { /* getSize() is the number of data entries; each block takes 2 entries (offset and length) @@ -67,36 +129,77 @@ public int getNumBlocks() { return Arrays.stream(getSize()).reduce(1, (x, y) -> x * y) / 2; } + /** + * Checks if the index is completely empty (no blocks exist). + * + * @return true if no blocks exist in the index, false otherwise + */ public boolean isEmpty() { return !IntStream.range(0, getNumBlocks()).anyMatch(this::exists); } + /** + * Gets the location of this index within the shard. + * + * @return the index location (START or END) + */ public IndexLocation getLocation() { return location; } + /** + * Gets the offset in this shard in bytes for the block at a grid position. + * + * @param gridPosition the n-dimensional position of the block in the shard grid + * @return the offset in bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist + */ public long getOffset(int... gridPosition) { return data[getOffsetIndex(gridPosition)]; } + /** + * Gets the offset in this shard in bytes for the block at a given index. + * + * @param index the flattened index of the block + * @return the offset in bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist + */ public long getOffsetByBlockIndex(int index) { return data[index * 2]; } + /** + * Gets the number of bytes for the block at a grid position. + * + * @param gridPosition the n-dimensional position of the block in the shard grid + * @return the number of bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist + */ public long getNumBytes(int... gridPosition) { return data[getNumBytesIndex(gridPosition)]; } + /** + * Gets the number of bytes for the block at a given index. + * + * @param index the flattened index of the block + * @return the number of bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist + */ public long getNumBytesByBlockIndex(int index) { return data[index * 2 + 1]; } + /** + * Sets the offset and number of bytes for a block at the specified position. + * + * @param offset the byte offset of the block in the shard + * @param nbytes the number of bytes the block occupies + * @param gridPosition the n-dimensional position of the block in the shard grid + */ public void set(long offset, long nbytes, int[] gridPosition) { final int i = getOffsetIndex(gridPosition); @@ -104,6 +207,22 @@ public void set(long offset, long nbytes, int[] gridPosition) { data[i + 1] = nbytes; } + /** + * Marks a block position as empty. + * + * @param gridPosition the n-dimensional position of the block to mark as empty + */ + public void setEmpty(int[] gridPosition) { + + set(EMPTY_INDEX_NBYTES, EMPTY_INDEX_NBYTES, gridPosition); + } + + /** + * Calculates the flattened array index for the offset value of a block. + * + * @param gridPosition the n-dimensional position of the block + * @return the index in the data array where the offset is stored + */ protected int getOffsetIndex(int... gridPosition) { int idx = (int) gridPosition[0]; @@ -115,11 +234,22 @@ protected int getOffsetIndex(int... gridPosition) { return idx * 2; } + /** + * Calculates the flattened array index for the number of bytes value of a block. + * + * @param gridPosition the n-dimensional position of the block + * @return the index in the data array where the number of bytes is stored + */ protected int getNumBytesIndex(int... gridPosition) { return getOffsetIndex(gridPosition) + 1; } + /** + * Calculates the total size of the index in bytes after compression. + * + * @return the total number of bytes the index occupies after applying all codecs + */ public long numBytes() { final int numEntries = Arrays.stream(getSize()).reduce(1, (x, y) -> x * y); @@ -133,6 +263,13 @@ public long numBytes() { return totalNumBytes; } + /** + * Reads the index data from a shard. + * + * @param shardData the ReadData containing the entire shard + * @param index the ShardIndex to populate with data + * @throws N5IOException if the read operation fails or the shard data has invalid length + */ public static void readFromShard(ReadData shardData, ShardIndex index) throws N5IOException { /* we require a length, so materialize if we don't have one. */ @@ -148,6 +285,14 @@ public static void readFromShard(ReadData shardData, ShardIndex index) throws N5 ShardIndex.read(indexData, index); } + /** + * Reads index data from a ReadData source. + * + * @param indexData the ReadData containing the index + * @param index the ShardIndex to populate with data + * @return true if the read was successful, false if the key doesn't exist + * @throws N5IOException if the read operation fails + */ public static boolean read( final ReadData indexData, final ShardIndex index ) { try (final InputStream in = indexData.inputStream()) { @@ -160,6 +305,13 @@ public static boolean read( final ReadData indexData, final ShardIndex index ) { } } + /** + * Reads index data from an InputStream. + * + * @param indexIn the InputStream containing the index data + * @param index the ShardIndex to populate with data + * @throws N5IOException if the read operation fails + */ public static void read(InputStream indexIn, final ShardIndex index) throws N5IOException { final ReadData dataIn = ReadData.from(indexIn); @@ -168,6 +320,13 @@ public static void read(InputStream indexIn, final ShardIndex index) throws N5IO System.arraycopy(indexBlock.getData(), 0, index.data, 0, index.data.length); } + /** + * Writes the index to an OutputStream. + * + * @param outputStream the OutputStream to write to + * @param index the ShardIndex to write + * @throws N5IOException if the write operation fails + */ public static void write( final OutputStream outputStream, final ShardIndex index ) throws N5IOException { final Codec.ArrayCodec indexCodec = index.indexAttributes.getArrayCodec(); @@ -175,12 +334,25 @@ public static void write( final OutputStream outputStream, final ShardIndex inde } + /** + * Gets the array codec used for encoding/decoding this index. + * + * @return the array codec + */ public Codec.ArrayCodec getArrayCodec() { return indexAttributes.getArrayCodec(); } + /** + * DatasetAttributes for the ShardIndex, used for codec operations. + */ private static class ShardIndexAttributes extends DatasetAttributes { + /** + * Creates attributes for the given ShardIndex. + * + * @param index the ShardIndex + */ public ShardIndexAttributes(ShardIndex index) { super( Arrays.stream(index.getSize()).mapToLong(it -> it).toArray(), @@ -191,11 +363,26 @@ public ShardIndexAttributes(ShardIndex index) { } } + /** + * Calculates the byte boundaries of the index within a shard. + * + * @param index the ShardIndex + * @param objectSize the total size of the shard in bytes + * @return the byte boundaries of the index + */ public static IndexByteBounds byteBounds(final ShardIndex index, long objectSize) { return byteBounds(index.numBytes(), index.location, objectSize); } + /** + * Calculates the byte boundaries of an index within a shard. + * + * @param indexSize the size of the index in bytes + * @param indexLocation the location of the index (START or END) + * @param objectSize the total size of the shard in bytes + * @return the byte boundaries of the index + */ public static IndexByteBounds byteBounds(final long indexSize, final IndexLocation indexLocation, final long objectSize) { if (indexLocation == IndexLocation.START) { @@ -205,12 +392,24 @@ public static IndexByteBounds byteBounds(final long indexSize, final IndexLocati } } + /** + * Represents the byte boundaries of an index within a shard. + */ public static class IndexByteBounds { + /** The start byte position (inclusive) */ public final long start; + /** The end byte position (inclusive) */ public final long end; + /** The size in bytes */ public final long size; + /** + * Creates byte boundaries. + * + * @param start the start position (inclusive) + * @param end the end position (inclusive) + */ public IndexByteBounds(long start, long end) { this.start = start; @@ -219,6 +418,12 @@ public IndexByteBounds(long start, long end) { } } + /** + * Creates an empty index data array filled with {@link #EMPTY_INDEX_NBYTES}. + * + * @param size the dimensions of the block grid + * @return an array filled with empty values + */ private static long[] emptyIndexData(final int[] size) { final int N = 2 * Arrays.stream(size).reduce(1, (x, y) -> x * y); @@ -227,6 +432,13 @@ private static long[] emptyIndexData(final int[] size) { return data; } + /** + * Prepends a value to an array. + * + * @param value the value to prepend + * @param array the original array + * @return a new array with the value prepended + */ private static int[] prepend(final int value, final int[] array) { final int[] indexBlockSize = new int[array.length + 1]; @@ -253,5 +465,4 @@ public boolean equals(Object other) { } return true; } -} - +} \ No newline at end of file From 2be0d47d23ac256906658d69aad95df1d3750d51 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 7 Jul 2025 19:04:49 -0400 Subject: [PATCH 262/423] refactor: mark getCompression() as deprecated * instead of private * to make updates a little easier --- .../saalfeldlab/n5/DatasetAttributes.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index d92819862..eede5d863 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -399,10 +399,13 @@ public ShardingCodec getShardingCodec() { * Only used for deserialization for N5 backwards compatibility. * {@link Compression} is no longer a special case. Prefer to reference {@link #getCodecs()} * Will return {@link RawCompression} if no compression is otherwise provided, for legacy compatibility. + *

      + * Deprecated in favor of {@link #getCodecs()}. * * @return compression Codec, if one was present, or else RawCompression */ - private Compression getCompression() { + @Deprecated + public Compression getCompression() { return Arrays.stream(byteCodecs) .filter(it -> it instanceof Compression) @@ -473,25 +476,19 @@ public static class DatasetAttributesAdapter implements JsonSerializer Date: Mon, 7 Jul 2025 19:08:13 -0400 Subject: [PATCH 263/423] fix/test: serialization of DatasetAttributes * ensure that n5-format serialization is unchanged * separate test of sharding functionality --- .../saalfeldlab/n5/DatasetAttributes.java | 26 +- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 4 +- .../saalfeldlab/n5/AbstractN5Test.java | 205 +--------- .../saalfeldlab/n5/N5BlockAsShardTest.java | 7 - .../saalfeldlab/n5/http/N5HttpTest.java | 10 - .../saalfeldlab/n5/shard/ShardTest.java | 349 +++++++++++++++++- 6 files changed, 364 insertions(+), 237 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index eede5d863..af3c4bf34 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -502,28 +502,16 @@ public static class DatasetAttributesAdapter implements JsonSerializer 1 is actually invalid, but this is checked on construction + if (codecs.length == 0) + obj.add(COMPRESSION_KEY, context.serialize(new RawCompression())); + else + obj.add(COMPRESSION_KEY, context.serialize(codecs[0])); - final BytesCodec[] byteCodecs = attributes.byteCodecs; - final ArrayCodec arrayCodec = attributes.arrayCodec; - final Codec[] allCodecs = new Codec[byteCodecs.length + 1]; - allCodecs[0] = arrayCodec; - System.arraycopy(byteCodecs, 0, allCodecs, 1, byteCodecs.length); - - return allCodecs; + return obj; } private static Compression getCompressionVersion0(final String compressionVersion0Name) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 4ce517e43..5c09a1124 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -62,11 +62,11 @@ public interface GsonUtils { static Gson registerGson(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, DatasetAttributes.getJsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); + gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, DatasetAttributes.getJsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBytes.byteOrderAdapter); gsonBuilder.registerTypeHierarchyAdapter(ShardingCodec.IndexLocation.class, ShardingCodec.indexLocationAdapter); + gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.disableHtmlEscaping(); return gsonBuilder.create(); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 260957589..d32235da8 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -57,12 +57,7 @@ import org.janelia.saalfeldlab.n5.N5Reader.Version; import org.janelia.saalfeldlab.n5.shard.InMemoryShard; import org.janelia.saalfeldlab.n5.shard.Shard; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.url.UriAttributeTest; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; -import org.janelia.saalfeldlab.n5.codec.RawBytes; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -238,18 +233,18 @@ public void testSetAttributeDoesntCreateGroup() { @Test public void testCreateDataset() { - final DatasetAttributes info; - try (N5Writer writer = createTempN5Writer()) { - writer.createDataset(datasetName, dimensions, blockSize, DataType.UINT64); + final DatasetAttributes info; + try (N5Writer writer = createTempN5Writer()) { + writer.createDataset(datasetName, dimensions, blockSize, DataType.UINT64); - assertTrue("Dataset does not exist", writer.exists(datasetName)); + assertTrue("Dataset does not exist", writer.exists(datasetName)); - info = writer.getDatasetAttributes(datasetName); - } - assertArrayEquals(dimensions, info.getDimensions()); - assertArrayEquals(blockSize, info.getBlockSize()); - assertEquals(DataType.UINT64, info.getDataType()); - assertEquals(0, info.getCodecs().length); + info = writer.getDatasetAttributes(datasetName); + } + assertArrayEquals(dimensions, info.getDimensions()); + assertArrayEquals(blockSize, info.getBlockSize()); + assertEquals(DataType.UINT64, info.getDataType()); + assertEquals(0, info.getCodecs().length); } @Test @@ -1177,186 +1172,6 @@ public void testReaderCreation() throws IOException, URISyntaxException { } } - @SuppressWarnings("unchecked") - @Test - public void testShardDelete() { - try (GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)createTempN5Writer()) { - final String datasetName = "testShardDelete"; - - // Create a sharded dataset - final int[] shardSize = new int[]{blockSize[0] * 2, blockSize[1] * 2, blockSize[2] * 2}; - final DatasetAttributes datasetAttributes = new DatasetAttributes( - dimensions, - shardSize, - blockSize, - DataType.UINT8, - new ShardingCodec( - blockSize, - new Codec[]{new N5BlockCodec()}, - new DeterministicSizeCodec[]{new RawBytes()}, - ShardingCodec.IndexLocation.END - ) - ); - writer.createDataset(datasetName, datasetAttributes); - - // Create blocks in different shards - final long[] shardPosition1 = {0, 0, 0}; - final long[] shardPosition2 = {0, 0, 1}; // Different shard in z dimension - final long[] shardPositionNonExistent = {0, 0, 2}; // Different shard in z dimension - - // Create blocks within the first shard - final long[] blockPosition1 = {0, 0, 0}; - final long[] blockPosition2 = {1, 0, 0}; - final long[] blockPosition3 = {0, 1, 0}; - - // Create a block in the second shard - final long[] blockPosition4 = {0, 0, 2}; // This will be in - // shardPosition2 - - final ByteArrayDataBlock block1 = new ByteArrayDataBlock(blockSize, blockPosition1, byteBlock); - final ByteArrayDataBlock block2 = new ByteArrayDataBlock(blockSize, blockPosition2, byteBlock); - final ByteArrayDataBlock block3 = new ByteArrayDataBlock(blockSize, blockPosition3, byteBlock); - final ByteArrayDataBlock block4 = new ByteArrayDataBlock(blockSize, blockPosition4, byteBlock); - - // Write blocks to create shards - writer.writeBlocks(datasetName, datasetAttributes, block1, block2, block3, block4); - - // Verify shards exist - final Shard shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); - final Shard shard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); - final Shard shardDNE = writer.readShard(datasetName, datasetAttributes, shardPositionNonExistent); - assertNotNull("Shard 1 should exist", shard1); - assertNotNull("Shard 2 should exist", shard2); - assertNull("Shard 3 should not exist", shardDNE); - assertEquals("Shard 1 should contain 3 blocks", 3, shard1.getBlocks().size()); - assertEquals("Shard 2 should contain 1 block", 1, shard2.getBlocks().size()); - - // Test deleteShard - boolean deleted1 = writer.deleteShard(datasetName, shardPosition1); - assertTrue("deleteShard should return true when shard exists", deleted1); - - // Verify shard is deleted - final Shard deletedShard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); - assertNull("Shard 1 should be deleted", deletedShard1); - - // Verify blocks in the deleted shard are gone - assertNull("Block 1 should be deleted", writer.readBlock(datasetName, datasetAttributes, blockPosition1)); - assertNull("Block 2 should be deleted", writer.readBlock(datasetName, datasetAttributes, blockPosition2)); - assertNull("Block 3 should be deleted", writer.readBlock(datasetName, datasetAttributes, blockPosition3)); - - // Verify other shard is unaffected - final Shard stillExistingShard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); - assertNotNull("Shard 2 should still exist", stillExistingShard2); - assertNotNull("Block 4 should still exist", writer.readBlock(datasetName, datasetAttributes, blockPosition4)); - - // Test deleting non-existent shard - boolean deletedAgain = writer.deleteShard(datasetName, shardPosition1); - assertFalse("deleteShard should return false when shard doesn't exist", deletedAgain); - - // Test deleting shard at invalid position - final long[] invalidShardPosition = {100, 100, 100}; - boolean deletedInvalid = writer.deleteShard(datasetName, invalidShardPosition); - assertFalse("deleteShard should return false for non-existent shard position", deletedInvalid); - } - } - - @SuppressWarnings("unchecked") - @Test - public void testShardedBlockDelete() { - try (GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)createTempN5Writer()) { - final String datasetName = "testShardBlockDelete"; - - // Create a sharded dataset - final int[] shardSize = new int[]{blockSize[0] * 2, blockSize[1] * 2, blockSize[2] * 2}; - final DatasetAttributes datasetAttributes = new DatasetAttributes( - dimensions, - shardSize, - blockSize, - DataType.UINT8, - new ShardingCodec( - blockSize, - new Codec[]{new N5BlockCodec()}, - new DeterministicSizeCodec[]{new RawBytes()}, - ShardingCodec.IndexLocation.END - ) - ); - writer.createDataset(datasetName, datasetAttributes); - - // Create blocks in different shards - final long[] shardPosition1 = {0, 0, 0}; - final long[] shardPosition2 = {0, 0, 1}; // Different shard in z - // dimension - final long[] shardPositionNonExistent = {0, 0, 2}; // Different - // shard in z - // dimension - - // Create blocks within the first shard - final long[] blockPosition1 = {0, 0, 0}; - final long[] blockPosition2 = {1, 0, 0}; - final long[] blockPosition3 = {0, 1, 0}; - - // Create a block in the second shard - final long[] blockPosition4 = {0, 0, 2}; // This will be in - // shardPosition2 - - final ByteArrayDataBlock block1 = new ByteArrayDataBlock(blockSize, blockPosition1, byteBlock); - final ByteArrayDataBlock block2 = new ByteArrayDataBlock(blockSize, blockPosition2, byteBlock); - final ByteArrayDataBlock block3 = new ByteArrayDataBlock(blockSize, blockPosition3, byteBlock); - final ByteArrayDataBlock block4 = new ByteArrayDataBlock(blockSize, blockPosition4, byteBlock); - - // Write blocks to create shards - writer.writeBlocks(datasetName, datasetAttributes, block1, block2, block3, block4); - - // Verify shards exist - Shard shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); - Shard shard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); - Shard shardDNE = writer.readShard(datasetName, datasetAttributes, shardPositionNonExistent); - assertNotNull("Shard 1 should exist", shard1); - assertNotNull("Shard 2 should exist", shard2); - assertNull("Shard 3 should not exist", shardDNE); - assertEquals("Shard 1 should contain 3 blocks", 3, shard1.getBlocks().size()); - assertEquals("Shard 2 should contain 1 block", 1, shard2.getBlocks().size()); - - // Test delete one block from a multi-block shard - boolean deleted1 = writer.deleteBlock(datasetName, blockPosition1); - assertTrue("deleteBlock1", deleted1); - shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); - assertNotNull("Shard 1 should still exist", shard1); - DataBlock blk1Read = writer.readBlock(datasetName, datasetAttributes, blockPosition1); - DataBlock blk2Read = writer.readBlock(datasetName, datasetAttributes, blockPosition2); - DataBlock blk3Read = writer.readBlock(datasetName, datasetAttributes, blockPosition3); - assertNull("Block 1 should not exist", blk1Read); - assertNotNull("Block 2 should exist", blk2Read); - assertNotNull("Block 3 should exist", blk3Read); - - // Test delete one block from a multi-block shard - boolean deleted2 = writer.deleteBlock(datasetName, blockPosition2); - assertTrue("deleteBlock2", deleted2); - shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); - assertNotNull("Shard 1 should still exist", shard1); - blk2Read = writer.readBlock(datasetName, datasetAttributes, blockPosition2); - blk3Read = writer.readBlock(datasetName, datasetAttributes, blockPosition3); - assertNull("Block 2 should not exist", blk2Read); - assertNotNull("Block 3 should exist", blk3Read); - - // Test delete last block from a multi-block shard - boolean deleted3 = writer.deleteBlock(datasetName, blockPosition3); - assertTrue("deleteBlock3", deleted3); - shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); - assertNull("Shard 1 should not exist", shard1); - blk3Read = writer.readBlock(datasetName, datasetAttributes, blockPosition3); - assertNull("Block 3 should not exist", blk3Read); - - // Test delete last block from a multi-block shard - boolean deleted4 = writer.deleteBlock(datasetName, blockPosition4); - assertTrue("deleteBlock4", deleted4); - shard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); - assertNull("Shard 2 should not exist", shard2); - DataBlock blk4Read = writer.readBlock(datasetName, datasetAttributes, blockPosition4); - assertNull("Block 2 should not exist", blk4Read); - } - } - @Test public void testDelete() { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java index b93ee0e45..ec3fbfbb6 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java @@ -52,11 +52,4 @@ public DataBlock readBlock(String pathName, DatasetAttributes datasetAttr }; } - @Override - @Ignore("irrelevant when creating a sharded dataset") - public void testShardDelete() { } - - @Override - @Ignore("irrelevant when creating a sharded dataset") - public void testShardedBlockDelete() { } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/N5HttpTest.java b/src/test/java/org/janelia/saalfeldlab/n5/http/N5HttpTest.java index a564f5425..672e3e0b6 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/N5HttpTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/N5HttpTest.java @@ -160,16 +160,6 @@ public void testVersion() throws NumberFormatException { } - @Ignore("N5Writer not supported for HTTP") - @Override public void testShardDelete() { - - } - - @Ignore("N5Writer not supported for HTTP") - @Override public void testShardedBlockDelete() { - - } - @Ignore("N5Writer not supported for HTTP") @Override public void testWriterSeparation() { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 850c3d2ae..7ba3e8be5 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -1,36 +1,63 @@ package org.janelia.saalfeldlab.n5.shard; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.Compression; +import org.janelia.saalfeldlab.n5.CompressionAdapter; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5FSTest; +import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.NameConfigAdapter; +import org.janelia.saalfeldlab.n5.DatasetAttributes.DatasetAttributesAdapter; +import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.codec.N5BlockCodec; import org.janelia.saalfeldlab.n5.codec.RawBytes; +import org.janelia.saalfeldlab.n5.codec.Codec.ArrayCodec; +import org.janelia.saalfeldlab.n5.codec.Codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; -import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; + +import java.io.File; +import java.lang.reflect.Type; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.ByteOrder; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Random; import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; @RunWith(Parameterized.class) public class ShardTest { @@ -42,14 +69,39 @@ public class ShardTest { @Override public N5Writer createTempN5Writer() { if (LOCAL_DEBUG) { - final N5Writer writer = N5Factory.createWriter("src/test/resources/test.n5"); - writer.remove(""); //Clear old when starting new test + final N5Writer writer = new ShardedN5Writer("src/test/resources/test.n5"); + writer.remove(""); // Clear old when starting new test return writer; } - return super.createTempN5Writer(); + + final String basePath = new File(tempN5PathName()).toURI().normalize().getPath(); + try { + String uri = new URI("file", null, basePath, null).toString(); + return new ShardedN5Writer(uri); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + return null; + } + + private String tempN5PathName() { + + try { + final File tmpFile = Files.createTempDirectory("n5-shard-test-").toFile(); + tmpFile.delete(); + tmpFile.mkdir(); + tmpFile.deleteOnExit(); + return tmpFile.getCanonicalPath(); + } catch (final Exception e) { + throw new RuntimeException(e); + } } }; + public static GsonBuilder gsonBuilder() { + return new GsonBuilder(); + } + @Parameterized.Parameters(name = "IndexLocation({0}), Index ByteOrder({1})") public static Collection data() { @@ -98,6 +150,13 @@ private DatasetAttributes getTestAttributes() { return getTestAttributes(new long[]{8, 8}, new int[]{4, 4}, new int[]{2, 2}); } + private DatasetAttributes getTestAttributes3d() { + + final int[] blockSize = {33, 22, 11}; + final int[] shardSize = {blockSize[0] * 2, blockSize[1] * 2, blockSize[2] * 2}; + return getTestAttributes(new long[]{10, 20, 30}, shardSize, blockSize); + } + @Test public void writeReadBlocksTest() { @@ -282,6 +341,172 @@ public void writeReadShardTest() { } } + @SuppressWarnings("unchecked") + @Test + public void testShardDelete() { + + final Random rnd = new Random(88); + try (GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)tempN5Factory.createTempN5Writer()) { + final String datasetName = "testShardDelete"; + + // Create a sharded dataset + final DatasetAttributes datasetAttributes = getTestAttributes3d(); + int[] blockSize = datasetAttributes.getBlockSize(); + writer.createDataset(datasetName, datasetAttributes); + + final int blockNumElements = Arrays.stream(blockSize).reduce(1, (x,y) -> x*y); + final byte[] byteBlock = new byte[blockNumElements]; + rnd.nextBytes(byteBlock); + + // Create blocks in different shards + final long[] shardPosition1 = {0, 0, 0}; + final long[] shardPosition2 = {0, 0, 1}; // Different shard in z dimension + final long[] shardPositionNonExistent = {0, 0, 2}; // Different shard in z dimension + + // Create blocks within the first shard + final long[] blockPosition1 = {0, 0, 0}; + final long[] blockPosition2 = {1, 0, 0}; + final long[] blockPosition3 = {0, 1, 0}; + + // Create a block in the second shard + final long[] blockPosition4 = {0, 0, 2}; // This will be in + // shardPosition2 + + final ByteArrayDataBlock block1 = new ByteArrayDataBlock(blockSize, blockPosition1, byteBlock); + final ByteArrayDataBlock block2 = new ByteArrayDataBlock(blockSize, blockPosition2, byteBlock); + final ByteArrayDataBlock block3 = new ByteArrayDataBlock(blockSize, blockPosition3, byteBlock); + final ByteArrayDataBlock block4 = new ByteArrayDataBlock(blockSize, blockPosition4, byteBlock); + + // Write blocks to create shards + writer.writeBlocks(datasetName, datasetAttributes, block1, block2, block3, block4); + + // Verify shards exist + final Shard shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); + final Shard shard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); + final Shard shardDNE = writer.readShard(datasetName, datasetAttributes, shardPositionNonExistent); + assertNotNull("Shard 1 should exist", shard1); + assertNotNull("Shard 2 should exist", shard2); + assertNull("Shard 3 should not exist", shardDNE); + assertEquals("Shard 1 should contain 3 blocks", 3, shard1.getBlocks().size()); + assertEquals("Shard 2 should contain 1 block", 1, shard2.getBlocks().size()); + + // Test deleteShard + boolean deleted1 = writer.deleteShard(datasetName, shardPosition1); + assertTrue("deleteShard should return true when shard exists", deleted1); + + // Verify shard is deleted + final Shard deletedShard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); + assertNull("Shard 1 should be deleted", deletedShard1); + + // Verify blocks in the deleted shard are gone + assertNull("Block 1 should be deleted", writer.readBlock(datasetName, datasetAttributes, blockPosition1)); + assertNull("Block 2 should be deleted", writer.readBlock(datasetName, datasetAttributes, blockPosition2)); + assertNull("Block 3 should be deleted", writer.readBlock(datasetName, datasetAttributes, blockPosition3)); + + // Verify other shard is unaffected + final Shard stillExistingShard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); + assertNotNull("Shard 2 should still exist", stillExistingShard2); + assertNotNull("Block 4 should still exist", writer.readBlock(datasetName, datasetAttributes, blockPosition4)); + + // Test deleting non-existent shard + boolean deletedAgain = writer.deleteShard(datasetName, shardPosition1); + assertFalse("deleteShard should return false when shard doesn't exist", deletedAgain); + + // Test deleting shard at invalid position + final long[] invalidShardPosition = {100, 100, 100}; + boolean deletedInvalid = writer.deleteShard(datasetName, invalidShardPosition); + assertFalse("deleteShard should return false for non-existent shard position", deletedInvalid); + } + } + + @SuppressWarnings("unchecked") + @Test + public void testShardedBlockDelete() { + + final Random rnd = new Random(88); + try (GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)tempN5Factory.createTempN5Writer()) { + final String datasetName = "testShardBlockDelete"; + + // Create a sharded dataset + final DatasetAttributes datasetAttributes = getTestAttributes3d(); + int[] blockSize = datasetAttributes.getBlockSize(); + writer.createDataset(datasetName, datasetAttributes); + + final int blockNumElements = Arrays.stream(blockSize).reduce(1, (x,y) -> x*y); + final byte[] byteBlock = new byte[blockNumElements]; + rnd.nextBytes(byteBlock); + + // Create blocks in different shards + final long[] shardPosition1 = {0, 0, 0}; + final long[] shardPosition2 = {0, 0, 1}; // Different shard in z dimension + final long[] shardPositionNonExistent = {0, 0, 2}; // Different shard in z dimension + + // Create blocks within the first shard + final long[] blockPosition1 = {0, 0, 0}; + final long[] blockPosition2 = {1, 0, 0}; + final long[] blockPosition3 = {0, 1, 0}; + + // Create a block in the second shard + final long[] blockPosition4 = {0, 0, 2}; // This will be in shardPosition2 + + final ByteArrayDataBlock block1 = new ByteArrayDataBlock(blockSize, blockPosition1, byteBlock); + final ByteArrayDataBlock block2 = new ByteArrayDataBlock(blockSize, blockPosition2, byteBlock); + final ByteArrayDataBlock block3 = new ByteArrayDataBlock(blockSize, blockPosition3, byteBlock); + final ByteArrayDataBlock block4 = new ByteArrayDataBlock(blockSize, blockPosition4, byteBlock); + + // Write blocks to create shards + writer.writeBlocks(datasetName, datasetAttributes, block1, block2, block3, block4); + + // Verify shards exist + Shard shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); + Shard shard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); + Shard shardDNE = writer.readShard(datasetName, datasetAttributes, shardPositionNonExistent); + assertNotNull("Shard 1 should exist", shard1); + assertNotNull("Shard 2 should exist", shard2); + assertNull("Shard 3 should not exist", shardDNE); + assertEquals("Shard 1 should contain 3 blocks", 3, shard1.getBlocks().size()); + assertEquals("Shard 2 should contain 1 block", 1, shard2.getBlocks().size()); + + // Test delete one block from a multi-block shard + boolean deleted1 = writer.deleteBlock(datasetName, blockPosition1); + assertTrue("deleteBlock1", deleted1); + shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); + assertNotNull("Shard 1 should still exist", shard1); + DataBlock blk1Read = writer.readBlock(datasetName, datasetAttributes, blockPosition1); + DataBlock blk2Read = writer.readBlock(datasetName, datasetAttributes, blockPosition2); + DataBlock blk3Read = writer.readBlock(datasetName, datasetAttributes, blockPosition3); + assertNull("Block 1 should not exist", blk1Read); + assertNotNull("Block 2 should exist", blk2Read); + assertNotNull("Block 3 should exist", blk3Read); + + // Test delete one block from a multi-block shard + boolean deleted2 = writer.deleteBlock(datasetName, blockPosition2); + assertTrue("deleteBlock2", deleted2); + shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); + assertNotNull("Shard 1 should still exist", shard1); + blk2Read = writer.readBlock(datasetName, datasetAttributes, blockPosition2); + blk3Read = writer.readBlock(datasetName, datasetAttributes, blockPosition3); + assertNull("Block 2 should not exist", blk2Read); + assertNotNull("Block 3 should exist", blk3Read); + + // Test delete last block from a multi-block shard + boolean deleted3 = writer.deleteBlock(datasetName, blockPosition3); + assertTrue("deleteBlock3", deleted3); + shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); + assertNull("Shard 1 should not exist", shard1); + blk3Read = writer.readBlock(datasetName, datasetAttributes, blockPosition3); + assertNull("Block 3 should not exist", blk3Read); + + // Test delete last block from a multi-block shard + boolean deleted4 = writer.deleteBlock(datasetName, blockPosition4); + assertTrue("deleteBlock4", deleted4); + shard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); + assertNull("Shard 2 should not exist", shard2); + DataBlock blk4Read = writer.readBlock(datasetName, datasetAttributes, blockPosition4); + assertNull("Block 2 should not exist", blk4Read); + } + } + @Test @Ignore("Nested sharding not supported ") public void writeReadNestedShards() { @@ -333,4 +558,120 @@ private DatasetAttributes getNestedShardCodecsAttributes(int[] blockSize) { ); } + /** + * An N5Writer that serializing the sharding codecs, enabling testing of + * shard functionality, despite the fact that the N5 format does not support + * sharding. + */ + public static class ShardedN5Writer extends N5FSWriter { + + Gson gson; + + TestDatasetAttributesAdapter adapter = new TestDatasetAttributesAdapter(); + + public ShardedN5Writer(String basePath) { + + this(basePath, new GsonBuilder()); + } + + public ShardedN5Writer(String basePath, GsonBuilder gsonBuilder) { + + super(basePath); + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); + gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, new TestDatasetAttributesAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBytes.byteOrderAdapter); + gsonBuilder.registerTypeHierarchyAdapter(ShardingCodec.IndexLocation.class, ShardingCodec.indexLocationAdapter); + gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.disableHtmlEscaping(); + gson = gsonBuilder.create(); + } + + @Override + public Gson getGson() { + + // the super constructor needs the gson instance, unfortunately + return gson == null ? super.gson : gson; + } + + @Override + public DatasetAttributes createDatasetAttributes(final JsonElement attributes) { + + final JsonDeserializationContext context = new JsonDeserializationContext() { + + @Override + public T deserialize(JsonElement json, Type typeOfT) throws JsonParseException { + + return getGson().fromJson(json, typeOfT); + } + }; + + return adapter.deserialize(attributes, DatasetAttributes.class, context); + } + } + + public static class TestDatasetAttributesAdapter extends DatasetAttributesAdapter { + + @Override + public DatasetAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { + + if (json == null || !json.isJsonObject()) + return null; + final JsonObject obj = json.getAsJsonObject(); + final boolean validKeySet = obj.has(DatasetAttributes.DIMENSIONS_KEY) + && obj.has(DatasetAttributes.BLOCK_SIZE_KEY) + && obj.has(DatasetAttributes.DATA_TYPE_KEY) + && (obj.has(DatasetAttributes.CODEC_KEY) || obj.has(DatasetAttributes.COMPRESSION_KEY)); + + if (!validKeySet) + return null; + + final long[] dimensions = context.deserialize(obj.get(DatasetAttributes.DIMENSIONS_KEY), long[].class); + final int[] blockSize = context.deserialize(obj.get(DatasetAttributes.BLOCK_SIZE_KEY), int[].class); + final int[] shardSize = context.deserialize(obj.get(DatasetAttributes.SHARD_SIZE_KEY), int[].class); + + final DataType dataType = context.deserialize(obj.get(DatasetAttributes.DATA_TYPE_KEY), DataType.class); + + final Codec[] codecs; + if (obj.has(DatasetAttributes.CODEC_KEY)) { + codecs = context.deserialize(obj.get(DatasetAttributes.CODEC_KEY), Codec[].class); + } else if (obj.has(DatasetAttributes.COMPRESSION_KEY)) { + final Compression compression = CompressionAdapter.getJsonAdapter().deserialize(obj.get(DatasetAttributes.COMPRESSION_KEY), Compression.class, + context); + codecs = new Codec[]{compression}; + } else { + return null; + } + return new DatasetAttributes(dimensions, shardSize, blockSize, dataType, codecs); + } + + @Override + public JsonElement serialize(DatasetAttributes src, Type typeOfSrc, JsonSerializationContext context) { + + final JsonObject obj = new JsonObject(); + obj.add(DatasetAttributes.DIMENSIONS_KEY, context.serialize(src.getDimensions())); + obj.add(DatasetAttributes.BLOCK_SIZE_KEY, context.serialize(src.getBlockSize())); + + final int[] shardSize = src.getShardSize(); + if (shardSize != null) { + obj.add(DatasetAttributes.SHARD_SIZE_KEY, context.serialize(shardSize)); + } + + obj.add(DatasetAttributes.DATA_TYPE_KEY, context.serialize(src.getDataType())); + obj.add(DatasetAttributes.CODEC_KEY, context.serialize(concatenateCodecs(src))); + return obj; + } + } + + private static Codec[] concatenateCodecs(final DatasetAttributes attributes) { + + final BytesCodec[] byteCodecs = attributes.getCodecs(); + final ArrayCodec arrayCodec = attributes.getArrayCodec(); + final Codec[] allCodecs = new Codec[byteCodecs.length + 1]; + allCodecs[0] = arrayCodec; + System.arraycopy(byteCodecs, 0, allCodecs, 1, byteCodecs.length); + + return allCodecs; + } + } From 1f93f685517069f6c189cde58fe181ed09c723c5 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 8 Jul 2025 13:24:15 -0400 Subject: [PATCH 264/423] fix: ShardingCodec.getCodecs --- .../java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index f6a5f12b3..f98359ac2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -97,7 +97,7 @@ public BytesCodec[] getCodecs() { Objects.requireNonNull(codecs); final BytesCodec[] bytesCodecs = new BytesCodec[codecs.length - 1]; for (int i = 1; i < codecs.length; i++) - bytesCodecs[i] = (BytesCodec)codecs[i]; + bytesCodecs[i-1] = (BytesCodec)codecs[i]; return bytesCodecs; } From 939cb77e27738b0689634e1e0554cb04a795914f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 8 Jul 2025 13:24:38 -0400 Subject: [PATCH 265/423] fix: InMemoryShard.fromShard returns null for null input --- .../java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index 10f6adb84..5741e6693 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -70,6 +70,9 @@ public ShardIndex getIndex() { public static InMemoryShard fromShard(Shard shard) { + if (shard == null) + return null; + if (shard instanceof InMemoryShard) return (InMemoryShard)shard; From 6ed67a8a67baee5b1e1a72c4bb410ddc8dae40d3 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 8 Jul 2025 13:24:56 -0400 Subject: [PATCH 266/423] test: TestN5ShardWriter should not register Compression adapter --- src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 7ba3e8be5..cc767f89c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -582,7 +582,6 @@ public ShardedN5Writer(String basePath, GsonBuilder gsonBuilder) { gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, new TestDatasetAttributesAdapter()); gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBytes.byteOrderAdapter); gsonBuilder.registerTypeHierarchyAdapter(ShardingCodec.IndexLocation.class, ShardingCodec.indexLocationAdapter); - gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.disableHtmlEscaping(); gson = gsonBuilder.create(); } From a9e982205b3f28ba5648605e0d07e9e714d4f9aa Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 9 Jul 2025 16:02:53 -0400 Subject: [PATCH 267/423] feat: add GridIterator.positionToIndex * useful in n5-imglib2 --- .../saalfeldlab/n5/util/GridIterator.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java index 283cfc24e..43258efaf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java @@ -107,6 +107,46 @@ public static void indexToPosition(long index, final int[] dimensions, final lon } } + final static public long positionToIndex(final long[] dimensions, final long[] position) { + long idx = 0; + int cumulativeSize = 1; + for (int i = 0; i < position.length; i++) { + idx += position[i] * cumulativeSize; + cumulativeSize *= dimensions[i]; + } + return idx; + } + + final static public long positionToIndex(final long[] dimensions, final int[] position) { + long idx = 0; + int cumulativeSize = 1; + for (int i = 0; i < position.length; i++) { + idx += position[i] * cumulativeSize; + cumulativeSize *= dimensions[i]; + } + return idx; + } + + final static public long positionToIndex(final int[] dimensions, final long[] position) { + long idx = 0; + int cumulativeSize = 1; + for (int i = 0; i < position.length; i++) { + idx += position[i] * cumulativeSize; + cumulativeSize *= dimensions[i]; + } + return idx; + } + + final static public long positionToIndex(final int[] dimensions, final int[] position) { + long idx = 0; + int cumulativeSize = 1; + for (int i = 0; i < position.length; i++) { + idx += position[i] * cumulativeSize; + cumulativeSize *= dimensions[i]; + } + return idx; + } + public static int[] long2int(final long[] src, final int[] tgt) { for (int d = 0; d < tgt.length; ++d) From 75772b8720a76595dc6f5fdaf39e67b78cf19c78 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 20 Aug 2025 15:36:28 -0400 Subject: [PATCH 268/423] feat: DatasetAttributes.getCodecs * downstream implementations (zarr) need access to the bytesCodecs --- .../java/org/janelia/saalfeldlab/n5/DatasetAttributes.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 181efdf7b..f32bf6b7a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -32,7 +32,6 @@ import java.util.Arrays; import java.util.HashMap; -import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.ArrayCodec; import org.janelia.saalfeldlab.n5.codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.DataBlockSerializer; @@ -140,6 +139,11 @@ public ArrayCodec getArrayCodec() { return arrayCodec; } + public BytesCodec[] getCodecs() { + + return byteCodecs; + } + @SuppressWarnings("unchecked") DataBlockSerializer getDataBlockSerializer() { return (DataBlockSerializer) dataBlockSerializer; From 7915ed364405b4e8db79d4fb58d01ce827d7884b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 20 Aug 2025 15:37:46 -0400 Subject: [PATCH 269/423] feat: getAttributesKey method * enables zarr to re-use the implementation here re: attributes --- .../saalfeldlab/n5/CachedGsonKeyValueN5Reader.java | 12 ++++++------ .../saalfeldlab/n5/CachedGsonKeyValueN5Writer.java | 8 ++++---- .../janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java | 2 +- .../org/janelia/saalfeldlab/n5/GsonN5Reader.java | 7 +++++++ .../org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 6 ++++++ .../saalfeldlab/n5/http/HttpReaderFsWriter.java | 5 +++++ 6 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index 6f7e7c0a8..3f0db47bf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -73,7 +73,7 @@ default DatasetAttributes getDatasetAttributes(final String pathName) { return null; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPath, N5KeyValueReader.ATTRIBUTES_JSON); + attributes = getCache().getAttributes(normalPath, getAttributesKey()); } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPath); } @@ -99,7 +99,7 @@ default T getAttribute( final JsonElement attributes; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + attributes = getCache().getAttributes(normalPathName, getAttributesKey()); } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName); } @@ -120,7 +120,7 @@ default T getAttribute( final String normalizedAttributePath = N5URI.normalizeAttributePath(key); JsonElement attributes; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + attributes = getCache().getAttributes(normalPathName, getAttributesKey()); } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName); } @@ -136,7 +136,7 @@ default boolean exists(final String pathName) { final String normalPathName = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) - return getCache().isGroup(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + return getCache().isGroup(normalPathName, getAttributesKey()); else { return existsFromContainer(normalPathName, null); } @@ -180,7 +180,7 @@ default boolean datasetExists(final String pathName) throws N5IOException { final String normalPathName = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) { - return getCache().isDataset(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + return getCache().isDataset(normalPathName, getAttributesKey()); } return isDatasetFromContainer(normalPathName); } @@ -212,7 +212,7 @@ default JsonElement getAttributes(final String pathName) throws N5IOException { /* If cached, return the cache */ if (cacheMeta()) { - return getCache().getAttributes(groupPath, N5KeyValueReader.ATTRIBUTES_JSON); + return getCache().getAttributes(groupPath, getAttributesKey()); } else { return GsonKeyValueN5Reader.super.getAttributes(groupPath); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index 5e6c85d15..e327463c0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -62,9 +62,9 @@ default void createGroup(final String path) throws N5Exception { // else if exists is true (then a dataset is present) so throw an exception to avoid // overwriting / invalidating existing data if (cacheMeta()) { - if (getCache().isGroup(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) + if (getCache().isGroup(normalPath, getAttributesKey())) return; - else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { + else if (getCache().exists(normalPath, getAttributesKey())) { throw new N5Exception("Can't make a group on existing path."); } } @@ -88,8 +88,8 @@ else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { for (final String child : pathParts) { final String childPath = parent.isEmpty() ? child : parent + "/" + child; - getCache().initializeNonemptyCache(childPath, N5KeyValueReader.ATTRIBUTES_JSON); - getCache().updateCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON); + getCache().initializeNonemptyCache(childPath, getAttributesKey()); + getCache().updateCacheInfo(childPath, getAttributesKey()); // only add if the parent exists and has children cached already if (parent != null && !child.isEmpty()) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index a742cc299..e19fecb98 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -165,6 +165,6 @@ default String absoluteGroupPath(final String normalGroupPath) { */ default String absoluteAttributesPath(final String normalPath) { - return getKeyValueAccess().compose(getURI(), normalPath, N5KeyValueReader.ATTRIBUTES_JSON); + return getKeyValueAccess().compose(getURI(), normalPath, getAttributesKey()); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index a0aab927a..d678a26f2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -43,6 +43,13 @@ public interface GsonN5Reader extends N5Reader { Gson getGson(); + /** + * Get the key for the {@link KeyValueAccess}, that is used for storing attributes. + * + * @return the attributes key + */ + String getAttributesKey(); + @Override default Map> listAttributes(final String pathName) throws N5Exception { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index e86b712ae..22731edd0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -165,6 +165,12 @@ public Gson getGson() { return gson; } + @Override + public String getAttributesKey() { + + return ATTRIBUTES_JSON; + } + @Override public KeyValueAccess getKeyValueAccess() { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index 68edb9d3f..52c560fff 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -273,4 +273,9 @@ public HttpRead writer.writeSerializedBlock(object, datasetPath, datasetAttributes, gridPosition); } + + @Override + public String getAttributesKey() { + return writer.getAttributesKey(); + } } From 51e510ddf9d77f04d6cedc898fcbf1e9a93a99ae Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 26 Aug 2025 16:47:33 -0400 Subject: [PATCH 270/423] fix: remove attribute should use setAttributes * not writeAttributes * this change allows zarr v3 to re-use this method --- .../java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index f7bc044a6..ba79c847b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -218,7 +218,7 @@ default T removeAttribute(final String pathName, final String key, final Cla throw new N5Exception.N5ClassCastException(e); } if (obj != null) { - writeAttributes(normalPath, attributes); + setAttributes(normalPath, attributes); } return obj; } From bce0138f64cd4fe495d05326b6d5d933acbdb03a Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 26 Aug 2025 16:49:57 -0400 Subject: [PATCH 271/423] RawBytes type should be "bytes" * so that it can be used for zarr v3 --- .../org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java index ca34e7a9e..47486058c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java @@ -11,7 +11,7 @@ public class RawBytesArrayCodec implements ArrayCodec { private static final long serialVersionUID = 3282569607795127005L; - public static final String TYPE = "rawbytes"; + public static final String TYPE = "bytes"; @NameConfig.Parameter(value = "endian", optional = true) private final ByteOrder byteOrder; From 7b9dee43f54c5941eb2805f00ff5134f3f48fc60 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 26 Aug 2025 16:50:44 -0400 Subject: [PATCH 272/423] feat: make registerGson an instance method * so that it can be overrided by ZarrV3Reader * remove static registerGson in GsonUtils --- .../java/org/janelia/saalfeldlab/n5/GsonUtils.java | 9 --------- .../org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 12 +++++++++++- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index e017a7c25..a683cafde 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -80,15 +80,6 @@ */ public interface GsonUtils { - static Gson registerGson(final GsonBuilder gsonBuilder) { - - gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); - gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); - gsonBuilder.disableHtmlEscaping(); - return gsonBuilder.create(); - } - /** * Reads the attributes json from a given {@link Reader}. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 22731edd0..b1bc01d36 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -33,6 +33,7 @@ import com.google.gson.JsonElement; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; +import org.janelia.saalfeldlab.n5.codec.Codec; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -127,7 +128,7 @@ protected N5KeyValueReader( throws N5Exception { this.keyValueAccess = keyValueAccess; - this.gson = GsonUtils.registerGson(gsonBuilder); + this.gson = registerGson(gsonBuilder).create(); this.cacheMeta = cacheMeta; this.cache = newCache(); @@ -159,6 +160,15 @@ private boolean inferExistence(String path) { return attributes != null || exists(path); } + protected GsonBuilder registerGson(final GsonBuilder gsonBuilder) { + + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); + gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.disableHtmlEscaping(); + return gsonBuilder; + } + @Override public Gson getGson() { From d78f161858986d5b30acd4aa08b312da7528b477 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 27 Aug 2025 11:58:14 +0200 Subject: [PATCH 273/423] temp --- pom.xml | 2 ++ .../java/org/janelia/saalfeldlab/n5/DatasetAttributes.java | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index 89858106c..a0660d188 100644 --- a/pom.xml +++ b/pom.xml @@ -146,6 +146,8 @@ sign,deploy-to-scijava 4.0.0 + + 24 diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 1127c658a..afa87ab86 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -156,6 +156,11 @@ public HashMap asMap() { return map; } + + record Coord(int[] pos, Coord nested) { + + } + static DatasetAttributes from( final long[] dimensions, final DataType dataType, From 609e4f2c0fa8f87fe796ceb9d832f598fdb01d93 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 27 Aug 2025 17:35:42 +0200 Subject: [PATCH 274/423] WIP --- .../saalfeldlab/n5/DatasetAttributes.java | 5 - .../janelia/saalfeldlab/n5/ShardStuff.java | 123 ++++++++++++++++++ 2 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index afa87ab86..1127c658a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -156,11 +156,6 @@ public HashMap asMap() { return map; } - - record Coord(int[] pos, Coord nested) { - - } - static DatasetAttributes from( final long[] dimensions, final DataType dataType, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java new file mode 100644 index 000000000..21789b653 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java @@ -0,0 +1,123 @@ +package org.janelia.saalfeldlab.n5; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +public class ShardStuff { + + public interface Block { + + /** + * Returns the number of elements in a box of given size. + * + * @param size + * the size + * + * @return the number of elements + */ + static int getNumElements(final int[] size) { + + int n = size[0]; + for (int i = 1; i < size.length; ++i) + n *= size[i]; + return n; + } + + /** + * Returns the size of this block. + *

      + * The size of a data block is expected to be smaller than or equal to the + * spacing of the block grid. The dimensionality of size is expected to be + * equal to the dimensionality of the dataset. Consistency is not enforced. + * + * @return size of the block + */ + int[] getSize(); + + /** + * Returns the grid position of this block. + *

      + * Grid position is with respect to the level of the block (Shard, nested Shard, DataBlock, ...). + * If a block has grid position (i, ...) the adjacent block (in X) has position (i+1, ...). + * Grid position (0, ...) is the origin of the image, that is, grid positions are not relative to the containing shard etc. + *

      + * The dimensionality of the grid position is expected to be equal to the + * dimensionality of the dataset. Consistency is not enforced. + * + * @return position on the block grid + */ + long[] getGridPosition(); + } + + + public interface Shard extends Block { + + // pos is relative to this shard + ReadData getElementData(int[] pos); + + // pos is relative to this shard + void setElementData(ReadData data, int[] pos); + } + + + public interface DataBlock extends Block { + + /** + * Returns the data object held by this data block. + * + * @return data object + */ + T getData(); + +// interface DataBlockFactory { +// DataBlock createDataBlock(int[] blockSize, long[] gridPosition, T data); +// } + } + + + public interface BlockCodec { + + ReadData encode(T block) throws N5IOException; + + T decode(ReadData readData, long[] gridPosition) throws N5IOException; + } + + + public interface DataBlockCodec extends BlockCodec> { + } + + + static class N5ReaderImpl { + + /** + * Reads a {@link DataBlock}. + * + * @param pathName + * dataset path + * @param datasetAttributes + * the dataset attributes + * @param gridPosition + * the grid position + * + * @return the data block + * + * @throws N5Exception + * the exception + */ + DataBlock readBlock( + final String pathName, + final DatasetAttributes datasetAttributes, + final long... gridPosition) throws N5Exception { + + + + + + + + + throw new UnsupportedOperationException(); + } + } + +} From 4e3d353c3201bff2aea68a3a0584728fc34331c2 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 28 Aug 2025 11:24:02 +0200 Subject: [PATCH 275/423] Remove superfluous modifiers --- .../java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java index ef5f3825a..cfeada9c8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java @@ -13,7 +13,7 @@ */ public interface BlockCodecInfo extends CodecInfo { - BlockCodec create(final DatasetAttributes attributes, final DataCodec... codecs); + BlockCodec create(DatasetAttributes attributes, DataCodec... codecs); default BlockCodec create(final DatasetAttributes attributes, final DataCodecInfo... codecInfos) { final DataCodec[] codecs = new DataCodec[codecInfos.length]; From 2f2e2fe0a0c46fb56d4e0fd7eeac34d865f112ce Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 28 Aug 2025 11:24:29 +0200 Subject: [PATCH 276/423] WIP BlockCodecInfo --- .../janelia/saalfeldlab/n5/ShardStuff.java | 86 ++++++++++++++++++- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java index 21789b653..d7773176d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java @@ -1,6 +1,12 @@ package org.janelia.saalfeldlab.n5; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodec; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.readdata.ReadData; public class ShardStuff { @@ -75,17 +81,91 @@ public interface DataBlock extends Block { } - public interface BlockCodec { + public interface BlockCodec { - ReadData encode(T block) throws N5IOException; + ReadData encode(B block) throws N5IOException; - T decode(ReadData readData, long[] gridPosition) throws N5IOException; + B decode(ReadData readData, long[] gridPosition) throws N5IOException; } public interface DataBlockCodec extends BlockCodec> { } + public static class BlockCodecs { + + private final List> shardCodecs = new ArrayList<>(); + private final DataBlockCodec dataBlockCodec; + private final int[] dataBlockSize; + + public BlockCodecs(final DataBlockCodec dataBlockCodec, final int[] dataBlockSize) { + this.dataBlockCodec = dataBlockCodec; + this.dataBlockSize = dataBlockSize; + } + + public BlockCodecs(final BlockCodec first, final BlockCodecs others) { + shardCodecs.add(first); + shardCodecs.addAll(others.shardCodecs); + dataBlockCodec = others.dataBlockCodec; + dataBlockSize = others.dataBlockSize; + } + + public int[] getDataBlockSize() { + return dataBlockSize; + } + } + + public interface BlockCodecInfo extends CodecInfo { + + /** + * Chunk size of the elements in this block. + * That is, (1, ...) for DataBlockCodecInfo, respectively inner block size for ShardCodecInfo. + */ + int[] getChunkSize(); + + /** + * Nested BlockCodec. + * ({@code null} for DataBlockCodec. + */ + BlockCodecInfo getInnerBlockCodecInfo(); + + DataCodecInfo[] getInnerDataCodecInfos(); + + BlockCodec create(int[] blockSize, DataCodecInfo... codecs); + + default BlockCodecs createRecursive(DataType dataType, int[] blockSize, DataCodecInfo... codecs) + { + final BlockCodecs blockCodecs = getInnerBlockCodecInfo().createRecursive( + dataType, getChunkSize(), + getInnerDataCodecInfos()); + final BlockCodec shardCodec = create(blockSize, codecs); + return new BlockCodecs<>(shardCodec, blockCodecs); + } + } + + private static DataCodec[] instantiate(final DataCodecInfo... codecInfos) { + final DataCodec[] codecs = new DataCodec[codecInfos.length]; + Arrays.setAll(codecs, i -> codecInfos[i].create()); + return codecs; + } + + public interface DataBlockCodecInfo extends BlockCodecInfo { + + @Override + default BlockCodec create(int[] blockSize, DataCodecInfo... codecs) { + // TODO: Fix BlockCodec hierarchy. It's a bad sign that we have to + // @Override this here. Maybe AbstractBlockCodecInfo extended by + // (Data)BlockCodecInfo and ShardCodecInfo + throw new UnsupportedOperationException(); + } + + DataBlockCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs); + + @Override + default BlockCodecs createRecursive(DataType dataType, int[] blockSize, DataCodecInfo... codecs) { + return new BlockCodecs<>(create(dataType, blockSize, codecs), blockSize); + } + } static class N5ReaderImpl { From 0f84cb5bb4842c0b5aa53c6cc6611e5f7f85441e Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 28 Aug 2025 13:39:36 +0200 Subject: [PATCH 277/423] WIP --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 2 + .../janelia/saalfeldlab/n5/ShardStuff.java | 170 +++++++++++++----- 2 files changed, 125 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index e30c7f88e..714b86d37 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -129,6 +129,8 @@ default String[] list(final String pathName) throws N5Exception { * @param gridPosition to the target data block * @return the absolute path to the data block ad gridPosition */ + // TODO: revise javadoc -> see development branch + // TODO: rename to reflect that it may also refer to Shards default String absoluteDataBlockPath( final String normalPath, final long... gridPosition) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java index d7773176d..ba3aaa88c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java @@ -1,5 +1,7 @@ package org.janelia.saalfeldlab.n5; +import com.google.gson.Gson; +import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -11,6 +13,13 @@ public class ShardStuff { + private static DataCodec[] instantiate(final DataCodecInfo... codecInfos) { + final DataCodec[] codecs = new DataCodec[codecInfos.length]; + Arrays.setAll(codecs, i -> codecInfos[i].create()); + return codecs; + } + + public interface Block { /** @@ -65,7 +74,6 @@ public interface Shard extends Block { void setElementData(ReadData data, int[] pos); } - public interface DataBlock extends Block { /** @@ -80,7 +88,6 @@ public interface DataBlock extends Block { // } } - public interface BlockCodec { ReadData encode(B block) throws N5IOException; @@ -88,34 +95,18 @@ public interface BlockCodec { B decode(ReadData readData, long[] gridPosition) throws N5IOException; } + public interface ShardCodec extends BlockCodec { + } public interface DataBlockCodec extends BlockCodec> { } - public static class BlockCodecs { - - private final List> shardCodecs = new ArrayList<>(); - private final DataBlockCodec dataBlockCodec; - private final int[] dataBlockSize; - - public BlockCodecs(final DataBlockCodec dataBlockCodec, final int[] dataBlockSize) { - this.dataBlockCodec = dataBlockCodec; - this.dataBlockSize = dataBlockSize; - } - - public BlockCodecs(final BlockCodec first, final BlockCodecs others) { - shardCodecs.add(first); - shardCodecs.addAll(others.shardCodecs); - dataBlockCodec = others.dataBlockCodec; - dataBlockSize = others.dataBlockSize; - } + public interface BlockCodecInfo extends CodecInfo { - public int[] getDataBlockSize() { - return dataBlockSize; - } + BlockCodecs createRecursive(DataType dataType, int[] blockSize, DataCodecInfo... codecs); } - public interface BlockCodecInfo extends CodecInfo { + public interface ShardCodecInfo extends BlockCodecInfo { /** * Chunk size of the elements in this block. @@ -133,32 +124,18 @@ public interface BlockCodecInfo extends CodecInfo { BlockCodec create(int[] blockSize, DataCodecInfo... codecs); - default BlockCodecs createRecursive(DataType dataType, int[] blockSize, DataCodecInfo... codecs) - { + @Override + default BlockCodecs createRecursive(DataType dataType, int[] blockSize, DataCodecInfo... codecs) { + final BlockCodec shardCodec = create(blockSize, codecs); final BlockCodecs blockCodecs = getInnerBlockCodecInfo().createRecursive( dataType, getChunkSize(), getInnerDataCodecInfos()); - final BlockCodec shardCodec = create(blockSize, codecs); - return new BlockCodecs<>(shardCodec, blockCodecs); + return new BlockCodecs<>(shardCodec, blockSize, blockCodecs); } } - private static DataCodec[] instantiate(final DataCodecInfo... codecInfos) { - final DataCodec[] codecs = new DataCodec[codecInfos.length]; - Arrays.setAll(codecs, i -> codecInfos[i].create()); - return codecs; - } - public interface DataBlockCodecInfo extends BlockCodecInfo { - @Override - default BlockCodec create(int[] blockSize, DataCodecInfo... codecs) { - // TODO: Fix BlockCodec hierarchy. It's a bad sign that we have to - // @Override this here. Maybe AbstractBlockCodecInfo extended by - // (Data)BlockCodecInfo and ShardCodecInfo - throw new UnsupportedOperationException(); - } - DataBlockCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs); @Override @@ -167,7 +144,100 @@ default BlockCodecs createRecursive(DataType dataType, int[] blockSize, D } } - static class N5ReaderImpl { + + public static class BlockCodecs { + + private final List shardCodecs = new ArrayList<>(); + + private final List chunkSizesInDataBlocks = new ArrayList<>(); + + private final DataBlockCodec dataBlockCodec; + + private final int[] dataBlockSize; + + public BlockCodecs(final DataBlockCodec dataBlockCodec, final int[] dataBlockSize) { + this.dataBlockCodec = dataBlockCodec; + this.dataBlockSize = dataBlockSize; + } + + public BlockCodecs(final ShardCodec codec, final int[] shardSize, final BlockCodecs others) { + this(others.dataBlockCodec, others.dataBlockSize); + shardCodecs.add(codec); + shardCodecs.addAll(others.shardCodecs); + + final int[] gridSize = new int[shardSize.length]; + Arrays.setAll(gridSize, d -> shardSize[d] / dataBlockSize[d]); + chunkSizesInDataBlocks.add(gridSize); + // TODO: verify shardSize is integer multiple of dataBlockSize. + // TODO: verify size is integer multiple of nested chunkSizeInDataBlocks. + + } + + public int[] getDataBlockSize() { + return dataBlockSize; + } + + public NestedBlockPosition getNestedDataBlockPosition(final long[] gridPos) { + if (shardCodecs.isEmpty()) { + return new NestedBlockPosition(new long[][] {gridPos}); + } + + final int n = gridPos.length; + final int m = shardCodecs.size() + 1; + + final long[][] nested = new long[m][n]; + final long[] pos = nested[m - 1]; + Arrays.setAll(pos, d -> gridPos[d]); + for (int i = 0; i < m - 1; ++i) { + final int[] gridSize = chunkSizesInDataBlocks.get(i); + for (int d = 0; d < n; ++d) { + nested[i][d] = pos[d] / gridSize[d]; + pos[d] = pos[d] % gridSize[d]; + } + } + + return new NestedBlockPosition(nested); + } + + public DataBlock decodeDataBlock(final ReadData keyData, final NestedBlockPosition pos) { + + + + + + throw new UnsupportedOperationException(); + } + } + + // TODO: Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...). + // For nested = {X,Y,Z} compare by X, then Y, then Z. + // For X = {x,y,z} compare by z, then y, then x. (flattening order) + public static class NestedBlockPosition { + + private final long[][] nested; + + NestedBlockPosition(final long[][] nested) { + // TODO: validation: nested != null, at least one level + this.nested = nested; + } + + public int depth() { + return nested.length; + } + + public long[] relativePosition(final int level) { + return nested[level]; + } + + public long[] keyPosition() { + return relativePosition(0); + } + } + + + // DUMMY + static abstract class N5ReaderImpl implements GsonKeyValueN5Reader { + /** * Reads a {@link DataBlock}. @@ -187,17 +257,23 @@ static class N5ReaderImpl { DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, + final BlockCodecs blockCodecs, // TODO: get from DatasetAttributes final long... gridPosition) throws N5Exception { + final NestedBlockPosition pos = blockCodecs.getNestedDataBlockPosition(gridPosition); + final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), pos.keyPosition()); - - - - - + try { + final ReadData keyData = getKeyValueAccess().createReadData(path); + return blockCodecs.decodeDataBlock(keyData, pos); + } catch (N5Exception.N5NoSuchKeyException e) { + return null; + } throw new UnsupportedOperationException(); } + + } } From f8abd61922874d32ccd50f9b0f1b664f3949d11e Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 28 Aug 2025 16:59:37 +0200 Subject: [PATCH 278/423] WIP --- .../janelia/saalfeldlab/n5/ShardStuff.java | 112 +++++++++++++++--- 1 file changed, 93 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java index ba3aaa88c..83c7fdc3c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java @@ -13,13 +13,6 @@ public class ShardStuff { - private static DataCodec[] instantiate(final DataCodecInfo... codecInfos) { - final DataCodec[] codecs = new DataCodec[codecInfos.length]; - Arrays.setAll(codecs, i -> codecInfos[i].create()); - return codecs; - } - - public interface Block { /** @@ -54,7 +47,7 @@ static int getNumElements(final int[] size) { *

      * Grid position is with respect to the level of the block (Shard, nested Shard, DataBlock, ...). * If a block has grid position (i, ...) the adjacent block (in X) has position (i+1, ...). - * Grid position (0, ...) is the origin of the image, that is, grid positions are not relative to the containing shard etc. + * Grid position (0, ...) is the origin of the dataset, that is, grid positions are not relative to the containing shard etc. *

      * The dimensionality of the grid position is expected to be equal to the * dimensionality of the dataset. Consistency is not enforced. @@ -68,12 +61,13 @@ static int getNumElements(final int[] size) { public interface Shard extends Block { // pos is relative to this shard - ReadData getElementData(int[] pos); + ReadData getElementData(long[] pos); // pos is relative to this shard - void setElementData(ReadData data, int[] pos); + void setElementData(ReadData data, long[] pos); } + public interface DataBlock extends Block { /** @@ -88,10 +82,21 @@ public interface DataBlock extends Block { // } } + + + + + + public interface BlockCodec { ReadData encode(B block) throws N5IOException; + /** + * {@code gridPosition} is with respect to the level of the block (Shard, nested Shard, DataBlock, ...). + * If a block has gridPosition (i, ...), then the adjacent block (in X) has position (i+1, ...). + * Grid position (0, ...) is the origin of the dataset, that is, grid positions are not relative to the containing shard etc. + */ B decode(ReadData readData, long[] gridPosition) throws N5IOException; } @@ -101,6 +106,11 @@ public interface ShardCodec extends BlockCodec { public interface DataBlockCodec extends BlockCodec> { } + + + + + public interface BlockCodecInfo extends CodecInfo { BlockCodecs createRecursive(DataType dataType, int[] blockSize, DataCodecInfo... codecs); @@ -122,11 +132,11 @@ public interface ShardCodecInfo extends BlockCodecInfo { DataCodecInfo[] getInnerDataCodecInfos(); - BlockCodec create(int[] blockSize, DataCodecInfo... codecs); + ShardCodec create(int[] blockSize, DataCodecInfo... codecs); @Override default BlockCodecs createRecursive(DataType dataType, int[] blockSize, DataCodecInfo... codecs) { - final BlockCodec shardCodec = create(blockSize, codecs); + final ShardCodec shardCodec = create(blockSize, codecs); final BlockCodecs blockCodecs = getInnerBlockCodecInfo().createRecursive( dataType, getChunkSize(), getInnerDataCodecInfos()); @@ -145,32 +155,54 @@ default BlockCodecs createRecursive(DataType dataType, int[] blockSize, D } + + + + public static class BlockCodecs { private final List shardCodecs = new ArrayList<>(); + private final DataBlockCodec dataBlockCodec; + + // size of shard at a given level in units of DataBlocks private final List chunkSizesInDataBlocks = new ArrayList<>(); - private final DataBlockCodec dataBlockCodec; + // number of nested elements in shard at a given level + private final List relativeChunkSizes = new ArrayList<>(); private final int[] dataBlockSize; public BlockCodecs(final DataBlockCodec dataBlockCodec, final int[] dataBlockSize) { this.dataBlockCodec = dataBlockCodec; this.dataBlockSize = dataBlockSize; + + final int[] gridSize = new int[dataBlockSize.length]; + Arrays.fill(gridSize, 1); + chunkSizesInDataBlocks.add(gridSize); } + /** + * @param shardSize in pixels + */ public BlockCodecs(final ShardCodec codec, final int[] shardSize, final BlockCodecs others) { this(others.dataBlockCodec, others.dataBlockSize); shardCodecs.add(codec); shardCodecs.addAll(others.shardCodecs); + // TODO: verify that shardSize is integer multiple of dataBlockSize. + // TODO: verify that shardSize is integer multiple of nested shardSize + final int[] gridSize = new int[shardSize.length]; Arrays.setAll(gridSize, d -> shardSize[d] / dataBlockSize[d]); chunkSizesInDataBlocks.add(gridSize); - // TODO: verify shardSize is integer multiple of dataBlockSize. - // TODO: verify size is integer multiple of nested chunkSizeInDataBlocks. + chunkSizesInDataBlocks.addAll(others.chunkSizesInDataBlocks); + final int[] relativeChunkSize = new int[shardSize.length]; + final int[] nestedGridSize = others.chunkSizesInDataBlocks.get(0); + Arrays.setAll(relativeChunkSize, d -> gridSize[d] / nestedGridSize[d]); + relativeChunkSizes.add(relativeChunkSize); + relativeChunkSizes.addAll(others.relativeChunkSizes); } public int[] getDataBlockSize() { @@ -199,13 +231,35 @@ public NestedBlockPosition getNestedDataBlockPosition(final long[] gridPos) { return new NestedBlockPosition(nested); } - public DataBlock decodeDataBlock(final ReadData keyData, final NestedBlockPosition pos) { + public DataBlock decodeDataBlock(final ReadData keyData, final NestedBlockPosition nestedPos) { + + + final int depth = nestedPos.depth(); + final int n = nestedPos.numDimensions(); + + ReadData data = keyData; + final long[] gridOffset = new long[n]; + for (int i = 0; i < depth; ++i) { + final long[] relativeGridPos = nestedPos.relativePosition(i); + final long[] gridPosition = new long[n]; + Arrays.setAll(gridPosition, d -> gridOffset[d] + relativeGridPos[d]); + + if ( i < depth - 1 ) { + final Shard shard = shardCodecs.get(i).decode(data, gridPosition); + data = shard.getElementData(relativeGridPos); + } else { + return dataBlockCodec.decode(data, gridPosition); + } + + final int[] relativeGridSize = relativeChunkSizes.get(i); + Arrays.setAll(gridOffset, d -> gridPosition[d] * relativeGridSize[d]); + } - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException(); } } @@ -225,6 +279,10 @@ public int depth() { return nested.length; } + public int numDimensions() { + return nested[0].length; + } + public long[] relativePosition(final int level) { return nested[level]; } @@ -269,11 +327,27 @@ DataBlock readBlock( } catch (N5Exception.N5NoSuchKeyException e) { return null; } - - throw new UnsupportedOperationException(); } } + + + + + + + + + + + + + + private static DataCodec[] instantiate(final DataCodecInfo... codecInfos) { + final DataCodec[] codecs = new DataCodec[codecInfos.length]; + Arrays.setAll(codecs, i -> codecInfos[i].create()); + return codecs; + } } From 72ede5419ff40b43883bedad94b3cd0d0643de73 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 28 Aug 2025 17:40:42 +0200 Subject: [PATCH 279/423] WIP --- .../janelia/saalfeldlab/n5/ShardStuff.java | 85 ++++++++++++++++--- 1 file changed, 71 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java index 83c7fdc3c..452c18a4a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java @@ -1,7 +1,5 @@ package org.janelia.saalfeldlab.n5; -import com.google.gson.Gson; -import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -209,6 +207,14 @@ public int[] getDataBlockSize() { return dataBlockSize; } + + + // TODO: Decide whether to use absolute of relative NestedBlockPosition. + // (For decoding we need both: + // shard.getElementData(RELATIVE_GRID_POSITION), + // (shard|dataBlock)Codec.decode(data, ABSOLUTE_GRID_POSITION) + // Maybe NestedBlockPosition should just contain both? + public NestedBlockPosition getNestedDataBlockPosition(final long[] gridPos) { if (shardCodecs.isEmpty()) { return new NestedBlockPosition(new long[][] {gridPos}); @@ -231,38 +237,77 @@ public NestedBlockPosition getNestedDataBlockPosition(final long[] gridPos) { return new NestedBlockPosition(nested); } - public DataBlock decodeDataBlock(final ReadData keyData, final NestedBlockPosition nestedPos) { + public NestedBlockPosition relativeToAbsolute(final NestedBlockPosition nestedPos) { + + } + + + + + + + + public Shard getShard(ReadData data, final NestedBlockPosition nestedPos) { + // TODO: validation: nestedPos should refer to a Shard (not a DataBlock) + + // TODO: handle missing shards final int depth = nestedPos.depth(); final int n = nestedPos.numDimensions(); - ReadData data = keyData; final long[] gridOffset = new long[n]; for (int i = 0; i < depth; ++i) { - final long[] relativeGridPos = nestedPos.relativePosition(i); + final long[] relativeGridPos = nestedPos.position(i); final long[] gridPosition = new long[n]; Arrays.setAll(gridPosition, d -> gridOffset[d] + relativeGridPos[d]); - if ( i < depth - 1 ) { - final Shard shard = shardCodecs.get(i).decode(data, gridPosition); - data = shard.getElementData(relativeGridPos); - } else { - return dataBlockCodec.decode(data, gridPosition); + final Shard shard = shardCodecs.get(i).decode(data, gridPosition); + if ( i == depth - 1 ) { + return shard; } + data = shard.getElementData(relativeGridPos); final int[] relativeGridSize = relativeChunkSizes.get(i); Arrays.setAll(gridOffset, d -> gridPosition[d] * relativeGridSize[d]); } + throw new IllegalStateException(); // we should never end up here + } + + public DataBlock extractDataBlock(ReadData data, final NestedBlockPosition nestedPos) { + + // TODO: validation: nestedPos should refer to a DataBlock (not a Shard) + + // TODO: handle missing shards / blocks + + final int depth = nestedPos.depth(); + final int n = nestedPos.numDimensions(); + + final long[] gridOffset = new long[n]; + for (int i = 0; i < depth; ++i) { + final long[] relativeGridPos = nestedPos.position(i); + final long[] gridPosition = new long[n]; + Arrays.setAll(gridPosition, d -> gridOffset[d] + relativeGridPos[d]); + if (i == depth - 1) { + return dataBlockCodec.decode(data, gridPosition); + } else { + final Shard shard = shardCodecs.get(i).decode(data, gridPosition); + data = shard.getElementData(relativeGridPos); + } - throw new UnsupportedOperationException(); + final int[] relativeGridSize = relativeChunkSizes.get(i); + Arrays.setAll(gridOffset, d -> gridPosition[d] * relativeGridSize[d]); + } + + throw new IllegalStateException(); // we should never end up here } } + // TODO: Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...). // For nested = {X,Y,Z} compare by X, then Y, then Z. // For X = {x,y,z} compare by z, then y, then x. (flattening order) @@ -283,16 +328,28 @@ public int numDimensions() { return nested[0].length; } - public long[] relativePosition(final int level) { + public long[] position(final int level) { return nested[level]; } public long[] keyPosition() { - return relativePosition(0); + return position(0); + } + + public NestedBlockPosition prefix(final int depth) { + // TODO: implement + throw new UnsupportedOperationException("TODO"); } } + // TODO: Implement NestedBlockPositions that recursively groups blocks under the same prefix. + // This would be the input to + + + + + // DUMMY static abstract class N5ReaderImpl implements GsonKeyValueN5Reader { @@ -323,7 +380,7 @@ DataBlock readBlock( try { final ReadData keyData = getKeyValueAccess().createReadData(path); - return blockCodecs.decodeDataBlock(keyData, pos); + return blockCodecs.extractDataBlock(keyData, pos); } catch (N5Exception.N5NoSuchKeyException e) { return null; } From a97e604c0fb805bf3513335b8d850fde6f8f49da Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 28 Aug 2025 18:28:38 +0200 Subject: [PATCH 280/423] WIP --- .../janelia/saalfeldlab/n5/ShardStuff.java | 135 +++++++++++------- 1 file changed, 80 insertions(+), 55 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java index 452c18a4a..a5a130519 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java @@ -166,6 +166,7 @@ public static class BlockCodecs { // size of shard at a given level in units of DataBlocks private final List chunkSizesInDataBlocks = new ArrayList<>(); + // TODO: remove? // number of nested elements in shard at a given level private final List relativeChunkSizes = new ArrayList<>(); @@ -175,9 +176,9 @@ public BlockCodecs(final DataBlockCodec dataBlockCodec, final int[] dataBlock this.dataBlockCodec = dataBlockCodec; this.dataBlockSize = dataBlockSize; - final int[] gridSize = new int[dataBlockSize.length]; - Arrays.fill(gridSize, 1); - chunkSizesInDataBlocks.add(gridSize); + final int[] ones = new int[dataBlockSize.length]; + Arrays.fill(ones, 1); + chunkSizesInDataBlocks.add(ones); } /** @@ -214,68 +215,54 @@ public int[] getDataBlockSize() { // shard.getElementData(RELATIVE_GRID_POSITION), // (shard|dataBlock)Codec.decode(data, ABSOLUTE_GRID_POSITION) // Maybe NestedBlockPosition should just contain both? + // ==> Trying that now... public NestedBlockPosition getNestedDataBlockPosition(final long[] gridPos) { if (shardCodecs.isEmpty()) { - return new NestedBlockPosition(new long[][] {gridPos}); + final long[][] nested = new long[][] {gridPos}; + return new NestedBlockPosition(nested, nested); } final int n = gridPos.length; final int m = shardCodecs.size() + 1; - final long[][] nested = new long[m][n]; - final long[] pos = nested[m - 1]; - Arrays.setAll(pos, d -> gridPos[d]); + final long[][] nestedAbsolute = new long[m][]; for (int i = 0; i < m - 1; ++i) { + nestedAbsolute[i] = new long[n]; final int[] gridSize = chunkSizesInDataBlocks.get(i); for (int d = 0; d < n; ++d) { - nested[i][d] = pos[d] / gridSize[d]; - pos[d] = pos[d] % gridSize[d]; + nestedAbsolute[i][d] = gridPos[d] / gridSize[d]; } } + nestedAbsolute[m - 1] = gridPos; // TODO: defensive clone() or not? - return new NestedBlockPosition(nested); - } - - public NestedBlockPosition relativeToAbsolute(final NestedBlockPosition nestedPos) { + final long[][] nestedRelative = new long[m][]; + nestedRelative[0] = nestedAbsolute[0]; + for (int i = 1; i < m; ++i) { + nestedRelative[i] = new long[n]; + final int[] relativeGridSize = relativeChunkSizes.get(i - 1); + for (int d = 0; d < n; ++d) { + nestedRelative[i][d] = nestedAbsolute[i][d] / relativeGridSize[d]; + } + } + return new NestedBlockPosition(nestedAbsolute, nestedRelative); } + // TODO + // ====> NEXT + // ====> NEXT + // ====> NEXT + // ====> NEXT + // ====> NEXT + // ====> NEXT + // 1) Rewrite extractDataBlock() to use relative/absolute coordinates form nestedPos + // 2) Extract something like "get the readData for nestedPos" which + // can then be re-used to implement extractDataBlock() and extractShard() non-recursively + // 3) See whether that helps to think about writing / updating blocks - - - - public Shard getShard(ReadData data, final NestedBlockPosition nestedPos) { - - // TODO: validation: nestedPos should refer to a Shard (not a DataBlock) - - // TODO: handle missing shards - - final int depth = nestedPos.depth(); - final int n = nestedPos.numDimensions(); - - final long[] gridOffset = new long[n]; - for (int i = 0; i < depth; ++i) { - final long[] relativeGridPos = nestedPos.position(i); - - final long[] gridPosition = new long[n]; - Arrays.setAll(gridPosition, d -> gridOffset[d] + relativeGridPos[d]); - - final Shard shard = shardCodecs.get(i).decode(data, gridPosition); - if ( i == depth - 1 ) { - return shard; - } - data = shard.getElementData(relativeGridPos); - - final int[] relativeGridSize = relativeChunkSizes.get(i); - Arrays.setAll(gridOffset, d -> gridPosition[d] * relativeGridSize[d]); - } - - throw new IllegalStateException(); // we should never end up here - } - public DataBlock extractDataBlock(ReadData data, final NestedBlockPosition nestedPos) { // TODO: validation: nestedPos should refer to a DataBlock (not a Shard) @@ -287,7 +274,7 @@ public DataBlock extractDataBlock(ReadData data, final NestedBlockPosition ne final long[] gridOffset = new long[n]; for (int i = 0; i < depth; ++i) { - final long[] relativeGridPos = nestedPos.position(i); + final long[] relativeGridPos = nestedPos.relativePosition(i); final long[] gridPosition = new long[n]; Arrays.setAll(gridPosition, d -> gridOffset[d] + relativeGridPos[d]); @@ -305,6 +292,38 @@ public DataBlock extractDataBlock(ReadData data, final NestedBlockPosition ne throw new IllegalStateException(); // we should never end up here } + + + +// public Shard getShard(ReadData data, final NestedBlockPosition nestedPos) { +// +// // TODO: validation: nestedPos should refer to a Shard (not a DataBlock) +// +// // TODO: handle missing shards +// +// final int depth = nestedPos.depth(); +// final int n = nestedPos.numDimensions(); +// +// final long[] gridOffset = new long[n]; +// for (int i = 0; i < depth; ++i) { +// final long[] relativeGridPos = nestedPos.relativePosition(i); +// +// final long[] gridPosition = new long[n]; +// Arrays.setAll(gridPosition, d -> gridOffset[d] + relativeGridPos[d]); +// +// final Shard shard = shardCodecs.get(i).decode(data, gridPosition); +// if ( i == depth - 1 ) { +// return shard; +// } +// data = shard.getElementData(relativeGridPos); +// +// final int[] relativeGridSize = relativeChunkSizes.get(i); +// Arrays.setAll(gridOffset, d -> gridPosition[d] * relativeGridSize[d]); +// } +// +// throw new IllegalStateException(); // we should never end up here +// } + } @@ -313,27 +332,33 @@ public DataBlock extractDataBlock(ReadData data, final NestedBlockPosition ne // For X = {x,y,z} compare by z, then y, then x. (flattening order) public static class NestedBlockPosition { - private final long[][] nested; + private final long[][] nestedRelative; + private final long[][] nestedAbsolute; - NestedBlockPosition(final long[][] nested) { - // TODO: validation: nested != null, at least one level - this.nested = nested; + NestedBlockPosition(final long[][] nestedAbsolute, final long[][] nestedRelative) { + // TODO: validation: nestedRelative != null, at least one level, nestedAbsolute.length == nestedRelative.length + this.nestedAbsolute = nestedAbsolute; + this.nestedRelative = nestedRelative; } public int depth() { - return nested.length; + return nestedRelative.length; } public int numDimensions() { - return nested[0].length; + return nestedRelative[0].length; + } + + public long[] relativePosition(final int level) { + return nestedRelative[level]; } - public long[] position(final int level) { - return nested[level]; + public long[] absolutePosition(final int level) { + return nestedAbsolute[level]; } public long[] keyPosition() { - return position(0); + return relativePosition(0); } public NestedBlockPosition prefix(final int depth) { From d9e2f373c22a4bae9fef34cc0581a405def3fc91 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 28 Aug 2025 21:56:47 +0200 Subject: [PATCH 281/423] WIP --- .../org/janelia/saalfeldlab/n5/ShardStuff.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java index a5a130519..e40596447 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java @@ -63,6 +63,8 @@ public interface Shard extends Block { // pos is relative to this shard void setElementData(ReadData data, long[] pos); + + // TODO: removeElement(long[] pos) } @@ -130,6 +132,8 @@ public interface ShardCodecInfo extends BlockCodecInfo { DataCodecInfo[] getInnerDataCodecInfos(); + // TODO: IndexCodec + ShardCodec create(int[] blockSize, DataCodecInfo... codecs); @Override @@ -270,24 +274,16 @@ public DataBlock extractDataBlock(ReadData data, final NestedBlockPosition ne // TODO: handle missing shards / blocks final int depth = nestedPos.depth(); - final int n = nestedPos.numDimensions(); - final long[] gridOffset = new long[n]; for (int i = 0; i < depth; ++i) { - final long[] relativeGridPos = nestedPos.relativePosition(i); - - final long[] gridPosition = new long[n]; - Arrays.setAll(gridPosition, d -> gridOffset[d] + relativeGridPos[d]); + final long[] gridPosition = nestedPos.absolutePosition(i); if (i == depth - 1) { return dataBlockCodec.decode(data, gridPosition); } else { final Shard shard = shardCodecs.get(i).decode(data, gridPosition); - data = shard.getElementData(relativeGridPos); + data = shard.getElementData(nestedPos.relativePosition(i + 1)); } - - final int[] relativeGridSize = relativeChunkSizes.get(i); - Arrays.setAll(gridOffset, d -> gridPosition[d] * relativeGridSize[d]); } throw new IllegalStateException(); // we should never end up here @@ -330,6 +326,8 @@ public DataBlock extractDataBlock(ReadData data, final NestedBlockPosition ne // TODO: Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...). // For nested = {X,Y,Z} compare by X, then Y, then Z. // For X = {x,y,z} compare by z, then y, then x. (flattening order) + + public static class NestedBlockPosition { private final long[][] nestedRelative; From 94fdf95350dcee9f938c8cab603aaa772d4acc1f Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 29 Aug 2025 08:25:50 +0200 Subject: [PATCH 282/423] move WIP to separate package --- .../saalfeldlab/n5/{ => shardstuff}/ShardStuff.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) rename src/main/java/org/janelia/saalfeldlab/n5/{ => shardstuff}/ShardStuff.java (97%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff.java similarity index 97% rename from src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java rename to src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff.java index e40596447..6e567d90e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff.java @@ -1,9 +1,14 @@ -package org.janelia.saalfeldlab.n5; +package org.janelia.saalfeldlab.n5.shardstuff; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader; +import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.N5URI; import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodec; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; From 4b65d41bd45812dad2292899a4836005bb91d520 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 29 Aug 2025 12:30:02 +0200 Subject: [PATCH 283/423] WIP NestedGrid --- .../saalfeldlab/n5/shardstuff/Nesting.java | 122 ++++++++++++++++++ .../saalfeldlab/n5/shardstuff/ShardStuff.java | 46 ++++--- .../n5/shardstuff/NestedGridTest.java | 61 +++++++++ 3 files changed, 209 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java new file mode 100644 index 000000000..b18b32704 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java @@ -0,0 +1,122 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + + +// TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// +// [ ] NestedGrid javadoc +// [ ] NestedPosition interface +// +// TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +import java.util.Arrays; + +public class Nesting { + + + /** + * TODO + */ + public static class NestedGrid { + + // num levels + private final int m; + + // num dimensions + private final int n; + + // s[i][d] is block size at level i relative to level 0 + private final int[][] s; + + // r[i][d] is block size at level i relative to level i-1 + private final int[][] r; + + // NB: as usual, level 0 is full resolution. + // TODO: BlockCodec stuff does this differently at the moment, but that should be changed ... + + /** + * {@code blockSizes[l][d]} is the block size at level {@code l} in dimension {@code d}. + * Level 0 is the highest resolution (smallest block sizes). + * + * @param blockSizes + * block sizes for all levels and dimensions. + */ + public NestedGrid(int[][] blockSizes) { + + // TODO: validate + // [ ] not null + // [ ] nesteds not null + // [ ] nesteds same length + // [ ] sizes match up to integer multiples + + m = blockSizes.length; + n = blockSizes[0].length; + s = new int[m][n]; + r = new int[m][n]; + for (int l = 0; l < m; ++l) { + final int k = Math.max(0, l - 1); + for (int d = 0; d < n; ++d) { + s[l][d] = blockSizes[l][d] / blockSizes[0][d]; + r[l][d] = blockSizes[l][d] / blockSizes[k][d]; + } + } + } + + public void absolutePosition( + final long[] sourcePos, + final int sourceLevel, + final long[] targetPos, + final int targetLevel) { + final int[] sk = s[sourceLevel]; + final int[] si = s[targetLevel]; + for (int d = 0; d < n; ++d) { + targetPos[d] = sourcePos[d] * sk[d] / si[d]; + } + } + + public void relativePosition( + final long[] sourcePos, + final int sourceLevel, + final long[] targetPos, + final int targetLevel) { + absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel); + if (targetLevel < m - 1) { + final int[] rj = r[targetLevel + 1]; + for (int d = 0; d < n; ++d) { + targetPos[d] %= rj[d]; + } + } + } + + public long[] absolutePosition( + final long[] sourcePos, + final int sourceLevel, + final int targetLevel) { + final long[] targetPos = new long[n]; + absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel); + return targetPos; + } + + public long[] absolutePosition( + final long[] sourcePos, + final int targetLevel) { + return absolutePosition(sourcePos, 0, targetLevel); + } + + public long[] relativePosition( + final long[] sourcePos, + final int sourceLevel, + final int targetLevel) { + final long[] targetPos = new long[n]; + relativePosition(sourcePos, sourceLevel, targetPos, targetLevel); + return targetPos; + } + + public long[] relativePosition( + final long[] sourcePos, + final int targetLevel) { + return relativePosition(sourcePos, 0, targetLevel); + } + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff.java index 6e567d90e..14a36432d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff.java @@ -219,12 +219,14 @@ public int[] getDataBlockSize() { - // TODO: Decide whether to use absolute of relative NestedBlockPosition. - // (For decoding we need both: - // shard.getElementData(RELATIVE_GRID_POSITION), - // (shard|dataBlock)Codec.decode(data, ABSOLUTE_GRID_POSITION) - // Maybe NestedBlockPosition should just contain both? - // ==> Trying that now... + // TODO: [+] Decide whether to use absolute of relative NestedBlockPosition. + // For decoding we need both: + // shard.getElementData(RELATIVE_GRID_POSITION), + // (shard|dataBlock)Codec.decode(data, ABSOLUTE_GRID_POSITION) + // Maybe NestedBlockPosition should just contain both? + // ==> Trying that now... + + // TODO: [ ] split into separate NestedGrid class public NestedBlockPosition getNestedDataBlockPosition(final long[] gridPos) { if (shardCodecs.isEmpty()) { @@ -259,17 +261,12 @@ public NestedBlockPosition getNestedDataBlockPosition(final long[] gridPos) { } - // TODO - // ====> NEXT - // ====> NEXT - // ====> NEXT - // ====> NEXT - // ====> NEXT - // ====> NEXT - // 1) Rewrite extractDataBlock() to use relative/absolute coordinates form nestedPos - // 2) Extract something like "get the readData for nestedPos" which - // can then be re-used to implement extractDataBlock() and extractShard() non-recursively - // 3) See whether that helps to think about writing / updating blocks + // TODO: + // + // [+] Rewrite extractDataBlock() to use relative/absolute coordinates form nestedPos + // [ ] Extract something like "get the readData for nestedPos" which + // can then be re-used to implement extractDataBlock() and extractShard() non-recursively + // [ ] See whether that helps to think about writing / updating blocks public DataBlock extractDataBlock(ReadData data, final NestedBlockPosition nestedPos) { @@ -328,9 +325,18 @@ public DataBlock extractDataBlock(ReadData data, final NestedBlockPosition ne } - // TODO: Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...). - // For nested = {X,Y,Z} compare by X, then Y, then Z. - // For X = {x,y,z} compare by z, then y, then x. (flattening order) + + + + + + // TODO: + // [ ] Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...). + // For nested = {X,Y,Z} compare by X, then Y, then Z. + // For X = {x,y,z} compare by z, then y, then x. (flattening order) + // [ ] Split into interface and implementation + // [ ] Add implementation based on NestedGrid and long[] gridPosition + public static class NestedBlockPosition { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java new file mode 100644 index 000000000..76d06ebb9 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java @@ -0,0 +1,61 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; +import org.junit.Assert; +import org.junit.Test; + +public class NestedGridTest { + + private static long absPosition1D(final NestedGrid grid, final int sourcePos, final int targetLevel) { + return grid.absolutePosition(new long[] {sourcePos}, targetLevel)[0]; + } + + @Test + public void testAbsolutePosition() { + int[][] blockSizes = {{1}, {3}, {6}, {24}}; + NestedGrid grid = new NestedGrid(blockSizes); + + Assert.assertEquals(38, absPosition1D(grid, 38, 0)); + Assert.assertEquals(12, absPosition1D(grid, 38, 1)); + Assert.assertEquals(6, absPosition1D(grid, 38, 2)); + Assert.assertEquals(1, absPosition1D(grid, 38, 3)); + } + + @Test + public void testAbsolutePositionChunkSize() { + int[][] blockSizes = {{10}, {30}, {60}, {240}}; + NestedGrid grid = new NestedGrid(blockSizes); + + Assert.assertEquals(38, absPosition1D(grid, 38, 0)); + Assert.assertEquals(12, absPosition1D(grid, 38, 1)); + Assert.assertEquals(6, absPosition1D(grid, 38, 2)); + Assert.assertEquals(1, absPosition1D(grid, 38, 3)); + } + + private static long relPosition1D(final NestedGrid grid, final int sourcePos, final int targetLevel) { + return grid.relativePosition(new long[] {sourcePos}, targetLevel)[0]; + } + + @Test + public void testRelativePosition() { + int[][] blockSizes = {{1}, {3}, {6}, {24}}; + NestedGrid grid = new NestedGrid(blockSizes); + + Assert.assertEquals(2, relPosition1D(grid, 38, 0)); + Assert.assertEquals(0, relPosition1D(grid, 38, 1)); + Assert.assertEquals(2, relPosition1D(grid, 38, 2)); + Assert.assertEquals(1, relPosition1D(grid, 38, 3)); + + } + + @Test + public void testRelativePositionChunkSize() { + int[][] blockSizes = {{10}, {30}, {60}, {240}}; + NestedGrid grid = new NestedGrid(blockSizes); + + Assert.assertEquals(2, relPosition1D(grid, 38, 0)); + Assert.assertEquals(0, relPosition1D(grid, 38, 1)); + Assert.assertEquals(2, relPosition1D(grid, 38, 2)); + Assert.assertEquals(1, relPosition1D(grid, 38, 3)); + } +} From 1dd88cbddca2555f2a0d7e4af18c6a393768aec8 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 29 Aug 2025 09:52:03 -0400 Subject: [PATCH 284/423] refactor: large renaming to align with master before merge --- .../janelia/saalfeldlab/n5/Compression.java | 10 +- .../saalfeldlab/n5/DatasetAttributes.java | 68 +++++----- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 2 +- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 8 +- .../org/janelia/saalfeldlab/n5/N5Writer.java | 22 ++-- ...taBlockSerializer.java => BlockCodec.java} | 2 +- .../{ArrayCodec.java => BlockCodecInfo.java} | 6 +- .../n5/codec/{Codec.java => CodecInfo.java} | 4 +- ...sCodec.java => ConcatenatedDataCodec.java} | 10 +- .../codec/{BytesCodec.java => DataCodec.java} | 16 +-- .../n5/codec/DeterministicSizeCodec.java | 4 +- ...raySerializer.java => FlatArrayCodec.java} | 60 ++++----- .../saalfeldlab/n5/codec/IdentityCodec.java | 2 +- .../n5/codec/IndexCodecAdapter.java | 20 +-- ...5ArrayCodec.java => N5BlockCodecInfo.java} | 11 +- ...ockSerializers.java => N5BlockCodecs.java} | 88 ++++++------- ...ArrayCodec.java => RawBlockCodecInfo.java} | 14 +-- .../saalfeldlab/n5/codec/RawBlockCodecs.java | 119 ++++++++++++++++++ .../n5/codec/RawDataBlockSerializers.java | 119 ------------------ .../n5/codec/checksum/ChecksumCodec.java | 9 +- .../saalfeldlab/n5/readdata/ReadData.java | 2 +- .../n5/shard/BlockAsShardCodec.java | 28 ++--- .../janelia/saalfeldlab/n5/shard/Shard.java | 5 +- .../saalfeldlab/n5/shard/ShardIndex.java | 15 +-- .../saalfeldlab/n5/shard/ShardingCodec.java | 42 +++---- .../saalfeldlab/n5/DatasetAttributesTest.java | 14 +-- .../saalfeldlab/n5/cache/N5CacheTest.java | 2 +- .../saalfeldlab/n5/codec/ArrayCodecTests.java | 28 ++--- .../saalfeldlab/n5/codec/BytesCodecTests.java | 11 +- .../saalfeldlab/n5/demo/BlockIterators.java | 10 +- .../n5/http/HttpReaderFsWriter.java | 5 +- .../n5/serialization/CodecSerialization.java | 15 ++- .../saalfeldlab/n5/shard/ShardIndexTest.java | 10 +- .../saalfeldlab/n5/shard/ShardTest.java | 53 ++++---- 34 files changed, 410 insertions(+), 424 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/codec/{DataBlockSerializer.java => BlockCodec.java} (97%) rename src/main/java/org/janelia/saalfeldlab/n5/codec/{ArrayCodec.java => BlockCodecInfo.java} (73%) rename src/main/java/org/janelia/saalfeldlab/n5/codec/{Codec.java => CodecInfo.java} (75%) rename src/main/java/org/janelia/saalfeldlab/n5/codec/{ConcatenatedBytesCodec.java => ConcatenatedDataCodec.java} (72%) rename src/main/java/org/janelia/saalfeldlab/n5/codec/{BytesCodec.java => DataCodec.java} (70%) rename src/main/java/org/janelia/saalfeldlab/n5/codec/{FlatArraySerializer.java => FlatArrayCodec.java} (82%) rename src/main/java/org/janelia/saalfeldlab/n5/codec/{N5ArrayCodec.java => N5BlockCodecInfo.java} (63%) rename src/main/java/org/janelia/saalfeldlab/n5/codec/{N5DataBlockSerializers.java => N5BlockCodecs.java} (73%) rename src/main/java/org/janelia/saalfeldlab/n5/codec/{RawBytesArrayCodec.java => RawBlockCodecInfo.java} (78%) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/RawDataBlockSerializers.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index e3d022732..a64407079 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -35,24 +35,24 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; -import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.DataCodec; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.scijava.annotations.Indexable; /** - * This interface is used to indicate that a {@link BytesCodec} can be + * This interface is used to indicate that a {@link DataCodec} can be * serialized as a "compression" for the N5 format (using the N5 API). *

      * N5Readers and N5Writers for the N5 format can declare BytesCodecs that * implement this interface so that the {@link CompressionAdapter} is used for * serialization. *

      - * See also: an alternative method for serializing general {@link Codec}s is + * See also: an alternative method for serializing general {@link CodecInfo}s is * with the {@link NameConfigAdapter}. * * @author Stephan Saalfeld */ -public interface Compression extends Serializable, BytesCodec { +public interface Compression extends Serializable, DataCodec { /** * Annotation for runtime discovery of compression schemes. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index efca0865d..4d2b3fd7d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -35,11 +35,11 @@ import com.google.gson.JsonParseException; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; -import org.janelia.saalfeldlab.n5.codec.ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.DataBlockSerializer; -import org.janelia.saalfeldlab.n5.codec.N5ArrayCodec; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; +import org.janelia.saalfeldlab.n5.codec.DataCodec; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.shard.BlockAsShardCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.util.Position; @@ -61,7 +61,7 @@ *

    • long[] : dimensions
      • *
    • int[] : blockSize
      • *
    • {@link DataType} : dataType
      • - *
    • {@link Codec}... : encode/decode routines
      • + *
    • {@link CodecInfo}... : encode/decode routines
      • * * * @author Stephan Saalfeld @@ -94,18 +94,18 @@ public class DatasetAttributes implements Serializable { private final DataType dataType; private final ShardingCodec shardingCodec; - private final ArrayCodec arrayCodec; - private final BytesCodec[] byteCodecs; + private final BlockCodecInfo arrayCodec; + private final DataCodec[] byteCodecs; - private final DataBlockSerializer dataBlockSerializer; + private final BlockCodec dataBlockSerializer; public DatasetAttributes( final long[] dimensions, final int[] shardSize, final int[] blockSize, final DataType dataType, - final ArrayCodec arrayCodec, - final BytesCodec... codecs) { + final BlockCodecInfo arrayCodec, + final DataCodec... codecs) { validateBlockShardSizes(dimensions, shardSize, blockSize); @@ -115,12 +115,12 @@ public DatasetAttributes( this.dataType = dataType; this.arrayCodec = arrayCodec == null ? defaultArrayCodec() : arrayCodec; - byteCodecs = Arrays.stream(codecs).filter(it -> !(it instanceof RawCompression)).toArray(BytesCodec[]::new); + byteCodecs = Arrays.stream(codecs).filter(it -> !(it instanceof RawCompression)).toArray(DataCodec[]::new); if (this.arrayCodec instanceof ShardingCodec) shardingCodec = (ShardingCodec)this.arrayCodec; else shardingCodec = new BlockAsShardCodec(this.arrayCodec); - dataBlockSerializer = this.shardingCodec.initialize(this, byteCodecs); + dataBlockSerializer = this.shardingCodec.create(this, byteCodecs); } @@ -151,9 +151,9 @@ else if (shardSize[i] % blockSize[i] != 0) } } - protected ArrayCodec defaultArrayCodec() { + protected BlockCodecInfo defaultArrayCodec() { - return new N5ArrayCodec(); + return new N5BlockCodecInfo(); } /** @@ -169,7 +169,7 @@ public DatasetAttributes( final long[] dimensions, final int[] blockSize, final DataType dataType, - final BytesCodec compression) { + final DataCodec compression) { this(dimensions, blockSize, blockSize, dataType, null, compression); } @@ -391,7 +391,7 @@ public ShardingCodec getShardingCodec() { *

      * Deprecated in favor of {@link #getCodecs()}. * - * @return compression Codec, if one was present, or else RawCompression + * @return compression CodecInfo, if one was present, or else RawCompression */ @Deprecated public Compression getCompression() { @@ -409,24 +409,24 @@ public DataType getDataType() { } /** - * Get the {@link ArrayCodec} for this dataset. + * Get the {@link BlockCodecInfo} for this dataset. * - * @return the {@code ArrayCodec} for this dataset + * @return the {@code BlockCodecInfo} for this dataset */ - public ArrayCodec getArrayCodec() { + public BlockCodecInfo getArrayCodec() { return arrayCodec; } - public BytesCodec[] getCodecs() { + public DataCodec[] getCodecs() { return byteCodecs; } @SuppressWarnings("unchecked") - public DataBlockSerializer getDataBlockSerializer() { + public BlockCodec getDataBlockSerializer() { - return (DataBlockSerializer)dataBlockSerializer; + return (BlockCodec)dataBlockSerializer; } public HashMap asMap() { @@ -470,23 +470,23 @@ public static class DatasetAttributesAdapter implements JsonSerializer 1 is actually invalid, but this is checked on construction if (codecs.length == 0) obj.add(COMPRESSION_KEY, context.serialize(new RawCompression())); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 121128ef5..54625dd56 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -121,7 +121,7 @@ default DataBlock readBlock( try { final ReadData keyData = getKeyValueAccess().createReadData(path); - //ArrayCodec knows how to get to the block from the shard, the DataBlockSerializer doesn't... + //BlockCodecInfo knows how to get to the block from the shard, the BlockCodec doesn't... if (datasetAttributes.isSharded()) { return datasetAttributes.getShardingCodec().decode(keyData, gridPosition); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 1c286484c..01eb21b1e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -38,8 +38,8 @@ import java.util.Map; import java.util.regex.Matcher; -import org.janelia.saalfeldlab.n5.codec.RawBytesArrayCodec; -import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -62,9 +62,9 @@ public interface GsonUtils { static Gson registerGson(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); + gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, DatasetAttributes.getJsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBytesArrayCodec.byteOrderAdapter); + gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBlockCodecInfo.byteOrderAdapter); gsonBuilder.registerTypeHierarchyAdapter(ShardingCodec.IndexLocation.class, ShardingCodec.indexLocationAdapter); gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.disableHtmlEscaping(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 115aa20ab..dbae8b706 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -28,9 +28,9 @@ */ package org.janelia.saalfeldlab.n5; -import org.janelia.saalfeldlab.n5.codec.ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; -import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodec; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.shard.Shard; import java.io.ByteArrayOutputStream; @@ -230,20 +230,20 @@ default void createDataset( final long[] dimensions, final int[] blockSize, final DataType dataType, - final Codec... codecs) throws N5Exception { + final CodecInfo... codecs) throws N5Exception { - final ArrayCodec arrayCodec; - final BytesCodec[] bytesCodecs; + final BlockCodecInfo arrayCodec; + final DataCodec[] bytesCodecs; if (codecs == null || codecs.length == 0) { arrayCodec = null; - bytesCodecs = new BytesCodec[0]; - } else if (codecs[0] instanceof ArrayCodec) { - arrayCodec = (ArrayCodec) codecs[0]; - bytesCodecs = new BytesCodec[codecs.length - 1]; + bytesCodecs = new DataCodec[0]; + } else if (codecs[0] instanceof BlockCodecInfo) { + arrayCodec = (BlockCodecInfo) codecs[0]; + bytesCodecs = new DataCodec[codecs.length - 1]; System.arraycopy(codecs, 1, bytesCodecs, 0, bytesCodecs.length); } else { arrayCodec = null; - bytesCodecs = new BytesCodec[codecs.length]; + bytesCodecs = new DataCodec[codecs.length]; System.arraycopy(codecs, 0, bytesCodecs, 0, bytesCodecs.length); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockSerializer.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java similarity index 97% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockSerializer.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java index 7c66b892b..8eb889661 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockSerializer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java @@ -38,7 +38,7 @@ * @param * type of the data contained in the DataBlock */ -public interface DataBlockSerializer { +public interface BlockCodec { ReadData encode(DataBlock dataBlock) throws N5IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ArrayCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java similarity index 73% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/ArrayCodec.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java index f3eaa11f7..39cc7bc1c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/ArrayCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java @@ -5,10 +5,10 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; /** - * {@code ArrayCodec}s encode {@link DataBlock}s into {@link ReadData} and + * {@code BlockCodecInfo}s encode {@link DataBlock}s into {@link ReadData} and * decode {@link ReadData} into {@link DataBlock}s. */ -public interface ArrayCodec extends Codec, DeterministicSizeCodec { +public interface BlockCodecInfo extends CodecInfo, DeterministicSizeCodec { default long[] getKeyPositionForBlock(final DatasetAttributes attributes, final DataBlock datablock) { @@ -20,7 +20,7 @@ default long[] getKeyPositionForBlock(final DatasetAttributes attributes, final return blockPosition; } - DataBlockSerializer initialize(final DatasetAttributes attributes, final BytesCodec... codecs); + BlockCodec create(final DatasetAttributes attributes, final DataCodec... codecs); @Override default long encodedSize(long size) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java similarity index 75% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java index 7b40b5818..79221a1eb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java @@ -5,13 +5,13 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; /** - * {@code Codec}s can encode and decode {@link ReadData} objects. + * {@code CodecInfo}s can encode and decode {@link ReadData} objects. *

      * Modeled after Codecs in * Zarr. */ @NameConfig.Prefix("codec") -public interface Codec extends Serializable { +public interface CodecInfo extends Serializable { String getType(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDataCodec.java similarity index 72% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDataCodec.java index 2daad7cdb..3f7dd2db2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDataCodec.java @@ -2,11 +2,11 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; -class ConcatenatedBytesCodec implements BytesCodec { +class ConcatenatedDataCodec implements DataCodec { - private final BytesCodec[] codecs; + private final DataCodec[] codecs; - ConcatenatedBytesCodec(final BytesCodec[] codecs) { + ConcatenatedDataCodec(final DataCodec[] codecs) { if (codecs == null) { throw new NullPointerException(); @@ -17,7 +17,7 @@ class ConcatenatedBytesCodec implements BytesCodec { @Override public ReadData encode(ReadData readData) { - for (BytesCodec codec : codecs) { + for (DataCodec codec : codecs) { readData = codec.encode(readData); } return readData; @@ -27,7 +27,7 @@ public ReadData encode(ReadData readData) { public ReadData decode(ReadData readData) { for (int i = codecs.length - 1; i >= 0; i--) { - final BytesCodec codec = codecs[i]; + final DataCodec codec = codecs[i]; readData = codec.decode(readData); } return readData; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java similarity index 70% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java index 800c0965d..e158b2141 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -4,17 +4,17 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; /** - * {@code BytesCodec}s transform one {@link ReadData} into another, + * {@code DataCodec}s transform one {@link ReadData} into another, * for example, compressing it. */ -public interface BytesCodec extends Codec { +public interface DataCodec extends CodecInfo { /** * Decode the given {@link ReadData}. *

      * The returned decoded {@code ReadData} reports {@link ReadData#length() * length()}{@code == decodedLength}. Decoding may be lazy or eager, - * depending on the {@code BytesCodec} implementation. + * depending on the {@code DataCodec} implementation. * * @param readData * data to decode @@ -29,7 +29,7 @@ public interface BytesCodec extends Codec { /** * Encode the given {@link ReadData}. *

      - * Encoding may be lazy or eager, depending on the {@code BytesCodec} + * Encoding may be lazy or eager, depending on the {@code DataCodec} * implementation. * * @param readData @@ -43,14 +43,14 @@ public interface BytesCodec extends Codec { ReadData encode(ReadData readData) throws N5IOException; /** - * Create a {@code BytesCodec} that sequentially applies {@code codecs} in + * Create a {@code DataCodec} that sequentially applies {@code codecs} in * the given order for encoding, and in reverse order for decoding. * * @param codecs * a list of BytesCodecs - * @return the concatenated BytesCodec + * @return the concatenated DataCodec */ - static BytesCodec concatenate(final BytesCodec... codecs) { + static DataCodec concatenate(final DataCodec... codecs) { if (codecs == null) throw new NullPointerException(); @@ -58,6 +58,6 @@ static BytesCodec concatenate(final BytesCodec... codecs) { if (codecs.length == 1) return codecs[0]; - return new ConcatenatedBytesCodec(codecs); + return new ConcatenatedDataCodec(codecs); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java index 9ac0a1fe0..3adaeccb5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodec.java @@ -1,10 +1,10 @@ package org.janelia.saalfeldlab.n5.codec; /** - * A {@link Codec} that can deterministically determine the size of encoded data from the size of the raw data and vice versa from the data length alone (i.e. encoding is data + * A {@link CodecInfo} that can deterministically determine the size of encoded data from the size of the raw data and vice versa from the data length alone (i.e. encoding is data * independent). */ -public interface DeterministicSizeCodec extends Codec { +public interface DeterministicSizeCodec extends CodecInfo { public abstract long encodedSize(long size); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/FlatArraySerializer.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FlatArrayCodec.java similarity index 82% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/FlatArraySerializer.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/FlatArrayCodec.java index 3573df913..129ae2844 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/FlatArraySerializer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FlatArrayCodec.java @@ -52,7 +52,7 @@ * @param * type of the data contained in the DataBlock */ -public abstract class FlatArraySerializer { +public abstract class FlatArrayCodec { public abstract ReadData serialize(T data) throws N5IOException; @@ -69,40 +69,40 @@ public T createData(final int numElements) { // ------------------- instances -------------------- // - public static final FlatArraySerializer BYTE = new ByteArraySerializer(); - public static final FlatArraySerializer SHORT_BIG_ENDIAN = new ShortArraySerializer(ByteOrder.BIG_ENDIAN); - public static final FlatArraySerializer INT_BIG_ENDIAN = new IntArraySerializer(ByteOrder.BIG_ENDIAN); - public static final FlatArraySerializer LONG_BIG_ENDIAN = new LongArraySerializer(ByteOrder.BIG_ENDIAN); - public static final FlatArraySerializer FLOAT_BIG_ENDIAN = new FloatArraySerializer(ByteOrder.BIG_ENDIAN); - public static final FlatArraySerializer DOUBLE_BIG_ENDIAN = new DoubleArraySerializer(ByteOrder.BIG_ENDIAN); + public static final FlatArrayCodec BYTE = new ByteArraySerializer(); + public static final FlatArrayCodec SHORT_BIG_ENDIAN = new ShortArraySerializer(ByteOrder.BIG_ENDIAN); + public static final FlatArrayCodec INT_BIG_ENDIAN = new IntArraySerializer(ByteOrder.BIG_ENDIAN); + public static final FlatArrayCodec LONG_BIG_ENDIAN = new LongArraySerializer(ByteOrder.BIG_ENDIAN); + public static final FlatArrayCodec FLOAT_BIG_ENDIAN = new FloatArraySerializer(ByteOrder.BIG_ENDIAN); + public static final FlatArrayCodec DOUBLE_BIG_ENDIAN = new DoubleArraySerializer(ByteOrder.BIG_ENDIAN); - public static final FlatArraySerializer SHORT_LITTLE_ENDIAN = new ShortArraySerializer(ByteOrder.LITTLE_ENDIAN); - public static final FlatArraySerializer INT_LITTLE_ENDIAN = new IntArraySerializer(ByteOrder.LITTLE_ENDIAN); - public static final FlatArraySerializer LONG_LITTLE_ENDIAN = new LongArraySerializer(ByteOrder.LITTLE_ENDIAN); - public static final FlatArraySerializer FLOAT_LITTLE_ENDIAN = new FloatArraySerializer(ByteOrder.LITTLE_ENDIAN); - public static final FlatArraySerializer DOUBLE_LITTLE_ENDIAN = new DoubleArraySerializer(ByteOrder.LITTLE_ENDIAN); + public static final FlatArrayCodec SHORT_LITTLE_ENDIAN = new ShortArraySerializer(ByteOrder.LITTLE_ENDIAN); + public static final FlatArrayCodec INT_LITTLE_ENDIAN = new IntArraySerializer(ByteOrder.LITTLE_ENDIAN); + public static final FlatArrayCodec LONG_LITTLE_ENDIAN = new LongArraySerializer(ByteOrder.LITTLE_ENDIAN); + public static final FlatArrayCodec FLOAT_LITTLE_ENDIAN = new FloatArraySerializer(ByteOrder.LITTLE_ENDIAN); + public static final FlatArrayCodec DOUBLE_LITTLE_ENDIAN = new DoubleArraySerializer(ByteOrder.LITTLE_ENDIAN); - public static final FlatArraySerializer STRING = new N5StringArraySerializer(); - public static final FlatArraySerializer ZARR_STRING = new ZarrStringArraySerializer(); - public static final FlatArraySerializer OBJECT = new ObjectArraySerializer(); + public static final FlatArrayCodec STRING = new N5StringArraySerializer(); + public static final FlatArrayCodec ZARR_STRING = new ZarrStringArraySerializer(); + public static final FlatArrayCodec OBJECT = new ObjectArraySerializer(); - public static FlatArraySerializer SHORT(ByteOrder order) { + public static FlatArrayCodec SHORT(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? SHORT_BIG_ENDIAN : SHORT_LITTLE_ENDIAN; } - public static FlatArraySerializer INT(ByteOrder order) { + public static FlatArrayCodec INT(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? INT_BIG_ENDIAN : INT_LITTLE_ENDIAN; } - public static FlatArraySerializer LONG(ByteOrder order) { + public static FlatArrayCodec LONG(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? LONG_BIG_ENDIAN : LONG_LITTLE_ENDIAN; } - public static FlatArraySerializer FLOAT(ByteOrder order) { + public static FlatArrayCodec FLOAT(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? FLOAT_BIG_ENDIAN : FLOAT_LITTLE_ENDIAN; } - public static FlatArraySerializer DOUBLE(ByteOrder order) { + public static FlatArrayCodec DOUBLE(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? DOUBLE_BIG_ENDIAN : DOUBLE_LITTLE_ENDIAN; } @@ -112,12 +112,12 @@ public static FlatArraySerializer DOUBLE(ByteOrder order) { private final int bytesPerElement; private final IntFunction dataFactory; - private FlatArraySerializer(int bytesPerElement, IntFunction dataFactory) { + private FlatArrayCodec(int bytesPerElement, IntFunction dataFactory) { this.bytesPerElement = bytesPerElement; this.dataFactory = dataFactory; } - private static final class ByteArraySerializer extends FlatArraySerializer { + private static final class ByteArraySerializer extends FlatArrayCodec { private ByteArraySerializer() { super(Byte.BYTES, byte[]::new); @@ -140,7 +140,7 @@ public byte[] deserialize(final ReadData readData, int numElements) throws N5IOE } } - private static final class ShortArraySerializer extends FlatArraySerializer { + private static final class ShortArraySerializer extends FlatArrayCodec { private final ByteOrder order; @@ -164,7 +164,7 @@ public short[] deserialize(final ReadData readData, int numElements) throws N5IO } } - private static final class IntArraySerializer extends FlatArraySerializer { + private static final class IntArraySerializer extends FlatArrayCodec { private final ByteOrder order; @@ -190,7 +190,7 @@ public int[] deserialize(final ReadData readData, int numElements) throws N5IOEx } } - private static final class LongArraySerializer extends FlatArraySerializer { + private static final class LongArraySerializer extends FlatArrayCodec { private final ByteOrder order; @@ -214,7 +214,7 @@ public long[] deserialize(final ReadData readData, int numElements) throws N5IOE } } - private static final class FloatArraySerializer extends FlatArraySerializer { + private static final class FloatArraySerializer extends FlatArrayCodec { private final ByteOrder order; @@ -238,7 +238,7 @@ public float[] deserialize(final ReadData readData, int numElements) throws N5IO } } - private static final class DoubleArraySerializer extends FlatArraySerializer { + private static final class DoubleArraySerializer extends FlatArrayCodec { private final ByteOrder order; @@ -262,7 +262,7 @@ public double[] deserialize(final ReadData readData, int numElements) throws N5I } } - private static final class N5StringArraySerializer extends FlatArraySerializer { + private static final class N5StringArraySerializer extends FlatArrayCodec { private static final Charset ENCODING = StandardCharsets.UTF_8; private static final String NULLCHAR = "\0"; @@ -285,7 +285,7 @@ public String[] deserialize(ReadData readData, int numElements) throws N5IOExcep } } - private static final class ZarrStringArraySerializer extends FlatArraySerializer { + private static final class ZarrStringArraySerializer extends FlatArrayCodec { private static final Charset ENCODING = StandardCharsets.UTF_8; @@ -333,7 +333,7 @@ public String[] deserialize(ReadData readData, int numElements) throws N5IOExcep } } - private static final class ObjectArraySerializer extends FlatArraySerializer { + private static final class ObjectArraySerializer extends FlatArrayCodec { ObjectArraySerializer() { super(-1, byte[]::new); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java index bcf63ed33..ac6525b9c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -4,7 +4,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @NameConfig.Name(IdentityCodec.TYPE) -public class IdentityCodec implements BytesCodec { +public class IdentityCodec implements DataCodec { private static final long serialVersionUID = 8354269325800855621L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IndexCodecAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IndexCodecAdapter.java index 1b553c128..07d892c49 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IndexCodecAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IndexCodecAdapter.java @@ -2,23 +2,23 @@ public class IndexCodecAdapter { - private final ArrayCodec arrayCodec; + private final BlockCodecInfo arrayCodec; private final DeterministicSizeCodec[] codecs; - public IndexCodecAdapter(final ArrayCodec arrayCodec, final DeterministicSizeCodec... codecs) { + public IndexCodecAdapter(final BlockCodecInfo arrayCodec, final DeterministicSizeCodec... codecs) { this.arrayCodec = arrayCodec; this.codecs = codecs; } - public ArrayCodec getArrayCodec() { + public BlockCodecInfo getArrayCodec() { return arrayCodec; } - public BytesCodec[] getCodecs() { + public DataCodec[] getCodecs() { - final BytesCodec[] bytesCodecs = new BytesCodec[codecs.length]; + final DataCodec[] bytesCodecs = new DataCodec[codecs.length]; System.arraycopy(codecs, 0, bytesCodecs, 0, codecs.length); return bytesCodecs; } @@ -31,15 +31,15 @@ public long encodedSize(long initialSize) { return totalNumBytes; } - public static IndexCodecAdapter create(final Codec... codecs) { + public static IndexCodecAdapter create(final CodecInfo... codecs) { if (codecs == null || codecs.length == 0) - return new IndexCodecAdapter(new RawBytesArrayCodec()); + return new IndexCodecAdapter(new RawBlockCodecInfo()); - if (codecs[0] instanceof ArrayCodec) - return new IndexCodecAdapter((ArrayCodec)codecs[0]); + if (codecs[0] instanceof BlockCodecInfo) + return new IndexCodecAdapter((BlockCodecInfo)codecs[0]); final DeterministicSizeCodec[] indexCodecs = new DeterministicSizeCodec[codecs.length]; System.arraycopy(codecs, 0, indexCodecs, 0, codecs.length); - return new IndexCodecAdapter(new RawBytesArrayCodec(), indexCodecs); + return new IndexCodecAdapter(new RawBlockCodecInfo(), indexCodecs); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5ArrayCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java similarity index 63% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/N5ArrayCodec.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java index 964d163f9..c9db14143 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5ArrayCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java @@ -2,11 +2,10 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -@NameConfig.Name(value = N5ArrayCodec.TYPE) -public class N5ArrayCodec implements ArrayCodec { +@NameConfig.Name(value = N5BlockCodecInfo.TYPE) +public class N5BlockCodecInfo implements BlockCodecInfo { private static final long serialVersionUID = 3523505403978222360L; @@ -14,9 +13,9 @@ public class N5ArrayCodec implements ArrayCodec { private transient DatasetAttributes attributes; - @Override public DataBlockSerializer initialize(final DatasetAttributes attributes, final BytesCodec... byteCodecs) { + @Override public BlockCodec create(final DatasetAttributes attributes, final DataCodec... byteCodecs) { this.attributes = attributes; - return N5DataBlockSerializers.create(attributes.getDataType(), new ConcatenatedBytesCodec(byteCodecs)); + return N5BlockCodecs.create(attributes.getDataType(), new ConcatenatedDataCodec(byteCodecs)); } @Override public long[] getKeyPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { @@ -32,7 +31,7 @@ public class N5ArrayCodec implements ArrayCodec { @Override public long encodedSize(long size) { final int[] blockSize = attributes.getBlockSize(); - int headerSize = new N5DataBlockSerializers.BlockHeader(blockSize, DataBlock.getNumElements(blockSize)).getSize(); + int headerSize = new N5BlockCodecs.BlockHeader(blockSize, DataBlock.getNumElements(blockSize)).getSize(); return headerSize + size; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5DataBlockSerializers.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java similarity index 73% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/N5DataBlockSerializers.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java index 6d0d39f12..078be04e9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5DataBlockSerializers.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java @@ -48,83 +48,83 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import static org.janelia.saalfeldlab.n5.N5Exception.*; -import static org.janelia.saalfeldlab.n5.codec.N5DataBlockSerializers.BlockHeader.MODE_DEFAULT; -import static org.janelia.saalfeldlab.n5.codec.N5DataBlockSerializers.BlockHeader.MODE_OBJECT; -import static org.janelia.saalfeldlab.n5.codec.N5DataBlockSerializers.BlockHeader.MODE_VARLENGTH; +import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.MODE_DEFAULT; +import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.MODE_OBJECT; +import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.MODE_VARLENGTH; -public class N5DataBlockSerializers { +public class N5BlockCodecs { - private static final DataBlockSerializerFactory BYTE = c -> new DefaultDataBlockSerializer<>(FlatArraySerializer.BYTE, ByteArrayDataBlock::new, c); - private static final DataBlockSerializerFactory SHORT = c -> new DefaultDataBlockSerializer<>(FlatArraySerializer.SHORT_BIG_ENDIAN, ShortArrayDataBlock::new, c); - private static final DataBlockSerializerFactory INT = c -> new DefaultDataBlockSerializer<>(FlatArraySerializer.INT_BIG_ENDIAN, IntArrayDataBlock::new, c); - private static final DataBlockSerializerFactory LONG = c -> new DefaultDataBlockSerializer<>(FlatArraySerializer.LONG_BIG_ENDIAN, LongArrayDataBlock::new, c); - private static final DataBlockSerializerFactory FLOAT = c -> new DefaultDataBlockSerializer<>(FlatArraySerializer.FLOAT_BIG_ENDIAN, FloatArrayDataBlock::new, c); - private static final DataBlockSerializerFactory DOUBLE = c -> new DefaultDataBlockSerializer<>(FlatArraySerializer.DOUBLE_BIG_ENDIAN, DoubleArrayDataBlock::new, c); - private static final DataBlockSerializerFactory STRING = c -> new StringDataBlockSerializer(c); - private static final DataBlockSerializerFactory OBJECT = c -> new ObjectDataBlockSerializer(c); + private static final BlockCodecFactory BYTE = c -> new DefaultBlockCodec<>(FlatArrayCodec.BYTE, ByteArrayDataBlock::new, c); + private static final BlockCodecFactory SHORT = c -> new DefaultBlockCodec<>(FlatArrayCodec.SHORT_BIG_ENDIAN, ShortArrayDataBlock::new, c); + private static final BlockCodecFactory INT = c -> new DefaultBlockCodec<>(FlatArrayCodec.INT_BIG_ENDIAN, IntArrayDataBlock::new, c); + private static final BlockCodecFactory LONG = c -> new DefaultBlockCodec<>(FlatArrayCodec.LONG_BIG_ENDIAN, LongArrayDataBlock::new, c); + private static final BlockCodecFactory FLOAT = c -> new DefaultBlockCodec<>(FlatArrayCodec.FLOAT_BIG_ENDIAN, FloatArrayDataBlock::new, c); + private static final BlockCodecFactory DOUBLE = c -> new DefaultBlockCodec<>(FlatArrayCodec.DOUBLE_BIG_ENDIAN, DoubleArrayDataBlock::new, c); + private static final BlockCodecFactory STRING = c -> new StringBlockCodec(c); + private static final BlockCodecFactory OBJECT = c -> new ObjectBlockCodec(c); - private N5DataBlockSerializers() {} + private N5BlockCodecs() {} - public static DataBlockSerializer create( + public static BlockCodec create( final DataType dataType, - final BytesCodec codec) { + final DataCodec codec) { - final DataBlockSerializerFactory factory; + final BlockCodecFactory factory; switch (dataType) { case UINT8: case INT8: - factory = N5DataBlockSerializers.BYTE; + factory = N5BlockCodecs.BYTE; break; case UINT16: case INT16: - factory = N5DataBlockSerializers.SHORT; + factory = N5BlockCodecs.SHORT; break; case UINT32: case INT32: - factory = N5DataBlockSerializers.INT; + factory = N5BlockCodecs.INT; break; case UINT64: case INT64: - factory = N5DataBlockSerializers.LONG; + factory = N5BlockCodecs.LONG; break; case FLOAT32: - factory = N5DataBlockSerializers.FLOAT; + factory = N5BlockCodecs.FLOAT; break; case FLOAT64: - factory = N5DataBlockSerializers.DOUBLE; + factory = N5BlockCodecs.DOUBLE; break; case STRING: - factory = N5DataBlockSerializers.STRING; + factory = N5BlockCodecs.STRING; break; case OBJECT: - factory = N5DataBlockSerializers.OBJECT; + factory = N5BlockCodecs.OBJECT; break; default: throw new IllegalArgumentException("Unsupported data type: " + dataType); } @SuppressWarnings("unchecked") - final DataBlockSerializerFactory tFactory = (DataBlockSerializerFactory)factory; + final BlockCodecFactory tFactory = (BlockCodecFactory)factory; return tFactory.create(codec); } - private interface DataBlockSerializerFactory { + private interface BlockCodecFactory { /** - * Create a {@link DataBlockSerializer} that uses the specified {@code - * BytesCodec} and de/serializes {@code DataBlock} to N5 format. + * Create a {@link BlockCodec} that uses the specified {@code + * DataCodec} and de/serializes {@code DataBlock} to N5 format. * - * @return N5 {@code DataBlockSerializer} for the specified {@code codec} + * @return N5 {@code BlockCodec} for the specified {@code codec} */ - DataBlockSerializer create(BytesCodec codec); + BlockCodec create(DataCodec codec); } - abstract static class N5AbstractDataBlockSerializer implements DataBlockSerializer { + abstract static class N5AbstractDataBlockSerializer implements BlockCodec { - private final FlatArraySerializer dataCodec; + private final FlatArrayCodec dataCodec; private final DataBlockFactory dataBlockFactory; - private final BytesCodec codec; + private final DataCodec codec; - N5AbstractDataBlockSerializer(FlatArraySerializer dataCodec, DataBlockFactory dataBlockFactory, BytesCodec codec) { + N5AbstractDataBlockSerializer(FlatArrayCodec dataCodec, DataBlockFactory dataBlockFactory, DataCodec codec) { this.dataCodec = dataCodec; this.dataBlockFactory = dataBlockFactory; this.codec = codec; @@ -168,12 +168,12 @@ public DataBlock decode(final ReadData readData, final long[] gridPosition) t /** * DataBlockCodec for all N5 data types, except STRING and OBJECT */ - private static class DefaultDataBlockSerializer extends N5AbstractDataBlockSerializer { + private static class DefaultBlockCodec extends N5AbstractDataBlockSerializer { - DefaultDataBlockSerializer( - final FlatArraySerializer dataCodec, + DefaultBlockCodec( + final FlatArrayCodec dataCodec, final DataBlockFactory dataBlockFactory, - final BytesCodec codec) { + final DataCodec codec) { super(dataCodec, dataBlockFactory, codec); } @@ -194,11 +194,11 @@ protected BlockHeader decodeBlockHeader(final InputStream in) throws N5IOExcepti /** * DataBlockCodec for N5 data type STRING */ - private static class StringDataBlockSerializer extends N5AbstractDataBlockSerializer { + private static class StringBlockCodec extends N5AbstractDataBlockSerializer { - StringDataBlockSerializer(final BytesCodec codec) { + StringBlockCodec(final DataCodec codec) { - super(FlatArraySerializer.STRING, StringDataBlock::new, codec); + super(FlatArrayCodec.STRING, StringDataBlock::new, codec); } @Override @@ -217,11 +217,11 @@ protected BlockHeader decodeBlockHeader(final InputStream in) throws N5IOExcepti /** * DataBlockCodec for N5 data type OBJECT */ - private static class ObjectDataBlockSerializer extends N5AbstractDataBlockSerializer { + private static class ObjectBlockCodec extends N5AbstractDataBlockSerializer { - ObjectDataBlockSerializer(final BytesCodec codec) { + ObjectBlockCodec(final DataCodec codec) { - super(FlatArraySerializer.OBJECT, ByteArrayDataBlock::new, codec); + super(FlatArrayCodec.OBJECT, ByteArrayDataBlock::new, codec); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java similarity index 78% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java index 4516a947b..b1ad6bdf7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java @@ -14,8 +14,8 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; -@NameConfig.Name(value = RawBytesArrayCodec.TYPE) -public class RawBytesArrayCodec implements ArrayCodec { +@NameConfig.Name(value = RawBlockCodecInfo.TYPE) +public class RawBlockCodecInfo implements BlockCodecInfo { private static final long serialVersionUID = 3282569607795127005L; @@ -24,12 +24,12 @@ public class RawBytesArrayCodec implements ArrayCodec { @NameConfig.Parameter(value = "endian", optional = true) private final ByteOrder byteOrder; - public RawBytesArrayCodec() { + public RawBlockCodecInfo() { this(ByteOrder.BIG_ENDIAN); } - public RawBytesArrayCodec(final ByteOrder byteOrder) { + public RawBlockCodecInfo(final ByteOrder byteOrder) { this.byteOrder = byteOrder; } @@ -45,14 +45,14 @@ public ByteOrder getByteOrder() { } @Override - public DataBlockSerializer initialize(final DatasetAttributes attributes, final BytesCodec... bytesCodecs) { + public BlockCodec create(final DatasetAttributes attributes, final DataCodec... bytesCodecs) { ensureValidByteOrder(attributes.getDataType(), getByteOrder()); - return RawDataBlockSerializers.create(attributes.getDataType(), byteOrder, attributes.getBlockSize(), BytesCodec.concatenate(bytesCodecs)); + return RawBlockCodecs.create(attributes.getDataType(), byteOrder, attributes.getBlockSize(), DataCodec.concatenate(bytesCodecs)); } - public static final RawBytesArrayCodec.ByteOrderAdapter byteOrderAdapter = new ByteOrderAdapter(); + public static final RawBlockCodecInfo.ByteOrderAdapter byteOrderAdapter = new ByteOrderAdapter(); public static void ensureValidByteOrder(final DataType dataType, final ByteOrder byteOrder) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java new file mode 100644 index 000000000..24907bbab --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java @@ -0,0 +1,119 @@ +package org.janelia.saalfeldlab.n5.codec; + +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock; +import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; +import org.janelia.saalfeldlab.n5.IntArrayDataBlock; +import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; +import org.janelia.saalfeldlab.n5.StringDataBlock; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +import java.nio.ByteOrder; + +public class RawBlockCodecs { + + private static final BlockCodecFactory BYTE = (byteOrder, blockSize, codec) -> new RawBlockCodec<>(FlatArrayCodec.BYTE, ByteArrayDataBlock::new, blockSize, codec); + private static final BlockCodecFactory SHORT = (byteOrder, blockSize, codec) -> new RawBlockCodec<>(FlatArrayCodec.SHORT(byteOrder), ShortArrayDataBlock::new, blockSize, codec); + private static final BlockCodecFactory INT = (byteOrder, blockSize, codec) -> new RawBlockCodec<>(FlatArrayCodec.INT(byteOrder), IntArrayDataBlock::new, blockSize, codec); + private static final BlockCodecFactory LONG = (byteOrder, blockSize, codec) -> new RawBlockCodec<>(FlatArrayCodec.LONG(byteOrder), LongArrayDataBlock::new, blockSize, codec); + private static final BlockCodecFactory FLOAT = (byteOrder, blockSize, codec) -> new RawBlockCodec<>(FlatArrayCodec.FLOAT(byteOrder), FloatArrayDataBlock::new, blockSize, codec); + private static final BlockCodecFactory DOUBLE = (byteOrder, blockSize, codec) -> new RawBlockCodec<>(FlatArrayCodec.DOUBLE(byteOrder), DoubleArrayDataBlock::new, blockSize, codec); + private static final BlockCodecFactory STRING = (byteOrder, blockSize, codec) -> new RawBlockCodec<>(FlatArrayCodec.ZARR_STRING, StringDataBlock::new, blockSize, codec); + private static final BlockCodecFactory OBJECT = (byteOrder, blockSize, codec) -> new RawBlockCodec<>(FlatArrayCodec.OBJECT, ByteArrayDataBlock::new, blockSize, codec); + + private RawBlockCodecs() {} + + public static BlockCodec create( + final DataType dataType, + final ByteOrder byteOrder, + final int[] blockSize, + final DataCodec codec) { + final BlockCodecFactory factory; + switch (dataType) { + case UINT8: + case INT8: + factory = RawBlockCodecs.BYTE; + break; + case UINT16: + case INT16: + factory = RawBlockCodecs.SHORT; + break; + case UINT32: + case INT32: + factory = RawBlockCodecs.INT; + break; + case UINT64: + case INT64: + factory = RawBlockCodecs.LONG; + break; + case FLOAT32: + factory = RawBlockCodecs.FLOAT; + break; + case FLOAT64: + factory = RawBlockCodecs.DOUBLE; + break; + case STRING: + factory = RawBlockCodecs.STRING; + break; + // TODO: What about OBJECT? + default: + throw new IllegalArgumentException("Unsupported data type: " + dataType); + } + final BlockCodecFactory tFactory = (BlockCodecFactory) factory; + return tFactory.create(byteOrder, blockSize, codec); + } + + private interface BlockCodecFactory { + + /** + * Create a {@link BlockCodec} that uses the specified {@code + * ByteOrder} and {@code BytesCodes} and de/serializes {@code + * DataBlock} of the specified {@code blockSize} to raw format. + * + * @return Raw {@code BlockCodec} + */ + BlockCodec create(ByteOrder byteOrder, int[] blockSize, DataCodec codec); + } + + private static class RawBlockCodec implements BlockCodec { + + private final FlatArrayCodec dataCodec; + private final DataBlock.DataBlockFactory dataBlockFactory; + private final int[] blockSize; + private final int numElements; + private final DataCodec codec; + + RawBlockCodec( + final FlatArrayCodec dataCodec, + final DataBlock.DataBlockFactory dataBlockFactory, + final int[] blockSize, + final DataCodec codec) { + + this.dataCodec = dataCodec; + this.dataBlockFactory = dataBlockFactory; + this.blockSize = blockSize; + this.numElements = DataBlock.getNumElements(blockSize); + this.codec = codec; + } + + @Override + public ReadData encode(DataBlock dataBlock) { + + return ReadData.from(out -> { + final ReadData blockData = dataCodec.serialize(dataBlock.getData()); + codec.encode(blockData).writeTo(out); + }); + } + + @Override + public DataBlock decode(ReadData readData, long[] gridPosition) { + + final ReadData decodeData = codec.decode(readData); + final T data = dataCodec.deserialize(decodeData, numElements); + return dataBlockFactory.createDataBlock(blockSize, gridPosition, data); + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawDataBlockSerializers.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawDataBlockSerializers.java deleted file mode 100644 index 609b2e583..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawDataBlockSerializers.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.janelia.saalfeldlab.n5.codec; - -import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock; -import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; -import org.janelia.saalfeldlab.n5.IntArrayDataBlock; -import org.janelia.saalfeldlab.n5.LongArrayDataBlock; -import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; -import org.janelia.saalfeldlab.n5.StringDataBlock; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - -import java.nio.ByteOrder; - -public class RawDataBlockSerializers { - - private static final DataBlockSerializerFactory BYTE = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.BYTE, ByteArrayDataBlock::new, blockSize, codec); - private static final DataBlockSerializerFactory SHORT = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.SHORT(byteOrder), ShortArrayDataBlock::new, blockSize, codec); - private static final DataBlockSerializerFactory INT = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.INT(byteOrder), IntArrayDataBlock::new, blockSize, codec); - private static final DataBlockSerializerFactory LONG = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.LONG(byteOrder), LongArrayDataBlock::new, blockSize, codec); - private static final DataBlockSerializerFactory FLOAT = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.FLOAT(byteOrder), FloatArrayDataBlock::new, blockSize, codec); - private static final DataBlockSerializerFactory DOUBLE = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.DOUBLE(byteOrder), DoubleArrayDataBlock::new, blockSize, codec); - private static final DataBlockSerializerFactory STRING = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.ZARR_STRING, StringDataBlock::new, blockSize, codec); - private static final DataBlockSerializerFactory OBJECT = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.OBJECT, ByteArrayDataBlock::new, blockSize, codec); - - private RawDataBlockSerializers() {} - - public static DataBlockSerializer create( - final DataType dataType, - final ByteOrder byteOrder, - final int[] blockSize, - final BytesCodec codec) { - final DataBlockSerializerFactory factory; - switch (dataType) { - case UINT8: - case INT8: - factory = RawDataBlockSerializers.BYTE; - break; - case UINT16: - case INT16: - factory = RawDataBlockSerializers.SHORT; - break; - case UINT32: - case INT32: - factory = RawDataBlockSerializers.INT; - break; - case UINT64: - case INT64: - factory = RawDataBlockSerializers.LONG; - break; - case FLOAT32: - factory = RawDataBlockSerializers.FLOAT; - break; - case FLOAT64: - factory = RawDataBlockSerializers.DOUBLE; - break; - case STRING: - factory = RawDataBlockSerializers.STRING; - break; - // TODO: What about OBJECT? - default: - throw new IllegalArgumentException("Unsupported data type: " + dataType); - } - final DataBlockSerializerFactory tFactory = (DataBlockSerializerFactory) factory; - return tFactory.create(byteOrder, blockSize, codec); - } - - private interface DataBlockSerializerFactory { - - /** - * Create a {@link DataBlockSerializer} that uses the specified {@code - * ByteOrder} and {@code BytesCodes} and de/serializes {@code - * DataBlock} of the specified {@code blockSize} to raw format. - * - * @return Raw {@code DataBlockSerializer} - */ - DataBlockSerializer create(ByteOrder byteOrder, int[] blockSize, BytesCodec codec); - } - - private static class RawDataBlockSerializer implements DataBlockSerializer { - - private final FlatArraySerializer dataCodec; - private final DataBlock.DataBlockFactory dataBlockFactory; - private final int[] blockSize; - private final int numElements; - private final BytesCodec codec; - - RawDataBlockSerializer( - final FlatArraySerializer dataCodec, - final DataBlock.DataBlockFactory dataBlockFactory, - final int[] blockSize, - final BytesCodec codec) { - - this.dataCodec = dataCodec; - this.dataBlockFactory = dataBlockFactory; - this.blockSize = blockSize; - this.numElements = DataBlock.getNumElements(blockSize); - this.codec = codec; - } - - @Override - public ReadData encode(DataBlock dataBlock) { - - return ReadData.from(out -> { - final ReadData blockData = dataCodec.serialize(dataBlock.getData()); - codec.encode(blockData).writeTo(out); - }); - } - - @Override - public DataBlock decode(ReadData readData, long[] gridPosition) { - - final ReadData decodeData = codec.decode(readData); - final T data = dataCodec.deserialize(decodeData, numElements); - return dataBlockFactory.createDataBlock(blockSize, gridPosition, data); - } - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java index fc0af52f7..b72ce626a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java @@ -8,17 +8,16 @@ import java.util.zip.CheckedOutputStream; import java.util.zip.Checksum; -import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** - * A {@link Codec} that appends a checksum to data when encoding and can validate against that checksum when decoding. + * A {@link CodecInfo} that appends a checksum to data when encoding and can validate against that checksum when decoding. */ -public abstract class ChecksumCodec implements BytesCodec, DeterministicSizeCodec { +public abstract class ChecksumCodec implements DataCodec, DeterministicSizeCodec { private static final long serialVersionUID = 3141427377277375077L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index a298897a1..e5fa6ddba 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -49,7 +49,7 @@ * accessed (e.g., {@link #allBytes()}, {@link #writeTo(OutputStream)}). *

      * {@code ReadData} can be {@code encoded} and {@code decoded} with a {@code - * Codec}, which will also be lazy if possible. + * CodecInfo}, which will also be lazy if possible. */ public interface ReadData { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java index 461f1945b..663c12602 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java @@ -4,12 +4,12 @@ import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.codec.ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; -import org.janelia.saalfeldlab.n5.codec.DataBlockSerializer; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; +import org.janelia.saalfeldlab.n5.codec.DataCodec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter; -import org.janelia.saalfeldlab.n5.codec.RawBytesArrayCodec; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @@ -17,7 +17,7 @@ public class BlockAsShardCodec extends ShardingCodec { private static final LongArrayDataBlock BLOCK_AS_SHARD_INDEX = new LongArrayDataBlock(new int[0], new long[0], new long[]{0, -1}); - private static final DataBlockSerializer BLOCK_AS_SHARD_BLOCK_SERIALIZER = new DataBlockSerializer() { + private static final BlockCodec BLOCK_AS_SHARD_BLOCK_SERIALIZER = new BlockCodec() { @Override public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { @@ -30,14 +30,14 @@ public class BlockAsShardCodec extends ShardingCodec { } }; - private static final RawBytesArrayCodec VIRTUAL_SHARD_INDEX_CODEC = new RawBytesArrayCodec() { + private static final RawBlockCodecInfo VIRTUAL_SHARD_INDEX_CODEC = new RawBlockCodecInfo() { - @Override public DataBlockSerializer initialize(DatasetAttributes attributes, BytesCodec... bytesCodecs) { + @Override public BlockCodec create(DatasetAttributes attributes, DataCodec... bytesCodecs) { return BLOCK_AS_SHARD_BLOCK_SERIALIZER; } }; - private static final BytesCodec[] EMPTY_SHARD_CODECS = new BytesCodec[0]; + private static final DataCodec[] EMPTY_SHARD_CODECS = new DataCodec[0]; private static final DeterministicSizeCodec[] NO_OP_INDEX_CODECS = new DeterministicSizeCodec[0]; private static final IndexCodecAdapter BLOCK_AS_SHARD_INDEX_CODEC_ADAPTER = new IndexCodecAdapter(VIRTUAL_SHARD_INDEX_CODEC, NO_OP_INDEX_CODECS) { @@ -48,9 +48,9 @@ public class BlockAsShardCodec extends ShardingCodec { } }; - final ArrayCodec datasetArrayCodec; + final BlockCodecInfo datasetArrayCodec; - public BlockAsShardCodec(ArrayCodec datasetArrayCodec) { + public BlockAsShardCodec(BlockCodecInfo datasetArrayCodec) { super(null, EMPTY_SHARD_CODECS, BLOCK_AS_SHARD_INDEX_CODEC_ADAPTER, IndexLocation.START); this.datasetArrayCodec = datasetArrayCodec; @@ -63,7 +63,7 @@ public ShardIndex createIndex(DatasetAttributes attributes) { } @Override - public ArrayCodec getArrayCodec() { + public BlockCodecInfo getArrayCodec() { return datasetArrayCodec; } @@ -81,10 +81,10 @@ public long[] getKeyPositionForBlock(DatasetAttributes attributes, long... block } @Override - public DataBlockSerializer initialize(DatasetAttributes attributes, BytesCodec... codecs) { + public BlockCodec create(DatasetAttributes attributes, DataCodec... codecs) { - dataBlockSerializer = datasetArrayCodec.initialize(attributes, codecs); - return (DataBlockSerializer)dataBlockSerializer; + dataBlockSerializer = datasetArrayCodec.create(attributes, codecs); + return (BlockCodec)dataBlockSerializer; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index 419cd2d62..01cdb2f1f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -9,8 +9,7 @@ import org.apache.commons.io.output.CountingOutputStream; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.codec.ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.DataBlockSerializer; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.util.GridIterator; @@ -139,7 +138,7 @@ default ReadData createReadData() throws N5IOException { final DatasetAttributes datasetAttributes = getDatasetAttributes(); ShardingCodec shardingCodec = datasetAttributes.getShardingCodec(); - final DataBlockSerializer arrayCodec = shardingCodec.getDataBlockSerializer(); + final BlockCodec arrayCodec = shardingCodec.getDataBlockSerializer(); final ShardIndex index = createIndex(); final long indexSize = index.numBytes(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index fdcb3114a..0b2533276 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -4,18 +4,15 @@ import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; -import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.codec.ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.DataBlockSerializer; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UncheckedIOException; import java.util.Arrays; import java.util.stream.IntStream; @@ -100,7 +97,7 @@ public ShardIndex(int[] shardBlockGridSize, final IndexCodecAdapter indexCodecAd * @param location where the index is stored (START or END of shard) * @param arrayCodec arrayCodec for the IndexCodecAdapter */ - public ShardIndex(int[] shardBlockGridSize, IndexLocation location, final ArrayCodec arrayCodec) { + public ShardIndex(int[] shardBlockGridSize, IndexLocation location, final BlockCodecInfo arrayCodec) { this(shardBlockGridSize, location, new IndexCodecAdapter(arrayCodec)); } @@ -111,7 +108,7 @@ public ShardIndex(int[] shardBlockGridSize, IndexLocation location, final ArrayC * @param shardBlockGridSize the dimensions of the block grid within the shard * @param arrayCodec arrayCodec for the IndexCodecAdapter */ - public ShardIndex(int[] shardBlockGridSize, final ArrayCodec arrayCodec) { + public ShardIndex(int[] shardBlockGridSize, final BlockCodecInfo arrayCodec) { this(shardBlockGridSize, IndexLocation.END, arrayCodec); } @@ -330,7 +327,7 @@ public static void read( final ReadData indexData, final ShardIndex index ) { public static void read(InputStream indexIn, final ShardIndex index) throws N5IOException { final ReadData dataIn = ReadData.from(indexIn); - final DataBlockSerializer shardIndexCodec = index.indexAttributes.getDataBlockSerializer(); + final BlockCodec shardIndexCodec = index.indexAttributes.getDataBlockSerializer(); final DataBlock indexBlock = shardIndexCodec.decode(dataIn, index.gridPosition); System.arraycopy(indexBlock.getData(), 0, index.data, 0, (int)index.data.length); } @@ -344,7 +341,7 @@ public static void read(InputStream indexIn, final ShardIndex index) throws N5IO */ public static void write( final OutputStream outputStream, final ShardIndex index ) throws N5IOException { - final DataBlockSerializer dataBlockSerializer = index.indexAttributes.getDataBlockSerializer(); + final BlockCodec dataBlockSerializer = index.indexAttributes.getDataBlockSerializer(); dataBlockSerializer.encode(index).writeTo(outputStream); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 93f4ef91f..5ca6e3a58 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -10,10 +10,10 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.codec.ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; -import org.janelia.saalfeldlab.n5.codec.Codec; -import org.janelia.saalfeldlab.n5.codec.DataBlockSerializer; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; +import org.janelia.saalfeldlab.n5.codec.DataCodec; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.N5Annotations; @@ -23,7 +23,7 @@ import java.util.Objects; @NameConfig.Name(ShardingCodec.TYPE) -public class ShardingCodec implements ArrayCodec { +public class ShardingCodec implements BlockCodecInfo { private static final long serialVersionUID = -5879797314954717810L; @@ -44,7 +44,7 @@ public enum IndexLocation { private final int[] blockSize; @NameConfig.Parameter(CODECS_KEY) - private final Codec[] codecs; + private final CodecInfo[] codecs; @NameConfig.Parameter(INDEX_CODECS_KEY) private final IndexCodecAdapter indexCodecs; @@ -52,7 +52,7 @@ public enum IndexLocation { @NameConfig.Parameter(value = INDEX_LOCATION_KEY, optional = true) private final IndexLocation indexLocation; - protected DataBlockSerializer dataBlockSerializer = null; + protected BlockCodec dataBlockSerializer = null; /** * Used via reflections by the NameConfig serializer. @@ -68,7 +68,7 @@ private ShardingCodec() { public ShardingCodec( final int[] blockSize, - final Codec[] codecs, + final CodecInfo[] codecs, final IndexCodecAdapter indexCodecs, final IndexLocation indexLocation) { @@ -80,8 +80,8 @@ public ShardingCodec( public ShardingCodec( final int[] blockSize, - final Codec[] codecs, - final Codec[] indexCodecs, + final CodecInfo[] codecs, + final CodecInfo[] indexCodecs, final IndexLocation indexLocation) { this(blockSize, codecs, IndexCodecAdapter.create(indexCodecs), indexLocation); @@ -92,25 +92,25 @@ public IndexLocation getIndexLocation() { return indexLocation; } - public ArrayCodec getArrayCodec() { + public BlockCodecInfo getArrayCodec() { Objects.requireNonNull(codecs); if (codecs.length == 0) - throw new IllegalArgumentException("Sharding Codec requires a single ArrayCodec. None found."); + throw new IllegalArgumentException("Sharding CodecInfo requires a single BlockCodecInfo. None found."); - return (ArrayCodec)codecs[0]; + return (BlockCodecInfo)codecs[0]; } - public DataBlockSerializer getDataBlockSerializer() { + public BlockCodec getDataBlockSerializer() { - return (DataBlockSerializer)dataBlockSerializer; + return (BlockCodec)dataBlockSerializer; } - public BytesCodec[] getCodecs() { + public DataCodec[] getCodecs() { Objects.requireNonNull(codecs); - final BytesCodec[] bytesCodecs = new BytesCodec[codecs.length - 1]; + final DataCodec[] bytesCodecs = new DataCodec[codecs.length - 1]; for (int i = 1; i < codecs.length; i++) - bytesCodecs[i-1] = (BytesCodec)codecs[i]; + bytesCodecs[i-1] = (DataCodec)codecs[i]; return bytesCodecs; } @@ -133,11 +133,11 @@ public long[] getKeyPositionForBlock(DatasetAttributes attributes, final long... } @Override - public DataBlockSerializer initialize(DatasetAttributes attributes, final BytesCodec[] codecs) { + public BlockCodec create(DatasetAttributes attributes, final DataCodec[] codecs) { this.attributes = attributes; - this.dataBlockSerializer = getArrayCodec().initialize(attributes, getCodecs()); - return ((DataBlockSerializer)dataBlockSerializer); + this.dataBlockSerializer = getArrayCodec().create(attributes, getCodecs()); + return ((BlockCodec)dataBlockSerializer); } public ReadData encode(DataBlock dataBlock) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java index 620b3072c..9d4b46174 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java @@ -42,10 +42,10 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; -import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; -import org.janelia.saalfeldlab.n5.codec.N5ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.RawBytesArrayCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.shard.InMemoryShard; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; @@ -161,8 +161,8 @@ public void testShardProperties() { DataType.UINT8, new ShardingCodec( blkSize, - new Codec[]{new N5ArrayCodec()}, - new DeterministicSizeCodec[]{new RawBytesArrayCodec()}, + new CodecInfo[]{new N5BlockCodecInfo()}, + new DeterministicSizeCodec[]{new RawBlockCodecInfo()}, IndexLocation.END ) ); @@ -196,8 +196,8 @@ public void testShardGrouping() { DataType.UINT8, new ShardingCodec( blkSize, - new Codec[]{ new N5ArrayCodec() }, - new DeterministicSizeCodec[]{new RawBytesArrayCodec()}, + new CodecInfo[]{ new N5BlockCodecInfo() }, + new DeterministicSizeCodec[]{new RawBlockCodecInfo()}, IndexLocation.END ) ); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java index ed6259f2d..d3f996ede 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java @@ -209,7 +209,7 @@ public void testChildManagement() { // Test addChildIfPresent on cached parent without children list cache.exists("parent2", null); - children = cache.list("parent2"); // initialize children array + children = cache.list("parent2"); // create children array cache.addChildIfPresent("parent2", "child"); children = cache.list("parent2"); assertTrue(Arrays.asList(children).contains("child")); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/ArrayCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/ArrayCodecTests.java index 017061cc8..9decefde8 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/ArrayCodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/ArrayCodecTests.java @@ -19,8 +19,6 @@ import org.janelia.saalfeldlab.n5.IntArrayDataBlock; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; -import org.janelia.saalfeldlab.n5.codec.ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.junit.Test; @@ -32,7 +30,7 @@ public class ArrayCodecTests { final int[] blockSize = {11, 7, 5}; private final BitShiftBytesCodec shiftCodec = new BitShiftBytesCodec(3); private final GzipCompression compressor = new GzipCompression(); - private final BytesCodec[][] bytesCodecs = new BytesCodec[][]{ + private final DataCodec[][] bytesCodecs = new DataCodec[][]{ {}, // empty: "raw" compression {compressor}, {shiftCodec}, @@ -50,14 +48,14 @@ public class ArrayCodecTests { @Test public void testN5ArrayCodec() throws Exception { for (DataType dataType : dataTypes) { - for (BytesCodec[] codecs : bytesCodecs) { + for (DataCodec[] codecs : bytesCodecs) { final DatasetAttributes attributes = new DatasetAttributes( new long[]{32, 32, 32}, blockSize, blockSize, dataType, - new N5ArrayCodec(), + new N5BlockCodecInfo(), codecs); testArrayCodecHelper(attributes); @@ -67,13 +65,13 @@ public void testN5ArrayCodec() throws Exception { @Test public void testRawBytesArrayCodecCodec() throws Exception { - // Test RawBytesArrayCodec codec with different byte orders and DataTypes + // Test RawBlockCodecInfo codec with different byte orders and DataTypes final ByteOrder[] byteOrders = {ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}; for (DataType dataType : dataTypes) { for (ByteOrder byteOrder : byteOrders) { - for (BytesCodec[] codecs : bytesCodecs) { + for (DataCodec[] codecs : bytesCodecs) { - final RawBytesArrayCodec codec = new RawBytesArrayCodec(byteOrder); + final RawBlockCodecInfo codec = new RawBlockCodecInfo(byteOrder); final DatasetAttributes attributes = new DatasetAttributes( new long[]{32, 32, 32}, blockSize, //shardSize @@ -95,7 +93,7 @@ private void testArrayCodecHelper(DatasetAttributes attributes) throws Exce // Create appropriate data block based on type DataBlock originalBlock = ((DataBlock)createRandomDataBlock(dataType, blockSize, gridPosition)); - final DataBlockSerializer codec = attributes.getDataBlockSerializer(); + final BlockCodec codec = attributes.getDataBlockSerializer(); // Test encode/decode roundtrip final ReadData encoded = codec.encode(originalBlock); @@ -115,7 +113,7 @@ public void testEmptyBlock() throws Exception { // Test handling of empty blocks final int[] blockSize = {0, 0}; final long[] gridPosition = {0, 0}; - final N5ArrayCodec arrayCodec = new N5ArrayCodec(); + final N5BlockCodecInfo arrayCodec = new N5BlockCodecInfo(); final DatasetAttributes attributes = new DatasetAttributes( new long[]{64, 64}, new int[]{8, 8}, @@ -123,7 +121,7 @@ public void testEmptyBlock() throws Exception { DataType.UINT8, arrayCodec); - final DataBlockSerializer codec = attributes.getDataBlockSerializer(); + final BlockCodec codec = attributes.getDataBlockSerializer(); // Test encode/decode final ByteArrayDataBlock emptyBlock = new ByteArrayDataBlock(blockSize, gridPosition, new byte[0]); @@ -142,7 +140,7 @@ public void testEncodedSizeCalculation() throws Exception { blockSize, blockSize, DataType.INT16, - new N5ArrayCodec()); + new N5BlockCodecInfo()); final DatasetAttributes rawArrayAttrs = new DatasetAttributes( @@ -150,12 +148,12 @@ public void testEncodedSizeCalculation() throws Exception { blockSize, blockSize, DataType.INT16, - new RawBytesArrayCodec()); + new RawBlockCodecInfo()); // Calculate expected sizes final long rawDataSize = blockSize[0] * blockSize[1] * 2; // INT16 has 2 bytes per element - // N5ArrayCodec adds a header + // N5BlockCodecInfo adds a header // the estimate of the encoded size final long n5EncodedSize = n5ArrayAttrs.getArrayCodec().encodedSize(rawDataSize); assertTrue("N5 encoded size should be larger than raw size", n5EncodedSize > rawDataSize); @@ -164,7 +162,7 @@ public void testEncodedSizeCalculation() throws Exception { ReadData n5EncodedDataBlock = n5ArrayAttrs.getDataBlockSerializer().encode(dataBlock); assertEquals("N5 actual encoded size should equal estimated size", n5EncodedSize, n5EncodedDataBlock.length()); - // RawBytesArrayCodec should not change size + // RawBlockCodecInfo should not change size final long rawEncodedSize = rawArrayAttrs.getArrayCodec().encodedSize(rawDataSize); assertEquals("Raw encoded size should equal input size", rawDataSize, rawEncodedSize); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java index 58b662a9c..eeb2f4250 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java @@ -10,7 +10,6 @@ import java.util.function.IntUnaryOperator; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.ReadData.OutputStreamOperator; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @@ -62,9 +61,9 @@ public int read() throws IOException { expected[i] = (byte)(2 * bytes[i] + 3); } - final BytesCodec a = new ByteFunctionCodec(x -> 2 * x, x -> x / 2); - final BytesCodec b = new ByteFunctionCodec(x -> x + 3, x -> x - 3 ); - final ConcatenatedBytesCodec ab = new ConcatenatedBytesCodec(new BytesCodec[]{a, b}); + final DataCodec a = new ByteFunctionCodec(x -> 2 * x, x -> x / 2); + final DataCodec b = new ByteFunctionCodec(x -> x + 3, x -> x - 3 ); + final ConcatenatedDataCodec ab = new ConcatenatedDataCodec(new DataCodec[]{a, b}); final ReadData encodedData = ab.encode(data).materialize(); assertArrayEquals(expected, encodedData.allBytes()); @@ -73,7 +72,7 @@ public int read() throws IOException { assertArrayEquals(bytes, decodedData.allBytes()); } - public static class ByteFunctionCodec implements BytesCodec { + public static class ByteFunctionCodec implements DataCodec { IntUnaryOperator encoder; IntUnaryOperator decoder; @@ -116,7 +115,7 @@ public void write(int b) throws IOException { } @NameConfig.Name(BitShiftBytesCodec.TYPE) - public static class BitShiftBytesCodec implements BytesCodec { + public static class BitShiftBytesCodec implements DataCodec { private static final String TYPE = "bitshift"; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java index 39c68d6b5..0a164e499 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java @@ -11,9 +11,9 @@ import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.RawCompression; -import org.janelia.saalfeldlab.n5.codec.N5ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.RawBytesArrayCodec; -import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; @@ -36,8 +36,8 @@ public static void shardBlockIterator() { DataType.UINT8, new ShardingCodec( new int[] {2, 2}, - new Codec[] { new N5ArrayCodec() }, - new DeterministicSizeCodec[] { new RawBytesArrayCodec() }, + new CodecInfo[] { new N5BlockCodecInfo() }, + new DeterministicSizeCodec[] { new RawBlockCodecInfo() }, IndexLocation.END )); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index fc4f17bac..38ceee229 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -32,7 +32,6 @@ import com.google.gson.JsonElement; import org.janelia.saalfeldlab.n5.CachedGsonKeyValueN5Reader; import org.janelia.saalfeldlab.n5.CachedGsonKeyValueN5Writer; -import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; @@ -40,7 +39,7 @@ import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.shard.Shard; import java.io.Serializable; @@ -309,7 +308,7 @@ public HttpRead writer.writeShard(path, datasetAttributes, shard); } - @Override public void createDataset(String datasetPath, long[] dimensions, int[] blockSize, DataType dataType, Codec... codecs) throws N5Exception { + @Override public void createDataset(String datasetPath, long[] dimensions, int[] blockSize, DataType dataType, CodecInfo... codecs) throws N5Exception { writer.createDataset(datasetPath, dimensions, blockSize, dataType, codecs); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java index 6fc37b45e..e5cf3a913 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java @@ -5,10 +5,9 @@ import org.janelia.saalfeldlab.n5.GsonUtils; import org.janelia.saalfeldlab.n5.GzipCompression; -import org.janelia.saalfeldlab.n5.codec.ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.DataCodec; import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; -import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.IdentityCodec; import org.junit.Before; import org.junit.Test; @@ -46,7 +45,7 @@ public void testCodecSerialization() { JsonElement.class); assertEquals("bitshift json", expectedBitShift, bitShiftJson); - final BytesCodec deserializedCodec = gson.fromJson(bitShiftJson, BytesCodec.class); + final DataCodec deserializedCodec = gson.fromJson(bitShiftJson, DataCodec.class); // Verify deserialized codec assertEquals("Deserialized codec should equal original", codec, deserializedCodec); } @@ -54,7 +53,7 @@ public void testCodecSerialization() { @Test public void testSerializeCodecArray() { - Codec[] codecs = new Codec[]{ + CodecInfo[] codecs = new CodecInfo[]{ new IdentityCodec() }; JsonArray jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); @@ -63,11 +62,11 @@ public void testSerializeCodecArray() { JsonElement.class); assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); - Codec[] codecsDeserialized = gson.fromJson(expected, Codec[].class); + CodecInfo[] codecsDeserialized = gson.fromJson(expected, CodecInfo[].class); assertEquals("codecs length not 1", 1, codecsDeserialized.length); assertTrue("first codec not identity", codecsDeserialized[0] instanceof IdentityCodec); - codecs = new Codec[]{ + codecs = new CodecInfo[]{ new GzipCompression() }; jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); @@ -76,7 +75,7 @@ public void testSerializeCodecArray() { JsonElement.class); assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); - codecsDeserialized = gson.fromJson(expected, Codec[].class); + codecsDeserialized = gson.fromJson(expected, CodecInfo[].class); assertEquals("codecs length not 1", 1, codecsDeserialized.length); assertTrue("second codec not gzip", codecsDeserialized[0] instanceof GzipCompression); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index 2e738f7ba..cd9565858 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -2,12 +2,10 @@ import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; -import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5FSTest; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter; -import org.janelia.saalfeldlab.n5.codec.N5ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.RawBytesArrayCodec; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; @@ -37,7 +35,7 @@ public void testOffsetIndex() { int[] shardBlockGridSize = new int[]{5, 4, 3}; ShardIndex index = new ShardIndex( shardBlockGridSize, - IndexLocation.END, new RawBytesArrayCodec()); + IndexLocation.END, new RawBlockCodecInfo()); GridIterator it = new GridIterator(shardBlockGridSize); int i = 0; @@ -50,7 +48,7 @@ public void testOffsetIndex() { shardBlockGridSize = new int[]{5, 4, 3, 13}; index = new ShardIndex( shardBlockGridSize, - IndexLocation.END, new RawBytesArrayCodec()); + IndexLocation.END, new RawBlockCodecInfo()); it = new GridIterator(shardBlockGridSize); i = 0; @@ -71,7 +69,7 @@ public void writeReadTest() throws IOException { final int[] shardBlockGridSize = new int[]{6, 5}; final IndexLocation indexLocation = IndexLocation.END; final IndexCodecAdapter indexCodecAdapter = new IndexCodecAdapter( - new RawBytesArrayCodec(), + new RawBlockCodecInfo(), new Crc32cChecksumCodec() ); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 02d281da7..46e4e2c76 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -14,18 +14,17 @@ import org.janelia.saalfeldlab.n5.NameConfigAdapter; import org.janelia.saalfeldlab.n5.DatasetAttributes.DatasetAttributesAdapter; import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; -import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodec; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodec; -import org.janelia.saalfeldlab.n5.codec.N5ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.RawBytesArrayCodec; -import org.janelia.saalfeldlab.n5.codec.ArrayCodec; -import org.janelia.saalfeldlab.n5.codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; @@ -138,8 +137,8 @@ private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, DataType.UINT8, new ShardingCodec( blockSize, - new Codec[]{new N5ArrayCodec()}, - new Codec[]{new RawBytesArrayCodec(), new Crc32cChecksumCodec()}, + new CodecInfo[]{new N5BlockCodecInfo()}, + new CodecInfo[]{new RawBlockCodecInfo(), new Crc32cChecksumCodec()}, indexLocation ) ); @@ -544,16 +543,16 @@ private DatasetAttributes getNestedShardCodecsAttributes(int[] blockSize) { // probably better to forget about this class - only use DatasetAttributes // and detect shading in another way final ShardingCodec innerShard = new ShardingCodec(innerShardSize, - new Codec[]{new N5ArrayCodec()}, - new DeterministicSizeCodec[]{new RawBytesArrayCodec(indexByteOrder), new Crc32cChecksumCodec()}, + new CodecInfo[]{new N5BlockCodecInfo()}, + new DeterministicSizeCodec[]{new RawBlockCodecInfo(indexByteOrder), new Crc32cChecksumCodec()}, IndexLocation.START); return new DatasetAttributes( dimensions, shardSize, blockSize, DataType.UINT8, new ShardingCodec( blockSize, - new Codec[]{innerShard}, - new DeterministicSizeCodec[]{new RawBytesArrayCodec(indexByteOrder), new Crc32cChecksumCodec()}, + new CodecInfo[]{innerShard}, + new DeterministicSizeCodec[]{new RawBlockCodecInfo(indexByteOrder), new Crc32cChecksumCodec()}, IndexLocation.END) ); } @@ -578,9 +577,9 @@ public ShardedN5Writer(String basePath, GsonBuilder gsonBuilder) { super(basePath); gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); + gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, new TestDatasetAttributesAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBytesArrayCodec.byteOrderAdapter); + gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBlockCodecInfo.byteOrderAdapter); gsonBuilder.registerTypeHierarchyAdapter(ShardingCodec.IndexLocation.class, ShardingCodec.indexLocationAdapter); gsonBuilder.disableHtmlEscaping(); gson = gsonBuilder.create(); @@ -631,25 +630,25 @@ public DatasetAttributes deserialize(JsonElement json, Type typeOfT, JsonDeseria final DataType dataType = context.deserialize(obj.get(DatasetAttributes.DATA_TYPE_KEY), DataType.class); - final Codec[] codecs; + final CodecInfo[] codecs; if (obj.has(DatasetAttributes.CODEC_KEY)) { - codecs = context.deserialize(obj.get(DatasetAttributes.CODEC_KEY), Codec[].class); + codecs = context.deserialize(obj.get(DatasetAttributes.CODEC_KEY), CodecInfo[].class); } else if (obj.has(DatasetAttributes.COMPRESSION_KEY)) { final Compression compression = CompressionAdapter.getJsonAdapter().deserialize(obj.get(DatasetAttributes.COMPRESSION_KEY), Compression.class, context); - codecs = new Codec[]{compression}; + codecs = new CodecInfo[]{compression}; } else { return null; } - final ArrayCodec arrayCodec; - final BytesCodec[] bytesCodecs; - if (codecs[0] instanceof ArrayCodec) { - arrayCodec = (ArrayCodec)codecs[0]; - bytesCodecs = new BytesCodec[codecs.length - 1]; + final BlockCodecInfo arrayCodec; + final DataCodec[] bytesCodecs; + if (codecs[0] instanceof BlockCodecInfo) { + arrayCodec = (BlockCodecInfo)codecs[0]; + bytesCodecs = new DataCodec[codecs.length - 1]; System.arraycopy(codecs, 1, bytesCodecs, 0, bytesCodecs.length); } else { arrayCodec = null; - bytesCodecs = (BytesCodec[])codecs; + bytesCodecs = (DataCodec[])codecs; } return new DatasetAttributes(dimensions, shardSize, blockSize, dataType, arrayCodec, bytesCodecs); } @@ -672,11 +671,11 @@ public JsonElement serialize(DatasetAttributes src, Type typeOfSrc, JsonSerializ } } - private static Codec[] concatenateCodecs(final DatasetAttributes attributes) { + private static CodecInfo[] concatenateCodecs(final DatasetAttributes attributes) { - final BytesCodec[] byteCodecs = attributes.getCodecs(); - final ArrayCodec arrayCodec = attributes.getArrayCodec(); - final Codec[] allCodecs = new Codec[byteCodecs.length + 1]; + final DataCodec[] byteCodecs = attributes.getCodecs(); + final BlockCodecInfo arrayCodec = attributes.getArrayCodec(); + final CodecInfo[] allCodecs = new CodecInfo[byteCodecs.length + 1]; allCodecs[0] = arrayCodec; System.arraycopy(byteCodecs, 0, allCodecs, 1, byteCodecs.length); From 37b51ce51200adc95109a0459372d8d691137afd Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 29 Aug 2025 17:24:57 +0200 Subject: [PATCH 285/423] WIP NestedPosition --- .../saalfeldlab/n5/shardstuff/Nesting.java | 116 +++++++++++++++++- 1 file changed, 114 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java index b18b32704..c5c3b9379 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java @@ -3,16 +3,119 @@ // TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // -// [ ] NestedGrid javadoc +// [ ] NestedGrid +// [ ] validation in constructor +// [ ] test for that validation +// [ ] javadoc +// // [ ] NestedPosition interface +// [+] LazyNestedPosition class +// [+] fields: NestedGrid, long[] position, int level +// [+] construct with source level 0 +// [+] minimal abs/rel access methods +// [+] toString() +// [-] extract NestedPosition abstract class +// ==> postpone until necessary +// [ ] equals / hashcode +// [ ] should we have prefix()? suffix()? head()? tail()? // // TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - import java.util.Arrays; public class Nesting { + public static void main(String[] args) { + final int[][] blockSizes = {{1,1,1}, {3,2,2}, {6,4,4}, {24,24,24}}; + final NestedGrid grid = new NestedGrid(blockSizes); + final NestedPosition pos = new NestedPosition(grid, new long[] {38, 7, 129}); + System.out.println("pos = " + pos); + System.out.println("key = " + Arrays.toString(pos.keyPosition())); + } + + + + + public static class NestedPosition { + + private final NestedGrid grid; + private final long[] position; + private final int level; + + public NestedPosition(final NestedGrid grid, final long[] position, final int level) { + this.grid = grid; + this.position = position; + this.level = level; + } + + public NestedPosition(final NestedGrid grid, final long[] position) { + this(grid, position, 0); + } + + /** + * Get the nesting level of this position. + *

      + * Positions with {@code level=0} refer to DataBlocks, positions with + * {@code level=1} refer to first-level shards (containing DataBlocks), + * and so on. + * + * @return nesting level + */ + public int level() { + return level; + } + + public int numDimensions() { + return grid.numDimensions(); + } + + /** + * Get the relative grid position at {@code level}, that is, relative + * offset within containing the {@code (level+1)} element. + * + * @param level + * requested nesting level + * + * @return relative grid position + */ + public long[] relativePosition(final int level) { + return grid.relativePosition(position, level); + } + + /** + * Get the absolute grid position at {@code level}. + * + * @param level + * requested nesting level + * + * @return absolute grid position + */ + public long[] absolutePosition(final int level) { + return grid.relativePosition(position, level); + } + + public long[] keyPosition() { + return relativePosition(grid.numLevels() - 1); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append('{'); + for ( int l = level; l < grid.numLevels(); ++l ) { + if ( l > level ) { + sb.append(" / "); + } + sb.append(Arrays.toString(relativePosition(l))); + } + sb.append(" (level ").append(level).append(")}"); + return sb.toString(); + } + + // TODO: equals() and hashCode() + // TODO: should we have prefix()? suffix()? head()? tail()? + } + /** * TODO @@ -62,6 +165,14 @@ public NestedGrid(int[][] blockSizes) { } } + public int numLevels() { + return m; + } + + public int numDimensions() { + return n; + } + public void absolutePosition( final long[] sourcePos, final int sourceLevel, @@ -119,4 +230,5 @@ public long[] relativePosition( } } + } From 637cb52bb6b5c8a22f0903dbaed4a4bb11fc14ef Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 29 Aug 2025 22:19:52 +0200 Subject: [PATCH 286/423] WIP nested decoding again --- .../saalfeldlab/n5/shardstuff/Nesting.java | 8 +- .../n5/shardstuff/ShardStuff2.java | 271 ++++++++++++++++++ 2 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java index c5c3b9379..08d204b74 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java @@ -14,10 +14,13 @@ // [+] construct with source level 0 // [+] minimal abs/rel access methods // [+] toString() -// [-] extract NestedPosition abstract class +// [-] extract NestedPosition interface // ==> postpone until necessary // [ ] equals / hashcode // [ ] should we have prefix()? suffix()? head()? tail()? +// [ ] Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...). +// For nested = {X,Y,Z} compare by Z, then Y, then X. +// For X = {x,y,z} compare by z, then y, then x. (flattening order) // // TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -78,6 +81,7 @@ public int numDimensions() { * * @return relative grid position */ + // TODO: rename to relative()? reads nicer in position.relative(l) ... public long[] relativePosition(final int level) { return grid.relativePosition(position, level); } @@ -90,10 +94,12 @@ public long[] relativePosition(final int level) { * * @return absolute grid position */ + // TODO: rename to absolute()? reads nicer in position.absolute(l) ... public long[] absolutePosition(final int level) { return grid.relativePosition(position, level); } + // TODO: rename to key()? to match absolute() and relative() ... public long[] keyPosition() { return relativePosition(grid.numLevels() - 1); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java new file mode 100644 index 000000000..e8001ea0f --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java @@ -0,0 +1,271 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.N5URI; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodec; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; + +public class ShardStuff2 { + + public interface Block { + + /** + * Returns the number of elements in a box of given size. + * + * @param size + * the size + * + * @return the number of elements + */ + static int getNumElements(final int[] size) { + + int n = size[0]; + for (int i = 1; i < size.length; ++i) + n *= size[i]; + return n; + } + + /** + * Returns the size of this block. + *

      + * The size of a data block is expected to be smaller than or equal to the + * spacing of the block grid. The dimensionality of size is expected to be + * equal to the dimensionality of the dataset. Consistency is not enforced. + * + * @return size of the block + */ + int[] getSize(); + + /** + * Returns the grid position of this block. + *

      + * Grid position is with respect to the level of the block (Shard, nested Shard, DataBlock, ...). + * If a block has grid position (i, ...) the adjacent block (in X) has position (i+1, ...). + * Grid position (0, ...) is the origin of the dataset, that is, grid positions are not relative to the containing shard etc. + *

      + * The dimensionality of the grid position is expected to be equal to the + * dimensionality of the dataset. Consistency is not enforced. + * + * @return position on the block grid + */ + long[] getGridPosition(); + } + + + public interface Shard extends Block { + + // pos is relative to this shard + ReadData getElementData(long[] pos); + + // pos is relative to this shard + void setElementData(ReadData data, long[] pos); + + // TODO: removeElement(long[] pos) + } + + + public interface DataBlock extends Block { + + /** + * Returns the data object held by this data block. + * + * @return data object + */ + T getData(); + +// interface DataBlockFactory { +// DataBlock createDataBlock(int[] blockSize, long[] gridPosition, T data); +// } + } + + + + + + + + public interface BlockCodec { + + ReadData encode(B block) throws N5IOException; + + /** + * {@code gridPosition} is with respect to the level of the block (Shard, nested Shard, DataBlock, ...). + * If a block has gridPosition (i, ...), then the adjacent block (in X) has position (i+1, ...). + * Grid position (0, ...) is the origin of the dataset, that is, grid positions are not relative to the containing shard etc. + */ + B decode(ReadData readData, long[] gridPosition) throws N5IOException; + } + + public interface ShardCodec extends BlockCodec { + } + + public interface DataBlockCodec extends BlockCodec> { + } + + + + public interface ShardCodecInfo extends BlockCodecInfo { + + /** + * Chunk size of the elements in this block. + * That is, (1, ...) for DataBlockCodecInfo, respectively inner block size for ShardCodecInfo. + */ + int[] getChunkSize(); + + /** + * Nested BlockCodec. + * ({@code null} for DataBlockCodec. + */ + BlockCodecInfo getInnerBlockCodecInfo(); + + DataCodecInfo[] getInnerDataCodecInfos(); + + // TODO: IndexCodec + + ShardCodec create(int[] blockSize, DataCodecInfo... codecs); + } + + public interface DataBlockCodecInfo extends BlockCodecInfo { + DataBlockCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs); + } + + + + // NB: Something needs to get the ReadData from the KVA. + // We cannot put readBlocks(...) logic into the (Data)BlockCodec, because that doesn't have access to the KVA. + // We could put readBlocks(ReadData, ...) into the ShardCodec, to read several blocks from the same Shard. + // Then the N5Reader only has to do one level of batching. + + // TODO: Consider making this really recursive. + // ... but postpone until the overall shape is a bit clearer. + + static class WipShardingCodec implements DataBlockCodec { + + private final NestedGrid grid; + private final DataBlockCodec dataBlockCodec; + private final ShardCodec[] shardCodecs; + + private final WipShardingCodec nestedCodec; // TODO: For exploring logic options, we create a nested WipShardingCodec. + + public WipShardingCodec(final NestedGrid grid, final DataBlockCodec dataBlockCodec, final ShardCodec[] shardCodecs, final WipShardingCodec nestedCodec) { + this.grid = grid; + this.dataBlockCodec = dataBlockCodec; + this.shardCodecs = shardCodecs; + this.nestedCodec = nestedCodec; + } + + public NestedPosition toNestedPosition(final long[] gridPosition) { + return new NestedPosition(grid, gridPosition); + } + + // TODO: implementation without nestedCodec + public DataBlock decode(ReadData readData, final NestedPosition position) throws N5IOException { + + // NB: Assuming position.level==0 (refers to a DataBlock) and has + // nesting corresponding (at least) to our grid. This should perhaps + // be verified, at least during development. + + for ( int l = grid.numLevels() - 1; l > 0; --l ) { + final Shard shard = shardCodecs[l].decode(readData, position.absolutePosition(l)); + readData = shard.getElementData(position.relativePosition(l - 1)); + } + return dataBlockCodec.decode(readData, position.absolutePosition(0)); + } + + // TODO: alternative implementation that uses nestedCodec + public DataBlock decode2(final ReadData readData, final NestedPosition position) throws N5IOException { + + // NB: Assuming position.level==0 (refers to a DataBlock) and has + // nesting corresponding (at least) to our grid. This should perhaps + // be verified, at least during development. + + if ( nestedCodec != null ) { + final int l = shardCodecs.length - 1; + final Shard shard = shardCodecs[l].decode(readData, position.absolutePosition(l)); + final ReadData nestedData = shard.getElementData(position.relativePosition(l - 1)); + return nestedCodec.decode(nestedData, position); + } else { + return dataBlockCodec.decode(readData, position.absolutePosition(0)); + } + } + + + + + + @Override + public ReadData encode(final DataBlock block) throws N5IOException { + throw new UnsupportedOperationException("TODO"); + } + + @Override + public DataBlock decode(final ReadData readData, final long[] gridPosition) throws N5IOException { + throw new UnsupportedOperationException("TODO"); + } + } + + static WipShardingCodec create( + final DataType dataType, + int[] blockSize, + BlockCodecInfo blockCodecInfo, + DataCodecInfo[] dataCodecInfos + ) { + final int m = nestingDepth(blockCodecInfo); + + // TODO: For exploring logic options, we create a nested WipShardingCodec. + // If that turns out to be useful, the below logic should be simplified + // to re-use info from the nested WipShardingCodec. + final WipShardingCodec nestedWipShardingCodec; + if ( m > 1 ) { + final ShardCodecInfo info = (ShardCodecInfo) blockCodecInfo; + nestedWipShardingCodec = create(dataType, info.getChunkSize(), info.getInnerBlockCodecInfo(), info.getInnerDataCodecInfos()); + } else { + nestedWipShardingCodec = null; + } + + final int[][] blockSizes = new int[m][]; + + // There are m codecs: 1 DataBlockCodec, m-1 ShardCodecs. + // Outermost codec comes last. + // To make the index math easier, shardCodecs[0] = null (the dataBlockCodec is at level 0). + final DataBlockCodec dataBlockCodec; + final ShardCodec[] shardCodecs = new ShardCodec[m]; + + for (int l = m - 1; l > 0; --l) { + final ShardCodecInfo info = (ShardCodecInfo) blockCodecInfo; // TODO instanceof check + blockSizes[l] = blockSize; + shardCodecs[l] = info.create(blockSize, dataCodecInfos); + blockCodecInfo = info.getInnerBlockCodecInfo(); + dataCodecInfos = info.getInnerDataCodecInfos(); + blockSize = info.getChunkSize(); + } + + final DataBlockCodecInfo info = (DataBlockCodecInfo) blockCodecInfo; // TODO instanceof check + blockSizes[0] = blockSize; + dataBlockCodec = info.create(dataType, blockSize, dataCodecInfos); + + final NestedGrid grid = new NestedGrid(blockSizes); + + return new WipShardingCodec<>(grid, dataBlockCodec, shardCodecs, nestedWipShardingCodec); + } + + private static int nestingDepth(BlockCodecInfo info) { + if (info instanceof ShardCodecInfo) { + return 1 + nestingDepth(((ShardCodecInfo) info).getInnerBlockCodecInfo()); + } else { + return 1; + } + } +} From 8d2232dfda0931cad1f34fca34315133e912e892 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 30 Aug 2025 22:09:12 +0200 Subject: [PATCH 287/423] WIP nested decoding --- .../n5/shardstuff/ShardStuff2.java | 140 +++++++++++++++--- 1 file changed, 116 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java index e8001ea0f..80c321667 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java @@ -1,17 +1,9 @@ package org.janelia.saalfeldlab.n5.shardstuff; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader; -import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.N5URI; -import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.codec.CodecInfo; -import org.janelia.saalfeldlab.n5.codec.DataCodec; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; @@ -115,6 +107,10 @@ public interface DataBlockCodec extends BlockCodec> { } + public interface BlockCodecInfo extends CodecInfo { + // when moving to Java 17+: + // this should be sealed, permitting ShardCodecInfo and DataBlockCodecInfo + } public interface ShardCodecInfo extends BlockCodecInfo { @@ -122,7 +118,7 @@ public interface ShardCodecInfo extends BlockCodecInfo { * Chunk size of the elements in this block. * That is, (1, ...) for DataBlockCodecInfo, respectively inner block size for ShardCodecInfo. */ - int[] getChunkSize(); + int[] getInnerBlockSize(); /** * Nested BlockCodec. @@ -143,6 +139,103 @@ public interface DataBlockCodecInfo extends BlockCodecInfo { + + // ----------------- + // --- "testing" --- + + public static void main(String[] args) { + + // DataBlocks are 3x3x3 + // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) + // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) + + final DataBlockCodecInfo c0 = new DummyDataBlockCodecInfo(); + final ShardCodecInfo c1 = new DummyShardCodecInfo( + new int[] {3, 3, 3}, + c0, + new DataCodecInfo[] {new RawCompression()} + ); + final ShardCodecInfo c2 = new DummyShardCodecInfo( + new int[] {6, 6, 6}, + c1, + new DataCodecInfo[] {new RawCompression()} + ); + + final WipShardingCodec wipShardingCodec = create(DataType.INT8, + new int[] {24, 24, 24}, + c2, + new DataCodecInfo[] {new RawCompression()}); + + System.out.println("wipShardingCodec = " + wipShardingCodec); + } + + public static class DummyShardCodecInfo implements ShardCodecInfo { + + private final int[] innerBlockSize; + private final BlockCodecInfo innerBlockCodecInfo; + private final DataCodecInfo[] innerDataCodecInfos; + + public DummyShardCodecInfo( + final int[] innerBlockSize, + final BlockCodecInfo innerBlockCodecInfo, + final DataCodecInfo[] innerDataCodecInfos) { + this.innerBlockSize = innerBlockSize; + this.innerBlockCodecInfo = innerBlockCodecInfo; + this.innerDataCodecInfos = innerDataCodecInfos; + } + + @Override + public int[] getInnerBlockSize() { + return innerBlockSize; + } + + @Override + public BlockCodecInfo getInnerBlockCodecInfo() { + return innerBlockCodecInfo; + } + + @Override + public DataCodecInfo[] getInnerDataCodecInfos() { + return innerDataCodecInfos; + } + + @Override + public ShardCodec create(final int[] blockSize, final DataCodecInfo... codecs) { + return null; + } + + @Override + public String getType() { + return "DummyShardCodec"; + } + } + + public static class DummyDataBlockCodecInfo implements DataBlockCodecInfo { + + @Override + public DataBlockCodec create(final DataType dataType, final int[] blockSize, final DataCodecInfo... codecs) { + return null; + } + + @Override + public String getType() { + return "DummyDataBlockCodecInfo"; + } + } + + // --- "testing" --- + // ----------------- + + + + + + + + + + + // NB: Something needs to get the ReadData from the KVA. // We cannot put readBlocks(...) logic into the (Data)BlockCodec, because that doesn't have access to the KVA. // We could put readBlocks(ReadData, ...) into the ShardCodec, to read several blocks from the same Shard. @@ -151,7 +244,7 @@ public interface DataBlockCodecInfo extends BlockCodecInfo { // TODO: Consider making this really recursive. // ... but postpone until the overall shape is a bit clearer. - static class WipShardingCodec implements DataBlockCodec { + static class WipShardingCodec { private final NestedGrid grid; private final DataBlockCodec dataBlockCodec; @@ -185,7 +278,7 @@ public DataBlock decode(ReadData readData, final NestedPosition position) thr } // TODO: alternative implementation that uses nestedCodec - public DataBlock decode2(final ReadData readData, final NestedPosition position) throws N5IOException { + public DataBlock decodeNested(final ReadData readData, final NestedPosition position) throws N5IOException { // NB: Assuming position.level==0 (refers to a DataBlock) and has // nesting corresponding (at least) to our grid. This should perhaps @@ -195,7 +288,7 @@ public DataBlock decode2(final ReadData readData, final NestedPosition positi final int l = shardCodecs.length - 1; final Shard shard = shardCodecs[l].decode(readData, position.absolutePosition(l)); final ReadData nestedData = shard.getElementData(position.relativePosition(l - 1)); - return nestedCodec.decode(nestedData, position); + return nestedCodec.decodeNested(nestedData, position); } else { return dataBlockCodec.decode(readData, position.absolutePosition(0)); } @@ -204,16 +297,15 @@ public DataBlock decode2(final ReadData readData, final NestedPosition positi + // --- DataBlockCodec --- - @Override - public ReadData encode(final DataBlock block) throws N5IOException { - throw new UnsupportedOperationException("TODO"); - } - - @Override - public DataBlock decode(final ReadData readData, final long[] gridPosition) throws N5IOException { - throw new UnsupportedOperationException("TODO"); - } +// @Override +// public ReadData encode(final DataBlock block) throws N5IOException { +// } +// +// @Override +// public DataBlock decode(final ReadData readData, final long[] gridPosition) throws N5IOException { +// } } static WipShardingCodec create( @@ -230,7 +322,7 @@ static WipShardingCodec create( final WipShardingCodec nestedWipShardingCodec; if ( m > 1 ) { final ShardCodecInfo info = (ShardCodecInfo) blockCodecInfo; - nestedWipShardingCodec = create(dataType, info.getChunkSize(), info.getInnerBlockCodecInfo(), info.getInnerDataCodecInfos()); + nestedWipShardingCodec = create(dataType, info.getInnerBlockSize(), info.getInnerBlockCodecInfo(), info.getInnerDataCodecInfos()); } else { nestedWipShardingCodec = null; } @@ -249,7 +341,7 @@ static WipShardingCodec create( shardCodecs[l] = info.create(blockSize, dataCodecInfos); blockCodecInfo = info.getInnerBlockCodecInfo(); dataCodecInfos = info.getInnerDataCodecInfos(); - blockSize = info.getChunkSize(); + blockSize = info.getInnerBlockSize(); } final DataBlockCodecInfo info = (DataBlockCodecInfo) blockCodecInfo; // TODO instanceof check From d041601408be9486b7b26fd8568370f03aaeb4e7 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 31 Aug 2025 22:43:58 +0200 Subject: [PATCH 288/423] WIP SegmentedReadData --- .../saalfeldlab/n5/readdata/SegmentStuff.java | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/SegmentStuff.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SegmentStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SegmentStuff.java new file mode 100644 index 000000000..f3f00338b --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SegmentStuff.java @@ -0,0 +1,90 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import java.util.List; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; + +public class SegmentStuff { + + public interface SegmentedReadData extends ReadData { + + /** + * Returns the {@code SegmentLocation} of {@code segment} in this {@code ReadData}. + *

      + * Note that this {@code ReadData} is not necessarily the source of the segment. + *

      + * The returned {@code SegmentLocation} may be {@code {offset=0, index=-1}}, which + * means that the segment comprises this whole {@code ReadData} (and the length of + * this {@code ReadData} is not yet known. + * + * @param segment + * the segment id + * + * @return location of the segment, or null if the segment is not contained in this ReadData + */ + SegmentLocation location(Segment segment); + + /** + * @return all segments contained in this {@code ReadData}. + */ + // TODO: return Collection instead? Or Segment[]? + List segments(); + + // TODO: return wrapper of readData with one segment comprising the whole ReadData + // length of segment could return readData.length() + static SegmentedReadData wrap(ReadData readData) { + throw new UnsupportedOperationException("TODO"); + } + + // TODO: return wrapper of readData with (new) segments at the given locations + static SegmentedReadData wrap(ReadData readData, SegmentLocation[] locations) { + throw new UnsupportedOperationException("TODO"); + } + + // TODO: return a SegmentedReadData wrapping a slice containing exactly the given segment + // throws IllegalArgumentException if segment is not contained in this ReadData + SegmentedReadData slice(Segment segment) throws IllegalArgumentException; + } + + // TODO: make Segment and SegmentLocation interfaces? Then we could make a + // class that implements both for use in SegmentedReadData implementation. + + /** + * A segment in a ReadData. + *

      + * Note that segments use object identity. + *

      + * + */ + public static class Segment { + + private final ReadData source; + + // TODO: add Segment id for debugging ... + + public Segment(final ReadData source) { + + this.source = source; + } + } + + public static class SegmentLocation { + + private final long offset; + private final long length; + + public SegmentLocation(final long offset, final long length) { + this.offset = offset; + this.length = length; + } + + public long offset() { + return offset; + } + + public long length() { + return length; + } + } + +} From 77048145abf9f1ceac8bc975e4119d51f9ba1088 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 1 Sep 2025 09:06:13 +0200 Subject: [PATCH 289/423] wip --- .../saalfeldlab/n5/readdata/SegmentStuff.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SegmentStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SegmentStuff.java index f3f00338b..7bd612f4d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SegmentStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SegmentStuff.java @@ -28,6 +28,9 @@ public interface SegmentedReadData extends ReadData { * @return all segments contained in this {@code ReadData}. */ // TODO: return Collection instead? Or Segment[]? + // Maybe: ordered by offset then length? Then offset of first + // segment and offset+length of last segment could be used to + // determine tight bounds of slice containing all segments. List segments(); // TODO: return wrapper of readData with one segment comprising the whole ReadData @@ -43,7 +46,12 @@ static SegmentedReadData wrap(ReadData readData, SegmentLocation[] locations) { // TODO: return a SegmentedReadData wrapping a slice containing exactly the given segment // throws IllegalArgumentException if segment is not contained in this ReadData - SegmentedReadData slice(Segment segment) throws IllegalArgumentException; + SegmentedReadData slice(Segment segment) throws IllegalArgumentException, N5IOException; + + // TODO: has all segments fully contained in requested slice. + @Override + SegmentedReadData slice(final long offset, final long length) throws N5IOException; + } // TODO: make Segment and SegmentLocation interfaces? Then we could make a @@ -87,4 +95,10 @@ public long length() { } } + + public static class ReadDataWrapper implements ReadData { + private final ReadData delegate; + } + + } From ed349e22d6e1940feca022379a64f098985b882c Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 2 Sep 2025 11:32:18 +0200 Subject: [PATCH 290/423] WIP SegmentedReadData --- pom.xml | 2 - .../saalfeldlab/n5/readdata/SegmentStuff.java | 104 ------- .../n5/readdata/segment/SegmentStuff.java | 254 ++++++++++++++++++ .../segment/SegmentedReadDataImpl.java | 144 ++++++++++ .../n5/readdata/segment/SegmentTest.java | 89 ++++++ 5 files changed, 487 insertions(+), 106 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/SegmentStuff.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java diff --git a/pom.xml b/pom.xml index a0660d188..89858106c 100644 --- a/pom.xml +++ b/pom.xml @@ -146,8 +146,6 @@ sign,deploy-to-scijava 4.0.0 - - 24 diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SegmentStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/SegmentStuff.java deleted file mode 100644 index 7bd612f4d..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/SegmentStuff.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.janelia.saalfeldlab.n5.readdata; - -import java.util.List; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; - -public class SegmentStuff { - - public interface SegmentedReadData extends ReadData { - - /** - * Returns the {@code SegmentLocation} of {@code segment} in this {@code ReadData}. - *

      - * Note that this {@code ReadData} is not necessarily the source of the segment. - *

      - * The returned {@code SegmentLocation} may be {@code {offset=0, index=-1}}, which - * means that the segment comprises this whole {@code ReadData} (and the length of - * this {@code ReadData} is not yet known. - * - * @param segment - * the segment id - * - * @return location of the segment, or null if the segment is not contained in this ReadData - */ - SegmentLocation location(Segment segment); - - /** - * @return all segments contained in this {@code ReadData}. - */ - // TODO: return Collection instead? Or Segment[]? - // Maybe: ordered by offset then length? Then offset of first - // segment and offset+length of last segment could be used to - // determine tight bounds of slice containing all segments. - List segments(); - - // TODO: return wrapper of readData with one segment comprising the whole ReadData - // length of segment could return readData.length() - static SegmentedReadData wrap(ReadData readData) { - throw new UnsupportedOperationException("TODO"); - } - - // TODO: return wrapper of readData with (new) segments at the given locations - static SegmentedReadData wrap(ReadData readData, SegmentLocation[] locations) { - throw new UnsupportedOperationException("TODO"); - } - - // TODO: return a SegmentedReadData wrapping a slice containing exactly the given segment - // throws IllegalArgumentException if segment is not contained in this ReadData - SegmentedReadData slice(Segment segment) throws IllegalArgumentException, N5IOException; - - // TODO: has all segments fully contained in requested slice. - @Override - SegmentedReadData slice(final long offset, final long length) throws N5IOException; - - } - - // TODO: make Segment and SegmentLocation interfaces? Then we could make a - // class that implements both for use in SegmentedReadData implementation. - - /** - * A segment in a ReadData. - *

      - * Note that segments use object identity. - *

      - * - */ - public static class Segment { - - private final ReadData source; - - // TODO: add Segment id for debugging ... - - public Segment(final ReadData source) { - - this.source = source; - } - } - - public static class SegmentLocation { - - private final long offset; - private final long length; - - public SegmentLocation(final long offset, final long length) { - this.offset = offset; - this.length = length; - } - - public long offset() { - return offset; - } - - public long length() { - return length; - } - } - - - public static class ReadDataWrapper implements ReadData { - private final ReadData delegate; - } - - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java new file mode 100644 index 000000000..b82bcfe11 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java @@ -0,0 +1,254 @@ +package org.janelia.saalfeldlab.n5.readdata.segment; + +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +public class SegmentStuff { + + public interface SegmentedReadData extends ReadData { + + /** + * Returns the {@code SegmentLocation} of {@code segment} in this {@code ReadData}. + *

      + * Note that this {@code ReadData} is not necessarily the source of the segment. + *

      + * The returned {@code SegmentLocation} may be {@code {offset=0, index=-1}}, which + * means that the segment comprises this whole {@code ReadData} (and the length of + * this {@code ReadData} is not yet known. + * + * @param segment + * the segment id + * + * @return location of the segment, or null + * @throws IllegalArgumentException if the segment is not contained in this ReadData + */ + SegmentLocation location(Segment segment) throws IllegalArgumentException; + + /** + * @return all segments contained in this {@code ReadData}. + */ + // Order is the same as the SegmentLocations given at construction + List segments(); + + // TODO: return wrapper of readData with one segment comprising the whole ReadData + // length of segment could return readData.length() + static SegmentedReadData wrap(ReadData readData) { + throw new UnsupportedOperationException("TODO"); + } + + /** + * Wrap {@code readData} and create segments at the given locations. + */ + // TODO: does not assume ordered locations. to not lose track, this method + // should return Pair where segments are in + // the same order as locations. + // TODO: use List instead of SegmentLocation[] + static SegmentedReadData wrap(ReadData readData, SegmentLocation... locations) { + return SegmentedReadDataImpl.wrap(readData, locations); + } + + /** + * Return a {@code SegmentedReadData} wrapping a slice containing exactly the given segment. + * + * @param segment + * @return + * @throws IllegalArgumentException if the segment is not contained in this ReadData + * @throws N5IOException + */ + SegmentedReadData slice(Segment segment) throws IllegalArgumentException, N5IOException; + + // TODO: has all segments fully contained in requested slice. + @Override + SegmentedReadData slice(final long offset, final long length) throws N5IOException; + + } + + /** + * A particular segment in a source {@code ReadData}. + */ + public interface Segment { + + ReadData source(); + } + + public interface SegmentLocation { + + Comparator COMPARATOR = Comparator + .comparingLong(SegmentLocation::offset) + .thenComparingLong(SegmentLocation::length); + + long offset(); + + long length(); + + static SegmentLocation at(final long offset, final long length) { + return new SegmentLocationImpl(offset, length); + } + } + + + + // TODO rename + static class SegmentedReadData_Single extends ReadDataWrapper implements SegmentedReadData { + + private class SegmentImpl implements Segment, SegmentLocation { + + @Override + public ReadData source() { + return delegate; + } + + @Override + public long offset() { + return 0; + } + + @Override + public long length() { + return delegate.length(); + } + } + + private final SegmentImpl segment = new SegmentImpl(); + + SegmentedReadData_Single(final ReadData delegate) { + super(delegate); + } + + @Override + public SegmentLocation location(final Segment s) { + return (segment.equals(s)) ? segment : null; + } + + @Override + public List segments() { + return Collections.singletonList(segment); + } + + @Override + public SegmentedReadData slice(final Segment s) throws IllegalArgumentException, N5IOException { + if (segment.equals(s)) { + return this; + } else { + throw new IllegalArgumentException("Provided segment is not part of this "); + } + } + + @Override + public SegmentedReadData slice(final long offset, final long length) throws N5IOException { + final ReadData delegateSlice = delegate.slice(offset, length); + if (offset == segment.offset() && length == segment.length()) { + return this; + } else { + throw new UnsupportedOperationException("TODO"); + } + } + } + + + static class SegmentImpl implements Segment { + + private final ReadData source; + + // TODO: add Segment id for debugging ... + + public SegmentImpl(final ReadData source) { + this.source = source; + } + + @Override + public ReadData source() { + return source; + } + } + + static class SegmentLocationImpl implements SegmentLocation { + + private final long offset; + private final long length; + + public SegmentLocationImpl(final long offset, final long length) { + this.offset = offset; + this.length = length; + } + + @Override + public long offset() { + return offset; + } + + @Override + public long length() { + return length; + } + + @Override + public String toString() { + return "SegmentLocation{" + + "offset=" + offset + + ", length=" + length + + '}'; + } + } + + static class ReadDataWrapper implements ReadData { + final ReadData delegate; + + ReadDataWrapper(final ReadData delegate) { + this.delegate = delegate; + } + + @Override + public long length() throws N5IOException { + return delegate.length(); + } + + @Override + public ReadData limit(final long length) throws N5IOException { + return delegate.limit(length); + } + + @Override + public ReadData slice(final long offset, final long length) throws N5IOException { + return delegate.slice(offset, length); + } + + @Override + public InputStream inputStream() throws N5IOException, IllegalStateException { + return delegate.inputStream(); + } + + @Override + public byte[] allBytes() throws N5IOException, IllegalStateException { + return delegate.allBytes(); + } + + @Override + public ByteBuffer toByteBuffer() throws N5IOException, IllegalStateException { + return delegate.toByteBuffer(); + } + + @Override + public ReadData materialize() throws N5IOException { + return delegate.materialize(); + } + + @Override + public void writeTo(final OutputStream outputStream) throws N5IOException, IllegalStateException { + delegate.writeTo(outputStream); + } + + @Override + public ReadData encode(final OutputStreamOperator encoder) { + return delegate.encode(encoder); + } + } + + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java new file mode 100644 index 000000000..095d7f630 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java @@ -0,0 +1,144 @@ +package org.janelia.saalfeldlab.n5.readdata.segment; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.Segment; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentLocation; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentLocationImpl; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentedReadData; + +class SegmentedReadDataImpl extends SegmentStuff.ReadDataWrapper implements SegmentedReadData { + + // segments, ordered by location + private final List segments; + + private final ReadData segmentSource; + + private final long offset; + + private static class SegmentImpl implements Segment, SegmentLocation { + + private final ReadData source; + + private final long offset; + + private final long length; + + public SegmentImpl(final ReadData source, final SegmentLocation location) { + this(source, location.offset(), location.length()); + } + + public SegmentImpl(final ReadData source, final long offset, final long length) { + this.source = source; + this.offset = offset; + this.length = length; + } + + @Override + public long offset() { + return offset; + } + + @Override + public long length() { + return length; + } + + @Override + public ReadData source() { + return source; + } + } + + // assumes segments are ordered by location + private SegmentedReadDataImpl(final ReadData delegate, final ReadData segmentSource, final long offset, final List segments) { + super(delegate); + this.segmentSource = segmentSource; + this.offset = offset; + this.segments = segments; + } + + // assumes segments are ordered by location + private SegmentedReadDataImpl(final ReadData delegate, final List segments) { + this(delegate, delegate, 0, segments); + } + + // TODO: does not assume ordered locations. to not lose track, this method + // should return Pair where segments are in + // the same order as locations. + // TODO: use List instead of SegmentLocation[] + public static SegmentedReadData wrap(final ReadData readData, final SegmentLocation[] locations) { + final List segments = new ArrayList<>(locations.length); + for (SegmentLocation l : locations) { + segments.add(new SegmentImpl(readData, l)); + } + segments.sort(SegmentLocation.COMPARATOR); + return new SegmentedReadDataImpl(readData, segments); + } + + @Override + public SegmentLocation location(final Segment segment) { + if (segmentSource.equals(segment.source()) && segment instanceof SegmentLocation) { + final SegmentLocation l = (SegmentLocation) segment; + return offset == 0 ? l : new SegmentLocationImpl(l.offset() - offset, l.length()); + } else { + throw new IllegalArgumentException(); + } + } + + @Override + public List segments() { + return segments; + } + + @Override + public SegmentedReadData slice(final Segment segment) throws IllegalArgumentException, N5Exception.N5IOException { + if (segmentSource.equals(segment.source()) && segment instanceof SegmentImpl) { + final SegmentImpl s = (SegmentImpl) segment; + return new SegmentedReadDataImpl( + delegate.slice(s.offset(), s.length()), + segmentSource, + this.offset + s.offset(), + Collections.singletonList(s)); + } else { + throw new IllegalArgumentException(); + } + } + + @Override + public SegmentedReadData slice(final long offset, final long length) throws N5Exception.N5IOException { + + final long sourceOffset = this.offset + offset; + + // fromIndex: find first segment with offset >= sourceOffset + int fromIndex = Collections.binarySearch(segments, new SegmentLocationImpl(sourceOffset, 0), SegmentLocation.COMPARATOR); + if (fromIndex < 0) { + fromIndex = -fromIndex - 1; + } + + // toIndex: find first segment with offset >= sourceOffset + length + int toIndex = Collections.binarySearch(segments, new SegmentLocationImpl(sourceOffset + length, 0), SegmentLocation.COMPARATOR); + if (toIndex < 0) { + toIndex = -toIndex - 1; + } + + // contained: find segments in [fromIndex, toIndex) with s.offset() + s.length() <= sourceOffset + length + final List candidates = segments.subList(fromIndex, toIndex); + final ArrayList contained = new ArrayList<>(candidates.size()); + candidates.forEach(s -> { + if (s.offset() + s.length() <= sourceOffset + length) { + contained.add(s); + } + }); + contained.trimToSize(); + + return new SegmentedReadDataImpl( + delegate.slice(offset, length), + segmentSource, + this.offset + offset, + contained); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java new file mode 100644 index 000000000..053ca4e6d --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java @@ -0,0 +1,89 @@ +package org.janelia.saalfeldlab.n5.readdata.segment; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentLocation; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentedReadData; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class SegmentTest { + + private ReadData readData; + + @Before + public void createReadData() { + + final byte[] data = new byte[100]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) i; + } + readData = ReadData.from(data); + } + + @Test + public void testWrap() { + + final SegmentLocation[] locations = { + SegmentLocation.at(0, 10), + SegmentLocation.at(10, 10), + SegmentLocation.at(40, 20)}; + final SegmentedReadData r = SegmentedReadData.wrap(readData, locations); + assertEquals(3, r.segments().size()); + + final SegmentLocation l0 = r.location(r.segments().get(0)); + assertEquals(0, l0.offset()); + assertEquals(10, l0.length()); + + final SegmentLocation l1 = r.location(r.segments().get(1)); + assertEquals(10, l1.offset()); + assertEquals(10, l1.length()); + + final SegmentLocation l2 = r.location(r.segments().get(2)); + assertEquals(40, l2.offset()); + assertEquals(20, l2.length()); + } + + @Test + public void testSlice() { + + final SegmentLocation[] locations = { + SegmentLocation.at(0, 10), + SegmentLocation.at(10, 10), + SegmentLocation.at(40, 20)}; + final SegmentedReadData r = SegmentedReadData.wrap(readData, locations); + final SegmentedReadData s = r.slice(10, 60); + assertEquals(60, s.length()); + + assertEquals(2, s.segments().size()); + + final SegmentLocation l0 = s.location(s.segments().get(0)); + assertEquals(0, l0.offset()); + assertEquals(10, l0.length()); + + final SegmentLocation l1 = s.location(s.segments().get(1)); + assertEquals(30, l1.offset()); + assertEquals(20, l1.length()); + } + + @Test + public void testSliceSegment() { + + final SegmentLocation[] locations = { + SegmentLocation.at(0, 10), + SegmentLocation.at(10, 10), + SegmentLocation.at(40, 20)}; + final SegmentedReadData r = SegmentedReadData.wrap(readData, locations); + final SegmentedReadData s = r.slice(r.segments().get(2)); + assertEquals(20, s.length()); + + assertEquals(1, s.segments().size()); + + final SegmentLocation l0 = s.location(s.segments().get(0)); + System.out.println("l0 = " + l0); + assertEquals(0, l0.offset()); + assertEquals(20, l0.length()); + } + +} From e32e7b07303488520735276a676e4eaa3f2b2c98 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 2 Sep 2025 12:49:15 +0200 Subject: [PATCH 291/423] Revise ReadData implementations to return itself from materialize() --- .../n5/KeyValueAccessReadData.java | 26 +++---- .../readdata/AbstractInputStreamReadData.java | 71 ------------------- .../n5/readdata/InputStreamReadData.java | 57 +++++++++++++-- .../saalfeldlab/n5/readdata/LazyReadData.java | 21 ++++-- .../saalfeldlab/n5/readdata/ReadData.java | 4 +- .../n5/readdata/ReadDataTests.java | 9 +-- 6 files changed, 85 insertions(+), 103 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessReadData.java index 77c3c4efd..eb16501de 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessReadData.java @@ -17,7 +17,7 @@ public class KeyValueAccessReadData implements ReadData { private final long offset; private long length; - KeyValueAccessReadData(LazyRead lazyRead) { + KeyValueAccessReadData(final LazyRead lazyRead) { this(lazyRead, 0, -1); } @@ -28,11 +28,11 @@ public class KeyValueAccessReadData implements ReadData { } @Override - public ReadData materialize() throws N5IOException { - if (materialized == null) - materialized = lazyRead.materialize(offset, length); - return materialized; - } + public ReadData materialize() throws N5IOException { + if (materialized == null) + materialized = lazyRead.materialize(offset, length); + return this; + } /** * Returns a {@link ReadData} whose length is limited to the given value. @@ -50,29 +50,31 @@ public ReadData materialize() throws N5IOException { public ReadData slice(final long offset, final long length) throws N5IOException { if (offset < 0) throw new IndexOutOfBoundsException("Negative offset: " + offset); - + if (materialized != null) return materialized.slice(offset, length); // if a slice of indeterminate length is requested, but the // length is already known, use the known length; - final int lengthArg; + final long lengthArg; if (this.length > 0 && length < 0) - lengthArg = (int)(this.length - offset); + lengthArg = this.length - offset; else - lengthArg = (int)length; + lengthArg = length; return new KeyValueAccessReadData(lazyRead, this.offset + offset, lengthArg); } @Override public InputStream inputStream() throws N5IOException, IllegalStateException { - return materialize().inputStream(); + materialize(); + return materialized.inputStream(); } @Override public byte[] allBytes() throws N5IOException, IllegalStateException { - return materialize().allBytes(); + materialize(); + return materialized.allBytes(); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java deleted file mode 100644 index 854f7d4f5..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/AbstractInputStreamReadData.java +++ /dev/null @@ -1,71 +0,0 @@ -/*- - * #%L - * Not HDF5 - * %% - * Copyright (C) 2017 - 2025 Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.readdata; - -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; - -import org.apache.commons.io.IOUtils; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; - -// not thread-safe -abstract class AbstractInputStreamReadData implements ReadData { - - private ByteArrayReadData bytes; - - @Override - public ReadData materialize() throws N5IOException { - if (bytes == null) { - final byte[] data; - final int length = (int) length(); - if (length >= 0) { - data = new byte[length]; - try( InputStream is = inputStream()) { - new DataInputStream(is).readFully(data); - } catch (IOException e) { - throw new N5IOException(e); - } - } else { - try( InputStream is = inputStream()) { - data = IOUtils.toByteArray(is); - } catch (IOException e) { - throw new N5IOException(e); - } - } - bytes = new ByteArrayReadData(data); - } - return bytes; - } - - @Override - public byte[] allBytes() throws N5IOException, IllegalStateException { - return materialize().allBytes(); - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java index 0c1f35128..1e87a36cb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -28,10 +28,14 @@ */ package org.janelia.saalfeldlab.n5.readdata; +import java.io.DataInputStream; +import java.io.IOException; import java.io.InputStream; +import org.apache.commons.io.IOUtils; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; // not thread-safe -class InputStreamReadData extends AbstractInputStreamReadData { +class InputStreamReadData implements ReadData { private final InputStream inputStream; private final int length; @@ -43,18 +47,57 @@ class InputStreamReadData extends AbstractInputStreamReadData { @Override public long length() { - return length; + return (bytes != null) ? bytes.length() : length; + } + + @Override + public ReadData slice(final long offset, final long length) throws N5IOException { + materialize(); + return bytes.slice(offset, length); } private boolean inputStreamCalled = false; @Override public InputStream inputStream() throws IllegalStateException { - if (inputStreamCalled) { - throw new IllegalStateException("InputStream() already called"); - } else { + if (bytes != null) { + return bytes.inputStream(); + } else if (!inputStreamCalled) { inputStreamCalled = true; return inputStream; + } else { + throw new IllegalStateException("InputStream() already called"); + } + } + + @Override + public byte[] allBytes() throws N5IOException, IllegalStateException { + materialize(); + return bytes.allBytes(); + } + + private ByteArrayReadData bytes; + + @Override + public ReadData materialize() throws N5IOException { + if (bytes == null) { + final byte[] data; + if (length >= 0) { + data = new byte[length]; + try (InputStream is = inputStream()) { + new DataInputStream(is).readFully(data); + } catch (IOException e) { + throw new N5IOException(e); + } + } else { + try (InputStream is = inputStream()) { + data = IOUtils.toByteArray(is); + } catch (IOException e) { + throw new N5IOException(e); + } + } + bytes = new ByteArrayReadData(data); } + return this; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java index 9134e0ba2..3ffb41bfb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -42,13 +42,13 @@ class LazyReadData implements ReadData { } /** - * Construct a {@code LazyReadData} that uses the given {@code OutputStreamEncoder} to + * Construct a {@code LazyReadData} that uses the given {@code OutputStreamOperator} to * encode the given {@code ReadData}. * * @param data * the ReadData to encode * @param encoder - * OutputStreamEncoder to use for encoding + * OutputStreamOperator to use for encoding */ LazyReadData(final ReadData data, final OutputStreamOperator encoder) { this(outputStream -> { @@ -69,23 +69,30 @@ public ReadData materialize() throws N5IOException { writeTo(baos); bytes = new ByteArrayReadData(baos.toByteArray()); } - return bytes; + return this; } @Override public long length() throws N5IOException { + return (bytes != null) ? bytes.length() : -1; + } - return materialize().length(); + @Override + public ReadData slice(final long offset, final long length) throws N5IOException { + materialize(); + return bytes.slice(offset, length); } @Override public InputStream inputStream() throws N5IOException, IllegalStateException { - return materialize().inputStream(); + materialize(); + return bytes.inputStream(); } @Override public byte[] allBytes() throws N5IOException, IllegalStateException { - return materialize().allBytes(); + materialize(); + return bytes.allBytes(); } @Override @@ -105,7 +112,7 @@ public void writeTo(final OutputStream outputStream) throws N5IOException, Illeg * {@code UnaryOperator} that wraps {@code OutputStream} to intercept {@code * close()} and call {@code flush()} instead */ - private static OutputStreamOperator interceptClose = o -> new ProxyOutputStream(o) { + private static final OutputStreamOperator interceptClose = o -> new ProxyOutputStream(o) { @Override public void close() throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index a298897a1..80348924f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -196,11 +196,11 @@ default void writeTo(OutputStream outputStream) throws N5IOException, IllegalSta // /** - * Returns a new ReadData that uses the given {@code OutputStreamEncoder} to + * Returns a new ReadData that uses the given {@code OutputStreamOperator} to * encode this ReadData. * * @param encoder - * OutputStreamEncoder to use for encoding + * OutputStreamOperator to use for encoding * * @return encoded ReadData */ diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index b2e7f3c27..e62f9e2c1 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -34,7 +34,7 @@ public void testLazyReadData() throws IOException { }); assertTrue(readData instanceof LazyReadData); - readDataTestHelper(readData, N); + readDataTestHelper(readData, -1, N); sliceTestHelper(readData, N); } @@ -49,7 +49,7 @@ public void testByteArrayReadData() throws IOException { ReadData readData = ReadData.from(data).materialize(); assertTrue(readData instanceof ByteArrayReadData); - readDataTestHelper(readData, N); + readDataTestHelper(readData, N, N); readDataTestEncodeHelper(readData, N); sliceTestHelper(readData, N); } @@ -67,7 +67,7 @@ public int read() throws IOException { }; final ReadData readData = ReadData.from(is, N); - readDataTestHelper(readData, N); + readDataTestHelper(readData, N, N); sliceTestHelper(readData, N); } @@ -92,9 +92,10 @@ public void testFileKvaReadData() throws IOException { sliceTestHelper(readData, N); } - private void readDataTestHelper( ReadData readData, int N ) throws IOException { + private void readDataTestHelper( ReadData readData, int N, int materializedN ) throws IOException { assertEquals("full length", N, readData.length()); + assertEquals("full length after materialize", materializedN, readData.materialize().length()); } private void readDataTestEncodeHelper( ReadData readData, int N ) throws IOException { From 02013f44e7e4d3020d3e2fafd0d6f93db5a18b1b Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 2 Sep 2025 12:59:48 +0200 Subject: [PATCH 292/423] WIP --- .../n5/readdata/segment/SegmentStuff.java | 2 +- .../segment/SegmentedReadDataImpl.java | 27 +++++++++++++++++++ .../n5/readdata/segment/SegmentTest.java | 22 ++++++++++++++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java index b82bcfe11..e5f3c140a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java @@ -39,7 +39,7 @@ public interface SegmentedReadData extends ReadData { // TODO: return wrapper of readData with one segment comprising the whole ReadData // length of segment could return readData.length() static SegmentedReadData wrap(ReadData readData) { - throw new UnsupportedOperationException("TODO"); + return SegmentedReadDataImpl.wrap(readData); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java index 095d7f630..3da6d6528 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java @@ -53,6 +53,29 @@ public ReadData source() { } } + private static class EnclosingSegmentImpl extends SegmentImpl { + /* + } else if (segmentSource.equals(segment.source()) && segment instanceof EnclosingSegmentImpl) { + final EnclosingSegmentImpl s = (EnclosingSegmentImpl) segment; + return new SegmentedReadDataImpl( + delegate.slice(s.offset(), s.length()), + segmentSource, + this.offset + s.offset(), + Collections.singletonList(s)); + + */ + public EnclosingSegmentImpl(final ReadData source) { + super(source, 0, -1); + } + + @Override + public long length() { + return source().length(); + } + } + + + // assumes segments are ordered by location private SegmentedReadDataImpl(final ReadData delegate, final ReadData segmentSource, final long offset, final List segments) { super(delegate); @@ -79,6 +102,10 @@ public static SegmentedReadData wrap(final ReadData readData, final SegmentLocat return new SegmentedReadDataImpl(readData, segments); } + public static SegmentedReadData wrap(final ReadData readData) { + return new SegmentedReadDataImpl(readData, Collections.singletonList(new EnclosingSegmentImpl(readData))); + } + @Override public SegmentLocation location(final Segment segment) { if (segmentSource.equals(segment.source()) && segment instanceof SegmentLocation) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java index 053ca4e6d..e8614f76c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java @@ -1,5 +1,7 @@ package org.janelia.saalfeldlab.n5.readdata.segment; +import java.io.ByteArrayInputStream; +import java.io.InputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentLocation; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentedReadData; @@ -12,6 +14,8 @@ public class SegmentTest { private ReadData readData; + private ReadData readDataUnknownLength; + @Before public void createReadData() { @@ -20,6 +24,7 @@ public void createReadData() { data[i] = (byte) i; } readData = ReadData.from(data); + readDataUnknownLength = ReadData.from(new ByteArrayInputStream(data)); } @Test @@ -45,6 +50,22 @@ public void testWrap() { assertEquals(20, l2.length()); } + @Test + public void testWrapFully() { + System.out.println("readDataUnknownLength.length() = " + readDataUnknownLength.length()); + final SegmentedReadData r = SegmentedReadData.wrap(readDataUnknownLength); + assertEquals(1, r.segments().size()); + + final SegmentLocation l0 = r.location(r.segments().get(0)); + assertEquals(0, l0.offset()); + assertEquals(-1, l0.length()); + + r.materialize(); + final SegmentLocation l0m = r.location(r.segments().get(0)); + assertEquals(0, l0m.offset()); + assertEquals(100, l0m.length()); + } + @Test public void testSlice() { @@ -81,7 +102,6 @@ public void testSliceSegment() { assertEquals(1, s.segments().size()); final SegmentLocation l0 = s.location(s.segments().get(0)); - System.out.println("l0 = " + l0); assertEquals(0, l0.offset()); assertEquals(20, l0.length()); } From f503f55ceac628ce2f03a379604bf59560dcc492 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 2 Sep 2025 21:30:17 +0200 Subject: [PATCH 293/423] WIP slicing --- .../n5/readdata/segment/SegmentStuff.java | 78 +------------------ .../segment/SegmentedReadDataImpl.java | 36 +++++---- .../n5/readdata/segment/SegmentTest.java | 72 +++++++++++++---- 3 files changed, 80 insertions(+), 106 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java index e5f3c140a..3452be0fd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java @@ -67,6 +67,8 @@ static SegmentedReadData wrap(ReadData readData, SegmentLocation... locations) { @Override SegmentedReadData slice(final long offset, final long length) throws N5IOException; + @Override + SegmentedReadData materialize() throws N5IOException; } /** @@ -92,82 +94,6 @@ static SegmentLocation at(final long offset, final long length) { } } - - - // TODO rename - static class SegmentedReadData_Single extends ReadDataWrapper implements SegmentedReadData { - - private class SegmentImpl implements Segment, SegmentLocation { - - @Override - public ReadData source() { - return delegate; - } - - @Override - public long offset() { - return 0; - } - - @Override - public long length() { - return delegate.length(); - } - } - - private final SegmentImpl segment = new SegmentImpl(); - - SegmentedReadData_Single(final ReadData delegate) { - super(delegate); - } - - @Override - public SegmentLocation location(final Segment s) { - return (segment.equals(s)) ? segment : null; - } - - @Override - public List segments() { - return Collections.singletonList(segment); - } - - @Override - public SegmentedReadData slice(final Segment s) throws IllegalArgumentException, N5IOException { - if (segment.equals(s)) { - return this; - } else { - throw new IllegalArgumentException("Provided segment is not part of this "); - } - } - - @Override - public SegmentedReadData slice(final long offset, final long length) throws N5IOException { - final ReadData delegateSlice = delegate.slice(offset, length); - if (offset == segment.offset() && length == segment.length()) { - return this; - } else { - throw new UnsupportedOperationException("TODO"); - } - } - } - - - static class SegmentImpl implements Segment { - - private final ReadData source; - - // TODO: add Segment id for debugging ... - - public SegmentImpl(final ReadData source) { - this.source = source; - } - - @Override - public ReadData source() { - return source; - } - } - static class SegmentLocationImpl implements SegmentLocation { private final long offset; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java index 3da6d6528..28b6ef171 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java @@ -3,7 +3,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.Segment; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentLocation; @@ -75,6 +75,11 @@ public long length() { } + @Override + public SegmentedReadData materialize() throws N5IOException { + delegate.materialize(); + return this; + } // assumes segments are ordered by location private SegmentedReadDataImpl(final ReadData delegate, final ReadData segmentSource, final long offset, final List segments) { @@ -122,32 +127,35 @@ public List segments() { } @Override - public SegmentedReadData slice(final Segment segment) throws IllegalArgumentException, N5Exception.N5IOException { - if (segmentSource.equals(segment.source()) && segment instanceof SegmentImpl) { - final SegmentImpl s = (SegmentImpl) segment; - return new SegmentedReadDataImpl( - delegate.slice(s.offset(), s.length()), - segmentSource, - this.offset + s.offset(), - Collections.singletonList(s)); - } else { - throw new IllegalArgumentException(); + public SegmentedReadData slice(final Segment segment) throws IllegalArgumentException, N5IOException { + if (segmentSource.equals(segment.source())) { + if (segment instanceof EnclosingSegmentImpl) { + return this; + } else if (segment instanceof SegmentImpl) { + final SegmentImpl s = (SegmentImpl) segment; + return new SegmentedReadDataImpl( + delegate.slice(s.offset(), s.length()), + segmentSource, + this.offset + s.offset(), + Collections.singletonList(s)); + } } + throw new IllegalArgumentException(); } @Override - public SegmentedReadData slice(final long offset, final long length) throws N5Exception.N5IOException { + public SegmentedReadData slice(final long offset, final long length) throws N5IOException { final long sourceOffset = this.offset + offset; // fromIndex: find first segment with offset >= sourceOffset - int fromIndex = Collections.binarySearch(segments, new SegmentLocationImpl(sourceOffset, 0), SegmentLocation.COMPARATOR); + int fromIndex = Collections.binarySearch(segments, new SegmentLocationImpl(sourceOffset, -1), SegmentLocation.COMPARATOR); if (fromIndex < 0) { fromIndex = -fromIndex - 1; } // toIndex: find first segment with offset >= sourceOffset + length - int toIndex = Collections.binarySearch(segments, new SegmentLocationImpl(sourceOffset + length, 0), SegmentLocation.COMPARATOR); + int toIndex = Collections.binarySearch(segments, new SegmentLocationImpl(sourceOffset + length, -1), SegmentLocation.COMPARATOR); if (toIndex < 0) { toIndex = -toIndex - 1; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java index e8614f76c..887469072 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java @@ -50,22 +50,6 @@ public void testWrap() { assertEquals(20, l2.length()); } - @Test - public void testWrapFully() { - System.out.println("readDataUnknownLength.length() = " + readDataUnknownLength.length()); - final SegmentedReadData r = SegmentedReadData.wrap(readDataUnknownLength); - assertEquals(1, r.segments().size()); - - final SegmentLocation l0 = r.location(r.segments().get(0)); - assertEquals(0, l0.offset()); - assertEquals(-1, l0.length()); - - r.materialize(); - final SegmentLocation l0m = r.location(r.segments().get(0)); - assertEquals(0, l0m.offset()); - assertEquals(100, l0m.length()); - } - @Test public void testSlice() { @@ -106,4 +90,60 @@ public void testSliceSegment() { assertEquals(20, l0.length()); } + @Test + public void testWrapFully() { + final SegmentedReadData r = SegmentedReadData.wrap(readDataUnknownLength); + assertEquals(1, r.segments().size()); + + final SegmentLocation l0 = r.location(r.segments().get(0)); + assertEquals(0, l0.offset()); + assertEquals(-1, l0.length()); + + final SegmentedReadData m = r.materialize(); + final SegmentLocation l0m = r.location(r.segments().get(0)); + assertEquals(0, l0m.offset()); + assertEquals(100, l0m.length()); + + final SegmentLocation l0m2 = m.location(m.segments().get(0)); + assertEquals(0, l0m2.offset()); + assertEquals(100, l0m2.length()); + } + + @Test + public void testSliceFullyWrapped() { + final SegmentedReadData r = SegmentedReadData.wrap(readDataUnknownLength); + final SegmentedReadData s = r.slice(0, 100); + assertEquals(100, s.length()); + + assertEquals(1, s.segments().size()); + + final SegmentLocation l0 = s.location(s.segments().get(0)); + assertEquals(0, l0.offset()); + assertEquals(100, l0.length()); + + final SegmentedReadData s0 = r.slice(0, 99); + assertEquals(99, s0.length()); + assertEquals(0, s0.segments().size()); + + final SegmentedReadData s1 = r.slice(1, 99); + assertEquals(99, s1.length()); + assertEquals(0, s1.segments().size()); + } + + @Test + public void testSliceSegmentFullyWrapped() { + + final SegmentedReadData r = SegmentedReadData.wrap(readDataUnknownLength); + final SegmentedReadData s = r.slice(r.segments().get(0)); + assertEquals(-1, s.length()); + + assertEquals(1, s.segments().size()); + + final SegmentLocation l0 = s.location(s.segments().get(0)); + assertEquals(0, l0.offset()); + assertEquals(-1, l0.length()); + } + + + } From 686da94f0e441015b1fc1a8a6f87f9f3a28992b6 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 2 Sep 2025 21:42:26 +0200 Subject: [PATCH 294/423] WIP clean up --- .../n5/readdata/segment/SegmentStuff.java | 60 +----------- .../segment/SegmentedReadDataImpl.java | 98 ++++++++++++++----- 2 files changed, 78 insertions(+), 80 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java index 3452be0fd..41c229805 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java @@ -53,6 +53,11 @@ static SegmentedReadData wrap(ReadData readData, SegmentLocation... locations) { return SegmentedReadDataImpl.wrap(readData, locations); } + @Override + default SegmentedReadData limit(final long length) throws N5IOException { + return slice(0, length); + } + /** * Return a {@code SegmentedReadData} wrapping a slice containing exactly the given segment. * @@ -122,59 +127,4 @@ public String toString() { '}'; } } - - static class ReadDataWrapper implements ReadData { - final ReadData delegate; - - ReadDataWrapper(final ReadData delegate) { - this.delegate = delegate; - } - - @Override - public long length() throws N5IOException { - return delegate.length(); - } - - @Override - public ReadData limit(final long length) throws N5IOException { - return delegate.limit(length); - } - - @Override - public ReadData slice(final long offset, final long length) throws N5IOException { - return delegate.slice(offset, length); - } - - @Override - public InputStream inputStream() throws N5IOException, IllegalStateException { - return delegate.inputStream(); - } - - @Override - public byte[] allBytes() throws N5IOException, IllegalStateException { - return delegate.allBytes(); - } - - @Override - public ByteBuffer toByteBuffer() throws N5IOException, IllegalStateException { - return delegate.toByteBuffer(); - } - - @Override - public ReadData materialize() throws N5IOException { - return delegate.materialize(); - } - - @Override - public void writeTo(final OutputStream outputStream) throws N5IOException, IllegalStateException { - delegate.writeTo(outputStream); - } - - @Override - public ReadData encode(final OutputStreamOperator encoder) { - return delegate.encode(encoder); - } - } - - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java index 28b6ef171..4f604b28d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java @@ -1,5 +1,8 @@ package org.janelia.saalfeldlab.n5.readdata.segment; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -10,21 +13,35 @@ import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentLocationImpl; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentedReadData; -class SegmentedReadDataImpl extends SegmentStuff.ReadDataWrapper implements SegmentedReadData { +class SegmentedReadDataImpl implements SegmentedReadData { - // segments, ordered by location - private final List segments; + /** + * The {@code ReadData} providing our data. This is either {@code segmentSource} in + * which case {@code offset==0}, or a slice into {@code segmentSource} in + * which case {@code offset} is the offset of the slice. + */ + private final ReadData delegate; + /** + * The {@link Segment#source() source} {@code ReadData} of all contained + * segments. + */ private final ReadData segmentSource; + /** + * The offset of {@code delegate} withr espect to {@code segmentSource}. + */ private final long offset; + /** + * Contained segments, ordered by location. + */ + private final List segments; + private static class SegmentImpl implements Segment, SegmentLocation { private final ReadData source; - private final long offset; - private final long length; public SegmentImpl(final ReadData source, final SegmentLocation location) { @@ -54,16 +71,7 @@ public ReadData source() { } private static class EnclosingSegmentImpl extends SegmentImpl { - /* - } else if (segmentSource.equals(segment.source()) && segment instanceof EnclosingSegmentImpl) { - final EnclosingSegmentImpl s = (EnclosingSegmentImpl) segment; - return new SegmentedReadDataImpl( - delegate.slice(s.offset(), s.length()), - segmentSource, - this.offset + s.offset(), - Collections.singletonList(s)); - - */ + public EnclosingSegmentImpl(final ReadData source) { super(source, 0, -1); } @@ -74,16 +82,9 @@ public long length() { } } - - @Override - public SegmentedReadData materialize() throws N5IOException { - delegate.materialize(); - return this; - } - // assumes segments are ordered by location private SegmentedReadDataImpl(final ReadData delegate, final ReadData segmentSource, final long offset, final List segments) { - super(delegate); + this.delegate = delegate; this.segmentSource = segmentSource; this.offset = offset; this.segments = segments; @@ -98,7 +99,7 @@ private SegmentedReadDataImpl(final ReadData delegate, final List s // should return Pair where segments are in // the same order as locations. // TODO: use List instead of SegmentLocation[] - public static SegmentedReadData wrap(final ReadData readData, final SegmentLocation[] locations) { + static SegmentedReadData wrap(final ReadData readData, final SegmentLocation[] locations) { final List segments = new ArrayList<>(locations.length); for (SegmentLocation l : locations) { segments.add(new SegmentImpl(readData, l)); @@ -107,7 +108,7 @@ public static SegmentedReadData wrap(final ReadData readData, final SegmentLocat return new SegmentedReadDataImpl(readData, segments); } - public static SegmentedReadData wrap(final ReadData readData) { + static SegmentedReadData wrap(final ReadData readData) { return new SegmentedReadDataImpl(readData, Collections.singletonList(new EnclosingSegmentImpl(readData))); } @@ -126,6 +127,11 @@ public List segments() { return segments; } + @Override + public long length() throws N5IOException { + return delegate.length(); + } + @Override public SegmentedReadData slice(final Segment segment) throws IllegalArgumentException, N5IOException { if (segmentSource.equals(segment.source())) { @@ -176,4 +182,46 @@ public SegmentedReadData slice(final long offset, final long length) throws N5IO this.offset + offset, contained); } + + @Override + public InputStream inputStream() throws N5IOException, IllegalStateException { + return delegate.inputStream(); + } + + @Override + public byte[] allBytes() throws N5IOException, IllegalStateException { + return delegate.allBytes(); + } + + @Override + public ByteBuffer toByteBuffer() throws N5IOException, IllegalStateException { + return delegate.toByteBuffer(); + } + + @Override + public SegmentedReadData materialize() throws N5IOException { + delegate.materialize(); + return this; + } + + @Override + public void writeTo(final OutputStream outputStream) throws N5IOException, IllegalStateException { + delegate.writeTo(outputStream); + } + + /** + * Returns a new ReadData that uses the given {@code OutputStreamOperator} to + * encode this SegmentedReadData. + *

      + * Note that segments are lost by encoding. + * + * @param encoder + * OutputStreamOperator to use for encoding + * + * @return encoded ReadData + */ + @Override + public ReadData encode(final OutputStreamOperator encoder) { + return delegate.encode(encoder); + } } From c6019fdbd2e1e7cd9b0f9be16fedb0e2b2aa4997 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 3 Sep 2025 22:58:34 +0200 Subject: [PATCH 295/423] WIP Concatenate --- .../n5/readdata/segment/Concatenate.java | 159 ++++++++++++++++++ .../segment/DefaultSegmentLocation.java | 30 ++++ ...mpl.java => DefaultSegmentedReadData.java} | 51 +++--- .../n5/readdata/segment/Segment.java | 11 ++ .../n5/readdata/segment/SegmentLocation.java | 18 ++ .../n5/readdata/segment/SegmentStuff.java | 124 +------------- .../readdata/segment/SegmentedReadData.java | 94 +++++++++++ .../n5/readdata/segment/ConcatenateTest.java | 82 +++++++++ .../n5/readdata/segment/SegmentTest.java | 36 +++- 9 files changed, 454 insertions(+), 151 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentLocation.java rename src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/{SegmentedReadDataImpl.java => DefaultSegmentedReadData.java} (74%) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenateTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java new file mode 100644 index 000000000..55c7eebd9 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java @@ -0,0 +1,159 @@ +package org.janelia.saalfeldlab.n5.readdata.segment; + +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +class Concatenate implements SegmentedReadData { + + private final List content; + private final ReadData delegate; + private final List segments; + private final List locations; + private final Map segmentTolocation; + private boolean locationsBuilt; + + Concatenate(final List content) { + this.content = content; + delegate = ReadData.from(os -> content.forEach(d -> d.writeTo(os))); + segments = new ArrayList<>(); + locations = new ArrayList<>(); + segmentTolocation = new HashMap<>(); + locationsBuilt = false; + } + + // constructor for slices + private Concatenate(final ReadData delegate, final List segments, + final List locations) { + content = null; + this.delegate = delegate; + this.segments = segments; + this.locations = locations; + segmentTolocation = new HashMap<>(); + for (int i = 0; i < segments.size(); i++) { + segmentTolocation.put(segments.get(i), locations.get(i)); + } + locationsBuilt = true; + } + + private boolean ensureKnownSize() { + if (!locationsBuilt) { + + long offset = 0; + for (int i = 0; i < content.size(); i++) { + final SegmentedReadData data = content.get(i); + if (data.length() < 0) { + throw new IllegalStateException("Some of concatenated ReadData don't know their length yet."); + } + segments.addAll(data.segments()); + for (Segment segment : data.segments()) { + final SegmentLocation l = data.location(segment); + locations.add(SegmentLocation.at(l.offset() + offset, l.length())); + } + offset += data.length(); + } + + for (int i = 0; i < segments.size(); i++) { + segmentTolocation.put(segments.get(i), locations.get(i)); + } + + locationsBuilt = true; + } + return true; + } + + @Override + public SegmentLocation location(final Segment segment) throws IllegalArgumentException { + ensureKnownSize(); + final SegmentLocation location = segmentTolocation.get(segment); + if (location == null) { + throw new IllegalArgumentException(); + } + return location; + } + + @Override + public List segments() { + return segments; + } + + @Override + public long length() throws N5Exception.N5IOException { + return delegate.length(); + } + + @Override + public SegmentedReadData slice(final Segment segment) throws IllegalArgumentException, N5Exception.N5IOException { + ensureKnownSize(); + final SegmentLocation l = location(segment); + return slice(l.offset(), l.length()); + } + + @Override + public SegmentedReadData slice(final long offset, final long length) throws N5Exception.N5IOException { + ensureKnownSize(); + + // fromIndex: find first segment with offset >= sourceOffset + int fromIndex = Collections.binarySearch(locations, SegmentLocation.at(offset, -1), SegmentLocation.COMPARATOR); + if (fromIndex < 0) { + fromIndex = -fromIndex - 1; + } + + // toIndex: find first segment with offset >= sourceOffset + length + int toIndex = Collections.binarySearch(locations, SegmentLocation.at(offset + length, -1), SegmentLocation.COMPARATOR); + if (toIndex < 0) { + toIndex = -toIndex - 1; + } + + // contained: find segments in [fromIndex, toIndex) with s.offset() + s.length() <= sourceOffset + length + final List containedSegments = new ArrayList<>(); + final List containedSegmentLocations = new ArrayList<>(); + for (int i = fromIndex; i < toIndex; ++i) { + final SegmentLocation l = locations.get(i); + if (l.offset() + l.length() <= offset + length) { + containedSegments.add(segments.get(i)); + containedSegmentLocations.add(SegmentLocation.at(l.offset() - offset, l.length())); + } + } + + return new Concatenate(delegate.slice(offset, length), containedSegments, containedSegmentLocations); + } + + @Override + public InputStream inputStream() throws N5Exception.N5IOException, IllegalStateException { + return delegate.inputStream(); + } + + @Override + public byte[] allBytes() throws N5Exception.N5IOException, IllegalStateException { + return delegate.allBytes(); + } + + @Override + public ByteBuffer toByteBuffer() throws N5Exception.N5IOException, IllegalStateException { + return delegate.toByteBuffer(); + } + + @Override + public SegmentedReadData materialize() throws N5Exception.N5IOException { + delegate.materialize(); + return this; + } + + @Override + public void writeTo(final OutputStream outputStream) throws N5Exception.N5IOException, IllegalStateException { + delegate.writeTo(outputStream); + } + + @Override + public ReadData encode(final OutputStreamOperator encoder) { + return delegate.encode(encoder); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentLocation.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentLocation.java new file mode 100644 index 000000000..3d63c6f55 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentLocation.java @@ -0,0 +1,30 @@ +package org.janelia.saalfeldlab.n5.readdata.segment; + +class DefaultSegmentLocation implements SegmentLocation { + + private final long offset; + private final long length; + + public DefaultSegmentLocation(final long offset, final long length) { + this.offset = offset; + this.length = length; + } + + @Override + public long offset() { + return offset; + } + + @Override + public long length() { + return length; + } + + @Override + public String toString() { + return "SegmentLocation{" + + "offset=" + offset + + ", length=" + length + + '}'; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java similarity index 74% rename from src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java rename to src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java index 4f604b28d..503db3584 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadDataImpl.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java @@ -8,12 +8,8 @@ import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.Segment; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentLocation; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentLocationImpl; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentedReadData; -class SegmentedReadDataImpl implements SegmentedReadData { +class DefaultSegmentedReadData implements SegmentedReadData { /** * The {@code ReadData} providing our data. This is either {@code segmentSource} in @@ -83,7 +79,7 @@ public long length() { } // assumes segments are ordered by location - private SegmentedReadDataImpl(final ReadData delegate, final ReadData segmentSource, final long offset, final List segments) { + private DefaultSegmentedReadData(final ReadData delegate, final ReadData segmentSource, final long offset, final List segments) { this.delegate = delegate; this.segmentSource = segmentSource; this.offset = offset; @@ -91,32 +87,43 @@ private SegmentedReadDataImpl(final ReadData delegate, final ReadData segmentSou } // assumes segments are ordered by location - private SegmentedReadDataImpl(final ReadData delegate, final List segments) { + private DefaultSegmentedReadData(final ReadData delegate, final List segments) { this(delegate, delegate, 0, segments); } - // TODO: does not assume ordered locations. to not lose track, this method - // should return Pair where segments are in - // the same order as locations. - // TODO: use List instead of SegmentLocation[] - static SegmentedReadData wrap(final ReadData readData, final SegmentLocation[] locations) { - final List segments = new ArrayList<>(locations.length); + static SegmentsAndData wrap(final ReadData readData, final List locations) { + final List sortedSegments = new ArrayList<>(locations.size()); for (SegmentLocation l : locations) { - segments.add(new SegmentImpl(readData, l)); + sortedSegments.add(new SegmentImpl(readData, l)); } - segments.sort(SegmentLocation.COMPARATOR); - return new SegmentedReadDataImpl(readData, segments); + final List segments = new ArrayList<>(sortedSegments); + + sortedSegments.sort(SegmentLocation.COMPARATOR); + final DefaultSegmentedReadData data = new DefaultSegmentedReadData(readData, sortedSegments); + + return new SegmentsAndData() { + + @Override + public List segments() { + return segments; + } + + @Override + public SegmentedReadData data() { + return data; + } + }; } static SegmentedReadData wrap(final ReadData readData) { - return new SegmentedReadDataImpl(readData, Collections.singletonList(new EnclosingSegmentImpl(readData))); + return new DefaultSegmentedReadData(readData, Collections.singletonList(new EnclosingSegmentImpl(readData))); } @Override public SegmentLocation location(final Segment segment) { if (segmentSource.equals(segment.source()) && segment instanceof SegmentLocation) { final SegmentLocation l = (SegmentLocation) segment; - return offset == 0 ? l : new SegmentLocationImpl(l.offset() - offset, l.length()); + return offset == 0 ? l : new DefaultSegmentLocation(l.offset() - offset, l.length()); } else { throw new IllegalArgumentException(); } @@ -139,7 +146,7 @@ public SegmentedReadData slice(final Segment segment) throws IllegalArgumentExce return this; } else if (segment instanceof SegmentImpl) { final SegmentImpl s = (SegmentImpl) segment; - return new SegmentedReadDataImpl( + return new DefaultSegmentedReadData( delegate.slice(s.offset(), s.length()), segmentSource, this.offset + s.offset(), @@ -155,13 +162,13 @@ public SegmentedReadData slice(final long offset, final long length) throws N5IO final long sourceOffset = this.offset + offset; // fromIndex: find first segment with offset >= sourceOffset - int fromIndex = Collections.binarySearch(segments, new SegmentLocationImpl(sourceOffset, -1), SegmentLocation.COMPARATOR); + int fromIndex = Collections.binarySearch(segments, SegmentLocation.at(sourceOffset, -1), SegmentLocation.COMPARATOR); if (fromIndex < 0) { fromIndex = -fromIndex - 1; } // toIndex: find first segment with offset >= sourceOffset + length - int toIndex = Collections.binarySearch(segments, new SegmentLocationImpl(sourceOffset + length, -1), SegmentLocation.COMPARATOR); + int toIndex = Collections.binarySearch(segments, SegmentLocation.at(sourceOffset + length, -1), SegmentLocation.COMPARATOR); if (toIndex < 0) { toIndex = -toIndex - 1; } @@ -176,7 +183,7 @@ public SegmentedReadData slice(final long offset, final long length) throws N5IO }); contained.trimToSize(); - return new SegmentedReadDataImpl( + return new DefaultSegmentedReadData( delegate.slice(offset, length), segmentSource, this.offset + offset, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java new file mode 100644 index 000000000..e5dba4e51 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java @@ -0,0 +1,11 @@ +package org.janelia.saalfeldlab.n5.readdata.segment; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +/** + * A particular segment in a source {@code ReadData}. + */ +public interface Segment { + + ReadData source(); +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java new file mode 100644 index 000000000..6525b57cf --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java @@ -0,0 +1,18 @@ +package org.janelia.saalfeldlab.n5.readdata.segment; + +import java.util.Comparator; + +public interface SegmentLocation { + + Comparator COMPARATOR = Comparator + .comparingLong(SegmentLocation::offset) + .thenComparingLong(SegmentLocation::length); + + long offset(); + + long length(); + + static SegmentLocation at(final long offset, final long length) { + return new DefaultSegmentLocation(offset, length); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java index 41c229805..362a19f94 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java @@ -1,130 +1,8 @@ package org.janelia.saalfeldlab.n5.readdata.segment; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - public class SegmentStuff { - public interface SegmentedReadData extends ReadData { - - /** - * Returns the {@code SegmentLocation} of {@code segment} in this {@code ReadData}. - *

      - * Note that this {@code ReadData} is not necessarily the source of the segment. - *

      - * The returned {@code SegmentLocation} may be {@code {offset=0, index=-1}}, which - * means that the segment comprises this whole {@code ReadData} (and the length of - * this {@code ReadData} is not yet known. - * - * @param segment - * the segment id - * - * @return location of the segment, or null - * @throws IllegalArgumentException if the segment is not contained in this ReadData - */ - SegmentLocation location(Segment segment) throws IllegalArgumentException; - - /** - * @return all segments contained in this {@code ReadData}. - */ - // Order is the same as the SegmentLocations given at construction - List segments(); - - // TODO: return wrapper of readData with one segment comprising the whole ReadData - // length of segment could return readData.length() - static SegmentedReadData wrap(ReadData readData) { - return SegmentedReadDataImpl.wrap(readData); - } - - /** - * Wrap {@code readData} and create segments at the given locations. - */ - // TODO: does not assume ordered locations. to not lose track, this method - // should return Pair where segments are in - // the same order as locations. - // TODO: use List instead of SegmentLocation[] - static SegmentedReadData wrap(ReadData readData, SegmentLocation... locations) { - return SegmentedReadDataImpl.wrap(readData, locations); - } - - @Override - default SegmentedReadData limit(final long length) throws N5IOException { - return slice(0, length); - } - - /** - * Return a {@code SegmentedReadData} wrapping a slice containing exactly the given segment. - * - * @param segment - * @return - * @throws IllegalArgumentException if the segment is not contained in this ReadData - * @throws N5IOException - */ - SegmentedReadData slice(Segment segment) throws IllegalArgumentException, N5IOException; - - // TODO: has all segments fully contained in requested slice. - @Override - SegmentedReadData slice(final long offset, final long length) throws N5IOException; - - @Override - SegmentedReadData materialize() throws N5IOException; - } - - /** - * A particular segment in a source {@code ReadData}. - */ - public interface Segment { - - ReadData source(); - } - - public interface SegmentLocation { - - Comparator COMPARATOR = Comparator - .comparingLong(SegmentLocation::offset) - .thenComparingLong(SegmentLocation::length); - - long offset(); - - long length(); - - static SegmentLocation at(final long offset, final long length) { - return new SegmentLocationImpl(offset, length); - } - } - - static class SegmentLocationImpl implements SegmentLocation { - - private final long offset; - private final long length; - - public SegmentLocationImpl(final long offset, final long length) { - this.offset = offset; - this.length = length; - } - - @Override - public long offset() { - return offset; - } - - @Override - public long length() { - return length; - } + public static void main(String[] args) { - @Override - public String toString() { - return "SegmentLocation{" + - "offset=" + offset + - ", length=" + length + - '}'; - } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java new file mode 100644 index 000000000..fb1f1168f --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java @@ -0,0 +1,94 @@ +package org.janelia.saalfeldlab.n5.readdata.segment; + +import java.util.Arrays; +import java.util.List; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +public interface SegmentedReadData extends ReadData { + + interface SegmentsAndData { + List segments(); + SegmentedReadData data(); + } + + /** + * Wrap {@code readData} and create one segment comprising the entire {@code readData}. + */ + static SegmentedReadData wrap(ReadData readData) { + return DefaultSegmentedReadData.wrap(readData); + } + + /** + * Wrap {@code readData} and create segments at the given locations. + */ + // TODO: does not assume ordered locations. to not lose track, this method + // should return Pair where segments are in + // the same order as locations. + static SegmentsAndData wrap(ReadData readData, SegmentLocation... locations) { + return wrap(readData, Arrays.asList(locations)); + } + + /** + * Wrap {@code readData} and create segments at the given locations. + */ + static SegmentsAndData wrap(ReadData readData, List locations) { + return DefaultSegmentedReadData.wrap(readData, locations); + } + + static SegmentedReadData concatenate(List readDatas) { + return new Concatenate(readDatas); + } + + + + /** + * Returns the {@code SegmentLocation} of {@code segment} in this {@code ReadData}. + *

      + * Note that this {@code ReadData} is not necessarily the source of the segment. + *

      + * The returned {@code SegmentLocation} may be {@code {offset=0, index=-1}}, which + * means that the segment comprises this whole {@code ReadData} (and the length of + * this {@code ReadData} is not yet known. + * + * @param segment + * the segment id + * + * @return location of the segment, or null + * + * @throws IllegalArgumentException + * if the segment is not contained in this ReadData + */ + SegmentLocation location(Segment segment) throws IllegalArgumentException; + + /** + * @return all segments contained in this {@code ReadData}. + */ + // Order is the same as the SegmentLocations given at construction + List segments(); + + @Override + default SegmentedReadData limit(final long length) throws N5Exception.N5IOException { + return slice(0, length); + } + + /** + * Return a {@code SegmentedReadData} wrapping a slice containing exactly the given segment. + * + * @param segment + * + * @return + * + * @throws IllegalArgumentException + * if the segment is not contained in this ReadData + * @throws N5Exception.N5IOException + */ + SegmentedReadData slice(Segment segment) throws IllegalArgumentException, N5Exception.N5IOException; + + // TODO: has all segments fully contained in requested slice. + @Override + SegmentedReadData slice(final long offset, final long length) throws N5Exception.N5IOException; + + @Override + SegmentedReadData materialize() throws N5Exception.N5IOException; +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenateTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenateTest.java new file mode 100644 index 000000000..ae24adfae --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenateTest.java @@ -0,0 +1,82 @@ +package org.janelia.saalfeldlab.n5.readdata.segment; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class ConcatenateTest { + + private ReadData readData; + + private ReadData readDataUnknownLength; + + @Before + public void createReadData() { + + final byte[] data = new byte[100]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) i; + } + readData = ReadData.from(data); + readDataUnknownLength = ReadData.from(new ByteArrayInputStream(data)); + } + + @Test + public void testConcatenate() { + + final SegmentLocation[] locations = { + SegmentLocation.at(10, 30), + SegmentLocation.at(0, 10), + SegmentLocation.at(40, 20)}; + final SegmentedReadData.SegmentsAndData segmentsAndData0 = SegmentedReadData.wrap(readData, locations); + final SegmentedReadData r0 = segmentsAndData0.data(); + final List segments0 = segmentsAndData0.segments(); + + final SegmentedReadData r1 = SegmentedReadData.wrap(readDataUnknownLength); + assertEquals(1, r1.segments().size()); + final List segments1 = Collections.singletonList(r1.segments().get(0)); + + final List datas = new ArrayList<>(); + datas.add(r0.slice(0,40)); + datas.add(r1); + datas.add(r0.slice(segments0.get(2))); + final SegmentedReadData c = SegmentedReadData.concatenate(datas); + + // TODO: create individual tests: + // Both materialize() and writeTo(OutputStream) should ensure that all SegmentLocations are known + // Otherwise we expect IllegalStateException + c.materialize(); +// c.writeTo(new ByteArrayOutputStream()); + +// for (Segment segment : segments0) { +// System.out.println("c.location(segment) = " + c.location(segment)); +// } +// System.out.println("c.location(segment) = " + c.location(segments1.get(0))); + + final SegmentLocation l1 = c.location(segments0.get(1)); + assertEquals(0, l1.offset()); + assertEquals(10, l1.length()); + + final SegmentLocation l0 = c.location(segments0.get(0)); + assertEquals(10, l0.offset()); + assertEquals(30, l0.length()); + + final SegmentLocation l3 = c.location(segments1.get(0)); + assertEquals(40, l3.offset()); + assertEquals(100, l3.length()); + + final SegmentLocation l2 = c.location(segments0.get(2)); + assertEquals(140, l2.offset()); + assertEquals(20, l2.length()); + + + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java index 887469072..83efabb1c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java @@ -1,10 +1,8 @@ package org.janelia.saalfeldlab.n5.readdata.segment; import java.io.ByteArrayInputStream; -import java.io.InputStream; +import java.util.List; import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentLocation; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentStuff.SegmentedReadData; import org.junit.Before; import org.junit.Test; @@ -34,7 +32,7 @@ public void testWrap() { SegmentLocation.at(0, 10), SegmentLocation.at(10, 10), SegmentLocation.at(40, 20)}; - final SegmentedReadData r = SegmentedReadData.wrap(readData, locations); + final SegmentedReadData r = SegmentedReadData.wrap(readData, locations).data(); assertEquals(3, r.segments().size()); final SegmentLocation l0 = r.location(r.segments().get(0)); @@ -50,6 +48,32 @@ public void testWrap() { assertEquals(20, l2.length()); } + @Test + public void testWrapOrder() { + + final SegmentLocation[] locations = { + SegmentLocation.at(10, 10), + SegmentLocation.at(0, 10), + SegmentLocation.at(40, 20)}; + final SegmentedReadData.SegmentsAndData segmentsAndData = SegmentedReadData.wrap(readData, locations); + final SegmentedReadData r = segmentsAndData.data(); + final List segments = segmentsAndData.segments(); + + assertEquals(3, segments.size()); + + final SegmentLocation l0 = r.location(segments.get(0)); + assertEquals(10, l0.offset()); + assertEquals(10, l0.length()); + + final SegmentLocation l1 = r.location(segments.get(1)); + assertEquals(0, l1.offset()); + assertEquals(10, l1.length()); + + final SegmentLocation l2 = r.location(segments.get(2)); + assertEquals(40, l2.offset()); + assertEquals(20, l2.length()); + } + @Test public void testSlice() { @@ -57,7 +81,7 @@ public void testSlice() { SegmentLocation.at(0, 10), SegmentLocation.at(10, 10), SegmentLocation.at(40, 20)}; - final SegmentedReadData r = SegmentedReadData.wrap(readData, locations); + final SegmentedReadData r = SegmentedReadData.wrap(readData, locations).data(); final SegmentedReadData s = r.slice(10, 60); assertEquals(60, s.length()); @@ -79,7 +103,7 @@ public void testSliceSegment() { SegmentLocation.at(0, 10), SegmentLocation.at(10, 10), SegmentLocation.at(40, 20)}; - final SegmentedReadData r = SegmentedReadData.wrap(readData, locations); + final SegmentedReadData r = SegmentedReadData.wrap(readData, locations).data(); final SegmentedReadData s = r.slice(r.segments().get(2)); assertEquals(20, s.length()); From 9d641f358564ed27ea62ed88d79420ae9fe1fafb Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 4 Sep 2025 10:41:03 +0200 Subject: [PATCH 296/423] WIP handle ReadData.slice with length < 0 --- .../n5/readdata/segment/Concatenate.java | 36 ++++++++++++----- .../segment/DefaultSegmentedReadData.java | 40 +++++++++++++------ 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java index 55c7eebd9..22a3868b3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java @@ -17,16 +17,18 @@ class Concatenate implements SegmentedReadData { private final ReadData delegate; private final List segments; private final List locations; - private final Map segmentTolocation; + private final Map segmentToLocation; private boolean locationsBuilt; + private long length; Concatenate(final List content) { this.content = content; delegate = ReadData.from(os -> content.forEach(d -> d.writeTo(os))); segments = new ArrayList<>(); locations = new ArrayList<>(); - segmentTolocation = new HashMap<>(); + segmentToLocation = new HashMap<>(); locationsBuilt = false; + length = -1; } // constructor for slices @@ -36,9 +38,9 @@ private Concatenate(final ReadData delegate, final List segments, this.delegate = delegate; this.segments = segments; this.locations = locations; - segmentTolocation = new HashMap<>(); + segmentToLocation = new HashMap<>(); for (int i = 0; i < segments.size(); i++) { - segmentTolocation.put(segments.get(i), locations.get(i)); + segmentToLocation.put(segments.get(i), locations.get(i)); } locationsBuilt = true; } @@ -59,9 +61,10 @@ private boolean ensureKnownSize() { } offset += data.length(); } + length = offset; for (int i = 0; i < segments.size(); i++) { - segmentTolocation.put(segments.get(i), locations.get(i)); + segmentToLocation.put(segments.get(i), locations.get(i)); } locationsBuilt = true; @@ -72,7 +75,7 @@ private boolean ensureKnownSize() { @Override public SegmentLocation location(final Segment segment) throws IllegalArgumentException { ensureKnownSize(); - final SegmentLocation location = segmentTolocation.get(segment); + final SegmentLocation location = segmentToLocation.get(segment); if (location == null) { throw new IllegalArgumentException(); } @@ -86,7 +89,18 @@ public List segments() { @Override public long length() throws N5Exception.N5IOException { - return delegate.length(); + if (length < 0) { + length = 0; + for (final ReadData data : content) { + final long l = data.length(); + if (l < 0) { + length = -1; + break; + } + length += l; + } + } + return length; } @Override @@ -99,6 +113,8 @@ public SegmentedReadData slice(final Segment segment) throws IllegalArgumentExce @Override public SegmentedReadData slice(final long offset, final long length) throws N5Exception.N5IOException { ensureKnownSize(); + final ReadData delegateSlice = delegate.slice(offset, length); + final long sliceLength = delegateSlice.length(); // fromIndex: find first segment with offset >= sourceOffset int fromIndex = Collections.binarySearch(locations, SegmentLocation.at(offset, -1), SegmentLocation.COMPARATOR); @@ -107,7 +123,7 @@ public SegmentedReadData slice(final long offset, final long length) throws N5Ex } // toIndex: find first segment with offset >= sourceOffset + length - int toIndex = Collections.binarySearch(locations, SegmentLocation.at(offset + length, -1), SegmentLocation.COMPARATOR); + int toIndex = Collections.binarySearch(locations, SegmentLocation.at(offset + sliceLength, -1), SegmentLocation.COMPARATOR); if (toIndex < 0) { toIndex = -toIndex - 1; } @@ -117,13 +133,13 @@ public SegmentedReadData slice(final long offset, final long length) throws N5Ex final List containedSegmentLocations = new ArrayList<>(); for (int i = fromIndex; i < toIndex; ++i) { final SegmentLocation l = locations.get(i); - if (l.offset() + l.length() <= offset + length) { + if (l.offset() + l.length() <= offset + sliceLength) { containedSegments.add(segments.get(i)); containedSegmentLocations.add(SegmentLocation.at(l.offset() - offset, l.length())); } } - return new Concatenate(delegate.slice(offset, length), containedSegments, containedSegmentLocations); + return new Concatenate(delegateSlice, containedSegments, containedSegmentLocations); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java index 503db3584..44c29ca68 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java @@ -159,7 +159,10 @@ public SegmentedReadData slice(final Segment segment) throws IllegalArgumentExce @Override public SegmentedReadData slice(final long offset, final long length) throws N5IOException { + final ReadData delegateSlice = delegate.slice(offset, length); final long sourceOffset = this.offset + offset; + final long sliceLength = delegateSlice.length(); + // fromIndex: find first segment with offset >= sourceOffset int fromIndex = Collections.binarySearch(segments, SegmentLocation.at(sourceOffset, -1), SegmentLocation.COMPARATOR); @@ -168,26 +171,37 @@ public SegmentedReadData slice(final long offset, final long length) throws N5IO } // toIndex: find first segment with offset >= sourceOffset + length - int toIndex = Collections.binarySearch(segments, SegmentLocation.at(sourceOffset + length, -1), SegmentLocation.COMPARATOR); - if (toIndex < 0) { - toIndex = -toIndex - 1; + int toIndex; + if (sliceLength < 0) { + toIndex = segments.size(); + } else { + toIndex = Collections.binarySearch(segments, SegmentLocation.at(sourceOffset + sliceLength, -1), SegmentLocation.COMPARATOR); + if (toIndex < 0) { + toIndex = -toIndex - 1; + } } // contained: find segments in [fromIndex, toIndex) with s.offset() + s.length() <= sourceOffset + length final List candidates = segments.subList(fromIndex, toIndex); - final ArrayList contained = new ArrayList<>(candidates.size()); - candidates.forEach(s -> { - if (s.offset() + s.length() <= sourceOffset + length) { - contained.add(s); - } - }); - contained.trimToSize(); + final List sliceSegments; + if (sliceLength < 0) { + sliceSegments = candidates; + } else { + final ArrayList contained = new ArrayList<>(candidates.size()); + candidates.forEach(s -> { + if (s.offset() + s.length() <= sourceOffset + sliceLength) { + contained.add(s); + } + }); + contained.trimToSize(); + sliceSegments = contained; + } return new DefaultSegmentedReadData( - delegate.slice(offset, length), + delegateSlice, segmentSource, - this.offset + offset, - contained); + sourceOffset, + sliceSegments); } @Override From d719fbbf68d8ae0700a04e1c12ef85e3c698227f Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 4 Sep 2025 22:11:43 +0200 Subject: [PATCH 297/423] Add ReadData.materialize javadoc (implementation note) --- .../java/org/janelia/saalfeldlab/n5/readdata/ReadData.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 80348924f..a4a4c7eaa 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -158,6 +158,11 @@ default ByteBuffer toByteBuffer() throws N5IOException, IllegalStateException { *

      * The returned {@code ReadData} has a known {@link #length} and multiple * {@link #inputStream InputStreams} can be opened on it. + *

      + * Implementation note: This should be preferably implemented to return + * {@code this}. For example, materialize into a new {@code byte[]}, {@code + * ReadData}, or similar and then delegate to this materialized version + * internally. * * @return * a materialized ReadData. From 44d5a75c70f9501f24e932a05dc4c472d4c2f462 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 3 Sep 2025 23:21:18 +0200 Subject: [PATCH 298/423] Add ReadData.requireLength --- .../n5/KeyValueAccessReadData.java | 19 ++++++++++++------- .../n5/readdata/ByteArrayReadData.java | 5 +++++ .../n5/readdata/InputStreamReadData.java | 11 +++++++++++ .../saalfeldlab/n5/readdata/LazyReadData.java | 14 +++++++++++++- .../saalfeldlab/n5/readdata/ReadData.java | 17 ++++++++++++++--- 5 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessReadData.java index eb16501de..2208eb8bf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessReadData.java @@ -29,8 +29,10 @@ public class KeyValueAccessReadData implements ReadData { @Override public ReadData materialize() throws N5IOException { - if (materialized == null) + if (materialized == null) { materialized = lazyRead.materialize(offset, length); + length = materialized.length(); + } return this; } @@ -78,13 +80,16 @@ public byte[] allBytes() throws N5IOException, IllegalStateException { } @Override - public long length() throws N5IOException { - if (materialized != null) - return materialized.length(); - if (length < 0) { - length = lazyRead.size() - offset; - } + public long length() { return length; } + @Override + public long requireLength() throws N5IOException { + if (length < 0) { + length = lazyRead.size() - offset; + } + return length; + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArrayReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArrayReadData.java index 92dcbd820..627f2a560 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArrayReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ByteArrayReadData.java @@ -65,6 +65,11 @@ public long length() { return length; } + @Override + public long requireLength() { + return length; + } + @Override public InputStream inputStream() { return new ByteArrayInputStream(data, offset, length); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java index 1e87a36cb..d0073e444 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java @@ -50,6 +50,17 @@ public long length() { return (bytes != null) ? bytes.length() : length; } + // TODO: remove? Should this be the default implementation? + @Override + public long requireLength() throws N5IOException { + long l = length(); + if ( l >= 0 ) { + return l; + } else { + return materialize().length(); + } + } + @Override public ReadData slice(final long offset, final long length) throws N5IOException { materialize(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java index 3ffb41bfb..044194deb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -73,10 +73,22 @@ public ReadData materialize() throws N5IOException { } @Override - public long length() throws N5IOException { + public long length() { return (bytes != null) ? bytes.length() : -1; } + // TODO: remove? Should this be the default implementation? + // TODO: could just always return materialize().length()? + @Override + public long requireLength() throws N5IOException { + long l = length(); + if ( l >= 0 ) { + return l; + } else { + return materialize().length(); + } + } + @Override public ReadData slice(final long offset, final long length) throws N5IOException { materialize(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index a4a4c7eaa..1c7faca07 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -58,13 +58,24 @@ public interface ReadData { * {@code -1}. * * @return number of bytes, if known, or -1 + */ + default long length() { + return -1; + } + + /** + * Returns number of bytes in this {@link ReadData}. If the length is not + * currently know, this method may retrieve the length using I/O operations, + * {@link #materialize} this {@code ReadData}, or perform any other steps + * necessary to obtain the length. + * + * @return number of bytes * * @throws N5IOException * if an I/O error occurs while trying to get the length */ - default long length() throws N5IOException { - return -1; - } + long requireLength() throws N5IOException; + // TODO: default: {materialize(); return length();} /** * Returns a {@link ReadData} whose length is limited to the given value. From c4ea774f623ecfc2200c803249a125e9b650ab05 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 5 Sep 2025 10:06:56 +0200 Subject: [PATCH 299/423] WIP requireLength --- .../n5/readdata/segment/Concatenate.java | 38 +++++++++++++------ .../segment/DefaultSegmentedReadData.java | 7 +++- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java index 22a3868b3..fd956789e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.Map; import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; class Concatenate implements SegmentedReadData { @@ -45,9 +46,14 @@ private Concatenate(final ReadData delegate, final List segments, locationsBuilt = true; } - private boolean ensureKnownSize() { + /** + * Verify that all {@code content} elements have known length. + * Builds {@code segments} and {@code locations} if they have not been built yet. + * + * @throws IllegalStateException if any of the concatenated ReadData don't know their length yet + */ + private void ensureKnownSize() throws IllegalStateException { if (!locationsBuilt) { - long offset = 0; for (int i = 0; i < content.size(); i++) { final SegmentedReadData data = content.get(i); @@ -69,7 +75,6 @@ private boolean ensureKnownSize() { locationsBuilt = true; } - return true; } @Override @@ -88,7 +93,7 @@ public List segments() { } @Override - public long length() throws N5Exception.N5IOException { + public long length() { if (length < 0) { length = 0; for (final ReadData data : content) { @@ -104,14 +109,25 @@ public long length() throws N5Exception.N5IOException { } @Override - public SegmentedReadData slice(final Segment segment) throws IllegalArgumentException, N5Exception.N5IOException { + public long requireLength() throws N5IOException { + if (length < 0) { + length = 0; + for (final ReadData data : content) { + length += data.requireLength(); + } + } + return length; + } + + @Override + public SegmentedReadData slice(final Segment segment) throws IllegalArgumentException, N5IOException { ensureKnownSize(); final SegmentLocation l = location(segment); return slice(l.offset(), l.length()); } @Override - public SegmentedReadData slice(final long offset, final long length) throws N5Exception.N5IOException { + public SegmentedReadData slice(final long offset, final long length) throws N5IOException { ensureKnownSize(); final ReadData delegateSlice = delegate.slice(offset, length); final long sliceLength = delegateSlice.length(); @@ -143,28 +159,28 @@ public SegmentedReadData slice(final long offset, final long length) throws N5Ex } @Override - public InputStream inputStream() throws N5Exception.N5IOException, IllegalStateException { + public InputStream inputStream() throws N5IOException, IllegalStateException { return delegate.inputStream(); } @Override - public byte[] allBytes() throws N5Exception.N5IOException, IllegalStateException { + public byte[] allBytes() throws N5IOException, IllegalStateException { return delegate.allBytes(); } @Override - public ByteBuffer toByteBuffer() throws N5Exception.N5IOException, IllegalStateException { + public ByteBuffer toByteBuffer() throws N5IOException, IllegalStateException { return delegate.toByteBuffer(); } @Override - public SegmentedReadData materialize() throws N5Exception.N5IOException { + public SegmentedReadData materialize() throws N5IOException { delegate.materialize(); return this; } @Override - public void writeTo(final OutputStream outputStream) throws N5Exception.N5IOException, IllegalStateException { + public void writeTo(final OutputStream outputStream) throws N5IOException, IllegalStateException { delegate.writeTo(outputStream); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java index 44c29ca68..350deb60c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java @@ -135,10 +135,15 @@ public List segments() { } @Override - public long length() throws N5IOException { + public long length() { return delegate.length(); } + @Override + public long requireLength() throws N5IOException { + return delegate.requireLength(); + } + @Override public SegmentedReadData slice(final Segment segment) throws IllegalArgumentException, N5IOException { if (segmentSource.equals(segment.source())) { From 0b2c658b9de74291439b6af9e8779b73505b4b70 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 5 Sep 2025 12:52:54 +0200 Subject: [PATCH 300/423] Segment.source() is the SegmentedReadData that originally defined it Idea is that this allows to slice() the Segment from its source(), so we don't need to keep the sources around explicitly in shard implementation. --- .../segment/DefaultSegmentedReadData.java | 29 ++++++++++++------- .../n5/readdata/segment/Segment.java | 2 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java index 350deb60c..210fa8e9a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java @@ -36,15 +36,15 @@ class DefaultSegmentedReadData implements SegmentedReadData { private static class SegmentImpl implements Segment, SegmentLocation { - private final ReadData source; + private final SegmentedReadData source; private final long offset; private final long length; - public SegmentImpl(final ReadData source, final SegmentLocation location) { + public SegmentImpl(final SegmentedReadData source, final SegmentLocation location) { this(source, location.offset(), location.length()); } - public SegmentImpl(final ReadData source, final long offset, final long length) { + public SegmentImpl(final SegmentedReadData source, final long offset, final long length) { this.source = source; this.offset = offset; this.length = length; @@ -61,14 +61,14 @@ public long length() { } @Override - public ReadData source() { + public SegmentedReadData source() { return source; } } private static class EnclosingSegmentImpl extends SegmentImpl { - public EnclosingSegmentImpl(final ReadData source) { + public EnclosingSegmentImpl(final SegmentedReadData source) { super(source, 0, -1); } @@ -88,18 +88,20 @@ private DefaultSegmentedReadData(final ReadData delegate, final ReadData segment // assumes segments are ordered by location private DefaultSegmentedReadData(final ReadData delegate, final List segments) { - this(delegate, delegate, 0, segments); + this.delegate = delegate; + this.segmentSource = this; + this.offset = 0; + this.segments = segments; } static SegmentsAndData wrap(final ReadData readData, final List locations) { final List sortedSegments = new ArrayList<>(locations.size()); + final DefaultSegmentedReadData data = new DefaultSegmentedReadData(readData, sortedSegments); for (SegmentLocation l : locations) { - sortedSegments.add(new SegmentImpl(readData, l)); + sortedSegments.add(new SegmentImpl(data, l)); } final List segments = new ArrayList<>(sortedSegments); - sortedSegments.sort(SegmentLocation.COMPARATOR); - final DefaultSegmentedReadData data = new DefaultSegmentedReadData(readData, sortedSegments); return new SegmentsAndData() { @@ -115,8 +117,15 @@ public SegmentedReadData data() { }; } + private DefaultSegmentedReadData(final ReadData delegate) { + this.delegate = delegate; + this.segmentSource = this; + this.offset = 0; + this.segments = Collections.singletonList(new EnclosingSegmentImpl(this)); + } + static SegmentedReadData wrap(final ReadData readData) { - return new DefaultSegmentedReadData(readData, Collections.singletonList(new EnclosingSegmentImpl(readData))); + return new DefaultSegmentedReadData(readData); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java index e5dba4e51..a72b7ca39 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java @@ -7,5 +7,5 @@ */ public interface Segment { - ReadData source(); + SegmentedReadData source(); } From d29a4f2e9bb219470bd9a811335052b7e72f670a Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 5 Sep 2025 12:54:56 +0200 Subject: [PATCH 301/423] Revise DataCodecInfo to take DataType and blockSize instead of DatasetAttributes This will make more sense when instantiating from nested ShardCodecs --- .../saalfeldlab/n5/codec/BlockCodecInfo.java | 10 +++------- .../org/janelia/saalfeldlab/n5/codec/DataCodec.java | 13 +++++++++++++ .../janelia/saalfeldlab/n5/codec/DataCodecInfo.java | 1 + .../saalfeldlab/n5/codec/N5BlockCodecInfo.java | 7 +++---- .../saalfeldlab/n5/codec/RawBlockCodecInfo.java | 7 +++---- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java index cfeada9c8..3c17fc19b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java @@ -1,7 +1,7 @@ package org.janelia.saalfeldlab.n5.codec; -import java.util.Arrays; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -13,13 +13,9 @@ */ public interface BlockCodecInfo extends CodecInfo { - BlockCodec create(DatasetAttributes attributes, DataCodec... codecs); + BlockCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs); default BlockCodec create(final DatasetAttributes attributes, final DataCodecInfo... codecInfos) { - final DataCodec[] codecs = new DataCodec[codecInfos.length]; - Arrays.setAll(codecs, i -> codecInfos[i].create()); - return create(attributes, codecs); + return create(attributes.getDataType(), attributes.getBlockSize(), codecInfos); } - - // TODO: Should we have both create() signatures? } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java index be40d2e59..5c7795f0f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.codec; +import java.util.Arrays; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -60,4 +61,16 @@ static DataCodec concatenate(final DataCodec... codecs) { return new ConcatenatedDataCodec(codecs); } + + static DataCodec create(final DataCodecInfo... codecInfos) { + + if (codecInfos == null) + throw new NullPointerException(); + + final DataCodec[] codecs = new DataCodec[codecInfos.length]; + Arrays.setAll(codecs, i -> codecInfos[i].create()); + + return DataCodec.concatenate(codecs); + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java index ca3ab8c89..c1bec9e09 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.codec; +import java.util.Arrays; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java index 70c3555b3..7daa2d2d8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java @@ -1,6 +1,6 @@ package org.janelia.saalfeldlab.n5.codec; -import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @NameConfig.Name(value = N5BlockCodecInfo.TYPE) @@ -17,8 +17,7 @@ public String getType() { } @Override - public BlockCodec create(final DatasetAttributes attributes, final DataCodec... dataCodecs) { - return N5BlockCodecs.create(attributes.getDataType(), DataCodec.concatenate(dataCodecs)); + public BlockCodec create(final DataType dataType, final int[] blockSize, final DataCodecInfo... codecInfos) { + return N5BlockCodecs.create(dataType, DataCodec.create(codecInfos)); } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java index ed93433f0..9d0c3b90b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java @@ -2,7 +2,6 @@ import java.nio.ByteOrder; import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @@ -37,9 +36,9 @@ public ByteOrder getByteOrder() { } @Override - public BlockCodec create(final DatasetAttributes attributes, final DataCodec... dataCodecs) { - ensureValidByteOrder(attributes.getDataType(), getByteOrder()); - return RawBlockCodecs.create(attributes.getDataType(), byteOrder, attributes.getBlockSize(), DataCodec.concatenate(dataCodecs)); + public BlockCodec create(final DataType dataType, final int[] blockSize, final DataCodecInfo... codecInfos) { + ensureValidByteOrder(dataType, getByteOrder()); + return RawBlockCodecs.create(dataType, byteOrder, blockSize, DataCodec.create(codecInfos)); } private static void ensureValidByteOrder(final DataType dataType, final ByteOrder byteOrder) { From 455afce7be403d9101b0701f9e34070de7cdbdab Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 5 Sep 2025 12:54:59 +0200 Subject: [PATCH 302/423] WIP --- .../n5/shardstuff/ShardStuff3.java | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff3.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff3.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff3.java new file mode 100644 index 000000000..bf9c88a4d --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff3.java @@ -0,0 +1,94 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import java.util.List; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; + +public class ShardStuff3 { + + + + public interface RawShard extends DataBlock { + + // pos is relative to this shard + ReadData getElementData(long[] pos); + + // pos is relative to this shard + void setElementData(ReadData data, long[] pos); + + // TODO: removeElement(long[] pos) + } + + static class BasicRawShard { + + BasicRawShard(final ReadData data, final int[] size) { + + // TODO: + // [ ] read shard index (just assume some defaults) + // [ ] construct SegmentedReadData + // [ ] construct Map or flattened Segment[] + // [ ] implement setElementData + // [ ] implement getElementData + + } + + + + } + + + + + + + public interface RawShardCodec extends BlockCodec { + + @Override + RawShard decode(ReadData readData, long[] gridPosition) throws N5IOException; + } + + + + + + + // TODO: rename? + public interface Sharding { + + List> readBlocks(ReadData readData, List positions); + + ReadData writeBlocks(ReadData readData, List> blocks); + } + + public interface ShardCodecInfo extends CodecInfo { + /** + * Chunk size of the elements in this block. + * That is, (1, ...) for DataBlockCodecInfo, respectively inner block size for ShardCodecInfo. + */ + int[] getInnerBlockSize(); + + /** + * Nested BlockCodec. + * ({@code null} for DataBlockCodec. + */ + BlockCodecInfo getInnerBlockCodecInfo(); + + DataCodecInfo[] getInnerDataCodecInfos(); + + // TODO: IndexCodec + + RawShardCodec createRaw(int[] blockSize, DataCodecInfo... codecs); + + // TODO: not sure about this one. + // This could recursively built the whole codec list? + // The result would have to be a BlockCodec. + // +// ??? create(DataType dataType, int[] blockSize, DataCodecInfo... codecs) {} + } +} From 7f87a746d96c9eb828e6e59d80d23971fafd7eb1 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 5 Sep 2025 13:14:25 +0200 Subject: [PATCH 303/423] WIP ShardIndex to/from Segments --- .../saalfeldlab/n5/shardstuff/ShardIndex.java | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java new file mode 100644 index 000000000..9441dfc1e --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java @@ -0,0 +1,183 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.function.IntFunction; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.Segment; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData.SegmentsAndData; + +public class ShardIndex { + + static class Flattened { + + final int[] size; + private final int[] stride; + final T[] data; + + Flattened(final int[] size, final IntFunction createArray) { + this.size = size; + stride = stride(size); + data = createArray.apply(numElements(size)); + } + + Flattened(final int[] size, final T[] data) { + this.size = size; + stride = stride(size); + this.data = data; + } + + T get(long... position) { + return data[index(position)]; + } + + void set(T value, long... position) { + data[index(position)] = value; + } + + private int index(long... position) { + int index = 0; + for (int i = 0; i < stride.length; i++) { + index += stride[i] * position[i]; + } + return index; + } + + public int[] size() { + return size; + } + } + + static int numElements(final int[] size) { + int numElements = 1; + for (int s : size) { + numElements *= s; + } + return numElements; + } + + static int[] stride(final int[] size) { + final int n = size.length; + final int[] stride = new int[n]; + stride[0] = 1; + for (int i = 1; i < n; i++) { + stride[i] = stride[i - 1] * size[i - 1]; + } + return stride; + } + + /** + * Special value indicating an empty block entry in the index. + * Used for both offset and length when a block doesn't exist. + */ + static final long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; + + // TODO do we need additional offset here? + static Flattened fromDataBlock( final DataBlock block ) { + + assert block.getSize()[ 0 ] == 2; + + final int[] blockSize = block.getSize(); + final long[] blockData = block.getData(); + + final int[] size = Arrays.copyOfRange(blockSize, 1, blockSize.length); + final int n = numElements(size); + final SegmentLocation[] locations = new SegmentLocation[n]; + + for (int i = 0; i < n; i++) { + long offset = blockData[ i * 2]; + long length = blockData[ i * 2 + 1]; + if (offset != EMPTY_INDEX_NBYTES && length != EMPTY_INDEX_NBYTES) { + locations[i] = SegmentLocation.at(offset, length); + } + } + return new Flattened<>(size, locations); + } + + // TODO do we use offset? If not, remove! + static DataBlock toDataBlock( final Flattened locations, final long offset ) { + + final SegmentLocation[] data = locations.data; + + final int[] blockSize = prepend(2, locations.size); + final long[] blockData = new long[data.length * 2]; + + for (int i = 0; i < data.length; ++i) { + if (data[i] != null) { + blockData[i * 2] = data[i].offset() + offset; + blockData[i * 2 + 1] = data[i].length(); + } else { + blockData[i * 2] = EMPTY_INDEX_NBYTES; + blockData[i * 2 + 1] = EMPTY_INDEX_NBYTES; + } + } + return new LongArrayDataBlock(blockSize, new long[blockSize.length], blockData); + } + + /** + * Prepends a value to an array. + * + * @param value the value to prepend + * @param array the original array + * @return a new array with the value prepended + */ + private static int[] prepend(final int value, final int[] array) { + + final int[] indexBlockSize = new int[array.length + 1]; + indexBlockSize[0] = value; + System.arraycopy(array, 0, indexBlockSize, 1, array.length); + return indexBlockSize; + } + + static Flattened locations(final Flattened segments, final SegmentedReadData readData) { + + final Segment[] data = segments.data; + final SegmentLocation[] locations = new SegmentLocation[data.length]; + for (int i = 0; i < data.length; ++i) { + final Segment segment = data[i]; + if ( segment != null ) { + locations[i] = readData.location(segment); + } + } + return new Flattened<>(segments.size, locations); + } + + interface SegmentIndexAndData { + Flattened index(); + SegmentedReadData data(); + } + + static SegmentIndexAndData segments(final Flattened locations, final ReadData readData) { + + final SegmentLocation[] locationsData = locations.data; + final Segment[] segmentsData = new Segment[locationsData.length]; + + final List presentLocations = new ArrayList<>(); + for (int i = 0; i < locationsData.length; i++) { + if (locationsData[i] != null) { + presentLocations.add(locationsData[i]); + } + } + + final SegmentsAndData segmentsAndData = SegmentedReadData.wrap(readData, presentLocations); + final Iterator presentSegments = segmentsAndData.segments().iterator(); + for (int i = 0; i < locationsData.length; i++) { + if (locationsData[i] != null) { + segmentsData[i] = presentSegments.next(); + } + } + + final Flattened index = new Flattened<>(locations.size, segmentsData); + final SegmentedReadData data = segmentsAndData.data(); + return new SegmentIndexAndData() { + @Override public Flattened index() {return index;} + @Override public SegmentedReadData data() {return data;} + }; + } +} From 55a0962a501634811cf0c27f5b70c643be90f323 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 6 Sep 2025 17:14:11 +0200 Subject: [PATCH 304/423] WIP --- .../saalfeldlab/n5/codec/BlockCodec.java | 22 ++++++ .../{ShardStuff3.java => RawShardStuff.java} | 70 +++++++++++++++++-- .../saalfeldlab/n5/shardstuff/ShardIndex.java | 4 ++ 3 files changed, 92 insertions(+), 4 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/shardstuff/{ShardStuff3.java => RawShardStuff.java} (58%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java index 8eb889661..4e564a40e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java @@ -43,4 +43,26 @@ public interface BlockCodec { ReadData encode(DataBlock dataBlock) throws N5IOException; DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException; + + /** + * Given the {@code blockSize} of a {@code DataBlock} return the size of + * the encoded block in bytes. + *

      + * A {@code UnsupportedOperationException} is thrown, if this {@code + * BlockCodec} cannot determine encoded size independent of block content. + * For example, if the block type contains var-length elements or if the + * serializer uses a non-deterministic {@code DataCodec}. + * + * @param blockSize + * size of the block to be encoded + * + * @return size of the encoded block in bytes + * + * @throws UnsupportedOperationException + * if this {@code DataBlockSerializer} cannot determine encoded size independent of block content + */ + default long encodedSize(int[] blockSize) throws UnsupportedOperationException { + // TODO: adapt https://github.com/saalfeldlab/n5/pull/165 to new naming + throw new UnsupportedOperationException("TODO: not implemented. See https://github.com/saalfeldlab/n5/pull/165."); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff3.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java similarity index 58% rename from src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff3.java rename to src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java index bf9c88a4d..85b30348e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff3.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java @@ -10,11 +10,11 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; -public class ShardStuff3 { +public class RawShardStuff { - public interface RawShard extends DataBlock { + public interface RawShard extends DataBlock { // pos is relative to this shard ReadData getElementData(long[] pos); @@ -25,7 +25,7 @@ public interface RawShard extends DataBlock { // TODO: removeElement(long[] pos) } - static class BasicRawShard { + static class BasicRawShard implements RawShard { BasicRawShard(final ReadData data, final int[] size) { @@ -38,8 +38,47 @@ static class BasicRawShard { } + // --- RawShard --- + @Override + public ReadData getElementData(final long[] pos) { + // TODO + throw new UnsupportedOperationException(); + } + + @Override + public void setElementData(final ReadData data, final long[] pos) { + // TODO + throw new UnsupportedOperationException(); + } + + + + // --- DataBlock --- + @Override + public int[] getSize() { + // TODO + throw new UnsupportedOperationException(); + } + + @Override + public long[] getGridPosition() { + // TODO + throw new UnsupportedOperationException(); + } + + @Override + public int getNumElements() { + // TODO + throw new UnsupportedOperationException(); + } + + @Override + public Void getData() { + // TODO + return null; + } } @@ -47,12 +86,35 @@ static class BasicRawShard { - public interface RawShardCodec extends BlockCodec { + public interface RawShardCodec extends BlockCodec { @Override RawShard decode(ReadData readData, long[] gridPosition) throws N5IOException; } + static class BasicRawShardCodec implements RawShardCodec { + + /** + * Number of elements (DataBlocks, nested shards) in each dimension per shard. + */ + private final int[] size; + + BasicRawShardCodec(final int[] size) { + this.size = size; + } + + @Override + public ReadData encode(final DataBlock dataBlock) throws N5IOException { + // TODO + throw new UnsupportedOperationException(); + } + + @Override + public RawShard decode(final ReadData readData, final long[] gridPosition) throws N5IOException { + // TODO + throw new UnsupportedOperationException(); + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java index 9441dfc1e..d6a130ad3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java @@ -15,6 +15,10 @@ public class ShardIndex { + public enum IndexLocation { + START, END + } + static class Flattened { final int[] size; From 30d9e4069bd6b97dab4b44d7f2b447a92861dc92 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 6 Sep 2025 21:43:03 +0200 Subject: [PATCH 305/423] clean up --- .../janelia/saalfeldlab/n5/readdata/segment/Segment.java | 2 -- .../saalfeldlab/n5/readdata/segment/SegmentStuff.java | 8 -------- 2 files changed, 10 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java index a72b7ca39..38d98ad73 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java @@ -1,7 +1,5 @@ package org.janelia.saalfeldlab.n5.readdata.segment; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - /** * A particular segment in a source {@code ReadData}. */ diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java deleted file mode 100644 index 362a19f94..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentStuff.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.janelia.saalfeldlab.n5.readdata.segment; - -public class SegmentStuff { - - public static void main(String[] args) { - - } -} From 816924cdc243de60eee6860710714452bf7ac619 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 6 Sep 2025 21:43:28 +0200 Subject: [PATCH 306/423] javadoc --- .../n5/readdata/segment/SegmentedReadData.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java index fb1f1168f..5745f3701 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java @@ -13,24 +13,29 @@ interface SegmentsAndData { } /** - * Wrap {@code readData} and create one segment comprising the entire {@code readData}. + * Wrap {@code readData} and create one segment comprising the entire {@code + * readData}. The segment can be retrieved as the first (and only) element + * of {@link SegmentedReadData#segments()}. */ static SegmentedReadData wrap(ReadData readData) { return DefaultSegmentedReadData.wrap(readData); } /** - * Wrap {@code readData} and create segments at the given locations. + * Wrap {@code readData} and create segments at the given locations. The + * order of segments in the returned {@link SegmentsAndData#segments()} list + * matches the order of the given {@code locations} (while the segments in the + * {@link SegmentsAndData#data()} are ordered by offset). */ - // TODO: does not assume ordered locations. to not lose track, this method - // should return Pair where segments are in - // the same order as locations. static SegmentsAndData wrap(ReadData readData, SegmentLocation... locations) { return wrap(readData, Arrays.asList(locations)); } /** - * Wrap {@code readData} and create segments at the given locations. + * Wrap {@code readData} and create segments at the given locations. The + * order of segments in the returned {@link SegmentsAndData#segments()} list + * matches the order of the given {@code locations} (while the segments in the + * {@link SegmentsAndData#data()} are ordered by offset). */ static SegmentsAndData wrap(ReadData readData, List locations) { return DefaultSegmentedReadData.wrap(readData, locations); From e0289b7c2f724f823ecf742f87ec2ff4c02a3954 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 6 Sep 2025 21:44:12 +0200 Subject: [PATCH 307/423] WIP RawShard deconding --- .../n5/shardstuff/RawShardStuff.java | 73 +++++++++++++------ .../saalfeldlab/n5/shardstuff/ShardIndex.java | 72 +++++++++++------- 2 files changed, 95 insertions(+), 50 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java index 85b30348e..473308003 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java @@ -8,7 +8,15 @@ import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.Segment; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; +import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.NDArray; +import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; +import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.SegmentIndexAndData; + +import static org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation.START; public class RawShardStuff { @@ -27,56 +35,65 @@ public interface RawShard extends DataBlock { static class BasicRawShard implements RawShard { - BasicRawShard(final ReadData data, final int[] size) { + private long[] gridPosition; - // TODO: - // [ ] read shard index (just assume some defaults) - // [ ] construct SegmentedReadData - // [ ] construct Map or flattened Segment[] - // [ ] implement setElementData - // [ ] implement getElementData + /** + * The ReadData from which the shard was constructed, or {@code null} + * for a new empty shard. + */ + private SegmentedReadData sourceData; + /** + * maps grid position of shard elements to {@link Segment}s. + */ + private NDArray index; + + BasicRawShard(final int[] size) { + sourceData = null; + index = new NDArray<>(size, Segment[]::new); } + BasicRawShard(final long[] gridPosition, final SegmentedReadData sourceData, final NDArray index) { + this.gridPosition = gridPosition; + this.sourceData = sourceData; + this.index = index; + } + + // --- RawShard --- @Override public ReadData getElementData(final long[] pos) { - // TODO - throw new UnsupportedOperationException(); + final Segment segment = index.get(pos); + return segment == null ? null : segment.source().slice(segment); } @Override public void setElementData(final ReadData data, final long[] pos) { - // TODO - throw new UnsupportedOperationException(); + final Segment segment = SegmentedReadData.wrap(data).segments().getFirst(); + index.set(segment, pos); } - // --- DataBlock --- @Override public int[] getSize() { - // TODO - throw new UnsupportedOperationException(); + return index.size(); } @Override public long[] getGridPosition() { - // TODO - throw new UnsupportedOperationException(); + return gridPosition; } @Override public int getNumElements() { - // TODO - throw new UnsupportedOperationException(); + return index.numElements(); } @Override public Void getData() { - // TODO return null; } } @@ -98,9 +115,16 @@ static class BasicRawShardCodec implements RawShardCodec { * Number of elements (DataBlocks, nested shards) in each dimension per shard. */ private final int[] size; + private final IndexLocation indexLocation; + private final BlockCodec indexCodec; + private final long indexBlockSizeInBytes; + + BasicRawShardCodec(final int[] size, final IndexLocation indexLocation, final BlockCodec indexCodec) { - BasicRawShardCodec(final int[] size) { this.size = size; + this.indexLocation = indexLocation; + this.indexCodec = indexCodec; + indexBlockSizeInBytes = indexCodec.encodedSize(ShardIndex.blockSizeFromIndexSize(size)); } @Override @@ -111,8 +135,13 @@ public ReadData encode(final DataBlock dataBlock) throws N5IOException { @Override public RawShard decode(final ReadData readData, final long[] gridPosition) throws N5IOException { - // TODO - throw new UnsupportedOperationException(); + + final long indexOffset = (indexLocation == START) ? 0 : (readData.requireLength() - indexBlockSizeInBytes); + final ReadData indexReadData = readData.slice(indexOffset, indexBlockSizeInBytes); + final DataBlock indexDataBlock = indexCodec.decode(indexReadData, new long[size.length]); + final NDArray locations = ShardIndex.fromDataBlock(indexDataBlock); + final SegmentIndexAndData segments = ShardIndex.segments(locations, readData); + return new BasicRawShard(gridPosition, segments.data(), segments.index()); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java index d6a130ad3..bc72dc933 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java @@ -19,21 +19,21 @@ public enum IndexLocation { START, END } - static class Flattened { + static class NDArray { final int[] size; private final int[] stride; final T[] data; - Flattened(final int[] size, final IntFunction createArray) { + NDArray(final int[] size, final IntFunction createArray) { this.size = size; - stride = stride(size); - data = createArray.apply(numElements(size)); + stride = getStrides(size); + data = createArray.apply(getNumElements(size)); } - Flattened(final int[] size, final T[] data) { + NDArray(final int[] size, final T[] data) { this.size = size; - stride = stride(size); + stride = getStrides(size); this.data = data; } @@ -56,9 +56,13 @@ private int index(long... position) { public int[] size() { return size; } + + public int numElements() { + return data.length; + } } - static int numElements(final int[] size) { + static int getNumElements(final int[] size) { int numElements = 1; for (int s : size) { numElements *= s; @@ -66,7 +70,7 @@ static int numElements(final int[] size) { return numElements; } - static int[] stride(final int[] size) { + static int[] getStrides(final int[] size) { final int n = size.length; final int[] stride = new int[n]; stride[0] = 1; @@ -82,43 +86,47 @@ static int[] stride(final int[] size) { */ static final long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; + /** + * Size of first dimension of the {@code DataBlock} representation of the shard index. + */ + private static final int LONGS_PER_BLOCK = 2; + // TODO do we need additional offset here? - static Flattened fromDataBlock( final DataBlock block ) { + static NDArray fromDataBlock( final DataBlock block ) { - assert block.getSize()[ 0 ] == 2; + assert block.getSize()[ 0 ] == LONGS_PER_BLOCK; - final int[] blockSize = block.getSize(); final long[] blockData = block.getData(); - final int[] size = Arrays.copyOfRange(blockSize, 1, blockSize.length); - final int n = numElements(size); + final int[] size = indexSizeFromBlockSize(block.getSize()); + final int n = getNumElements(size); final SegmentLocation[] locations = new SegmentLocation[n]; for (int i = 0; i < n; i++) { - long offset = blockData[ i * 2]; - long length = blockData[ i * 2 + 1]; + long offset = blockData[i * LONGS_PER_BLOCK]; + long length = blockData[i * LONGS_PER_BLOCK + 1]; if (offset != EMPTY_INDEX_NBYTES && length != EMPTY_INDEX_NBYTES) { locations[i] = SegmentLocation.at(offset, length); } } - return new Flattened<>(size, locations); + return new NDArray<>(size, locations); } // TODO do we use offset? If not, remove! - static DataBlock toDataBlock( final Flattened locations, final long offset ) { + static DataBlock toDataBlock( final NDArray locations, final long offset ) { final SegmentLocation[] data = locations.data; - final int[] blockSize = prepend(2, locations.size); + final int[] blockSize = blockSizeFromIndexSize(locations.size); final long[] blockData = new long[data.length * 2]; for (int i = 0; i < data.length; ++i) { if (data[i] != null) { - blockData[i * 2] = data[i].offset() + offset; - blockData[i * 2 + 1] = data[i].length(); + blockData[i * LONGS_PER_BLOCK] = data[i].offset() + offset; + blockData[i * LONGS_PER_BLOCK + 1] = data[i].length(); } else { - blockData[i * 2] = EMPTY_INDEX_NBYTES; - blockData[i * 2 + 1] = EMPTY_INDEX_NBYTES; + blockData[i * LONGS_PER_BLOCK] = EMPTY_INDEX_NBYTES; + blockData[i * LONGS_PER_BLOCK + 1] = EMPTY_INDEX_NBYTES; } } return new LongArrayDataBlock(blockSize, new long[blockSize.length], blockData); @@ -139,7 +147,15 @@ private static int[] prepend(final int value, final int[] array) { return indexBlockSize; } - static Flattened locations(final Flattened segments, final SegmentedReadData readData) { + static int[] blockSizeFromIndexSize(final int[] indexSize) { + return prepend(LONGS_PER_BLOCK, indexSize); + } + + static int[] indexSizeFromBlockSize(final int[] blockSize) { + return Arrays.copyOfRange(blockSize, 1, blockSize.length); + } + + static NDArray locations(final NDArray segments, final SegmentedReadData readData) { final Segment[] data = segments.data; final SegmentLocation[] locations = new SegmentLocation[data.length]; @@ -149,15 +165,15 @@ static Flattened locations(final Flattened segments, f locations[i] = readData.location(segment); } } - return new Flattened<>(segments.size, locations); + return new NDArray<>(segments.size, locations); } interface SegmentIndexAndData { - Flattened index(); + NDArray index(); SegmentedReadData data(); } - static SegmentIndexAndData segments(final Flattened locations, final ReadData readData) { + static SegmentIndexAndData segments(final NDArray locations, final ReadData readData) { final SegmentLocation[] locationsData = locations.data; final Segment[] segmentsData = new Segment[locationsData.length]; @@ -177,10 +193,10 @@ static SegmentIndexAndData segments(final Flattened locations, } } - final Flattened index = new Flattened<>(locations.size, segmentsData); + final NDArray index = new NDArray<>(locations.size, segmentsData); final SegmentedReadData data = segmentsAndData.data(); return new SegmentIndexAndData() { - @Override public Flattened index() {return index;} + @Override public NDArray index() {return index;} @Override public SegmentedReadData data() {return data;} }; } From 8f7e4c301f1268b83dffb6fbc9ecb0796eafd3a7 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 6 Sep 2025 21:54:00 +0200 Subject: [PATCH 308/423] javadoc --- .../saalfeldlab/n5/shardstuff/ShardIndex.java | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java index bc72dc933..10ca7502f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java @@ -15,10 +15,20 @@ public class ShardIndex { + private ShardIndex() { + // utility class. should not be instantiated. + } + public enum IndexLocation { START, END } + /** + * Access flat {@code T[]} array as n-dimensional array. + * + * @param + * element type + */ static class NDArray { final int[] size; @@ -94,10 +104,7 @@ static int[] getStrides(final int[] size) { // TODO do we need additional offset here? static NDArray fromDataBlock( final DataBlock block ) { - assert block.getSize()[ 0 ] == LONGS_PER_BLOCK; - final long[] blockData = block.getData(); - final int[] size = indexSizeFromBlockSize(block.getSize()); final int n = getNumElements(size); final SegmentLocation[] locations = new SegmentLocation[n]; @@ -147,14 +154,26 @@ private static int[] prepend(final int value, final int[] array) { return indexBlockSize; } + /** + * Prepends {@code LONGS_PER_BLOCK} to the {@code indexSize} array. + */ static int[] blockSizeFromIndexSize(final int[] indexSize) { return prepend(LONGS_PER_BLOCK, indexSize); } + /** + * Strips first element (should be {@code LONGS_PER_BLOCK} from the {@code blockSize} array. + */ static int[] indexSizeFromBlockSize(final int[] blockSize) { + assert blockSize[ 0 ] == LONGS_PER_BLOCK; return Arrays.copyOfRange(blockSize, 1, blockSize.length); } + /** + * Retrieves the {@code SegmentLocation} of each non-null {@code Segment} in + * {@code segments}. Returns a {@code NDArray} with entries + * corresponding tho the {@code segments} entries. + */ static NDArray locations(final NDArray segments, final SegmentedReadData readData) { final Segment[] data = segments.data; @@ -173,6 +192,12 @@ interface SegmentIndexAndData { SegmentedReadData data(); } + /** + * Puts a {@code Segment} at each non-null {@code SegmentLocation} in {@code + * locations} on the given {@code readData}. Returns both the {@code + * SegmentedReadData} with these segments and a {@code NDArray} + * with segment entries corresponding to the {@code locations} entries. + */ static SegmentIndexAndData segments(final NDArray locations, final ReadData readData) { final SegmentLocation[] locationsData = locations.data; From 9f56850e25ab0baece538f283a7f731521fd9c9f Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 6 Sep 2025 22:21:37 +0200 Subject: [PATCH 309/423] refactor --- .../n5/shardstuff/RawShardStuff.java | 94 ++++++++++++------- .../saalfeldlab/n5/shardstuff/ShardIndex.java | 2 +- 2 files changed, 59 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java index 473308003..f2513c5e9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java @@ -21,65 +21,70 @@ public class RawShardStuff { + public static class RawShardData { - public interface RawShard extends DataBlock { + private final SegmentedReadData sourceData; - // pos is relative to this shard - ReadData getElementData(long[] pos); + private final NDArray index; - // pos is relative to this shard - void setElementData(ReadData data, long[] pos); + RawShardData(final int[] size) { + sourceData = null; + index = new NDArray<>(size, Segment[]::new); + } - // TODO: removeElement(long[] pos) - } + RawShardData(final SegmentedReadData sourceData, final NDArray index) { + this.sourceData = sourceData; + this.index = index; + } - static class BasicRawShard implements RawShard { + RawShardData(final SegmentIndexAndData segmentIndexAndData) { + this(segmentIndexAndData.data(), segmentIndexAndData.index()); + } - private long[] gridPosition; /** * The ReadData from which the shard was constructed, or {@code null} * for a new empty shard. */ - private SegmentedReadData sourceData; + public SegmentedReadData sourceData() { + return sourceData; + } /** - * maps grid position of shard elements to {@link Segment}s. + * Maps grid position of shard elements to {@link Segment}s. */ - private NDArray index; - - BasicRawShard(final int[] size) { - sourceData = null; - index = new NDArray<>(size, Segment[]::new); + public NDArray index() { + return index; } + } - BasicRawShard(final long[] gridPosition, final SegmentedReadData sourceData, final NDArray index) { - this.gridPosition = gridPosition; - this.sourceData = sourceData; - this.index = index; - } + public static class RawShard implements DataBlock { - // --- RawShard --- + private final long[] gridPosition; + + private final RawShardData shardData; + + RawShard(final long[] gridPosition, final RawShardData shardData) { + this.gridPosition = gridPosition; + this.shardData = shardData; + } - @Override public ReadData getElementData(final long[] pos) { - final Segment segment = index.get(pos); + final Segment segment = shardData.index().get(pos); return segment == null ? null : segment.source().slice(segment); } - @Override public void setElementData(final ReadData data, final long[] pos) { final Segment segment = SegmentedReadData.wrap(data).segments().getFirst(); - index.set(segment, pos); + shardData.index().set(segment, pos); } - - // --- DataBlock --- + // --- DataBlock --- @Override public int[] getSize() { - return index.size(); + return shardData.index().size(); } @Override @@ -89,12 +94,12 @@ public long[] getGridPosition() { @Override public int getNumElements() { - return index.numElements(); + return shardData.index().numElements(); } @Override - public Void getData() { - return null; + public RawShardData getData() { + return shardData; } } @@ -103,7 +108,7 @@ public Void getData() { - public interface RawShardCodec extends BlockCodec { + public interface RawShardCodec extends BlockCodec { @Override RawShard decode(ReadData readData, long[] gridPosition) throws N5IOException; @@ -128,8 +133,25 @@ static class BasicRawShardCodec implements RawShardCodec { } @Override - public ReadData encode(final DataBlock dataBlock) throws N5IOException { - // TODO + public ReadData encode(final DataBlock shard) throws N5IOException { + + // TODO: + // [ ] concatenate slices for all non-null segments in shard.getData().index() + // [ ] if IndexLocation == END: + // [ ] write concatenated ReadData + // [ ] get SegmentLocations in concatenated ReadData + // [ ] transform to DataBlock + // [ ] write index + // [ ] package this whole sequence as OutputStreamWriter + // [ ] if IndexLocation == START: + // [ ] materialize concatenated ReadData + // [ ] get SegmentLocations in concatenated ReadData, adding indexBlockSizeInBytes as offset. + // [ ] transform to DataBlock + // [ ] write index + // [ ] write concatenated ReadData + // [ ] package this whole sequence as OutputStreamWriter (or just the final writing steps) + // [ ] return ReadData.from(OutputStreamWriter) + throw new UnsupportedOperationException(); } @@ -141,7 +163,7 @@ public RawShard decode(final ReadData readData, final long[] gridPosition) throw final DataBlock indexDataBlock = indexCodec.decode(indexReadData, new long[size.length]); final NDArray locations = ShardIndex.fromDataBlock(indexDataBlock); final SegmentIndexAndData segments = ShardIndex.segments(locations, readData); - return new BasicRawShard(gridPosition, segments.data(), segments.index()); + return new RawShard(gridPosition, new RawShardData(segments)); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java index 10ca7502f..697b53d67 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java @@ -29,7 +29,7 @@ public enum IndexLocation { * @param * element type */ - static class NDArray { + public static class NDArray { final int[] size; private final int[] stride; From 69f27ed2590b8c1a8f0441f1986a82fff79b455e Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 6 Sep 2025 22:52:25 +0200 Subject: [PATCH 310/423] WIP RawShard encoding --- .../n5/shardstuff/RawShardStuff.java | 44 +++++++++++++++---- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java index f2513c5e9..564ba1911 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.shardstuff; +import java.util.ArrayList; import java.util.List; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; @@ -8,6 +9,7 @@ import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.ReadData.OutputStreamWriter; import org.janelia.saalfeldlab.n5.readdata.segment.Segment; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; @@ -16,6 +18,7 @@ import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.SegmentIndexAndData; +import static org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation.END; import static org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation.START; public class RawShardStuff { @@ -134,22 +137,45 @@ static class BasicRawShardCodec implements RawShardCodec { @Override public ReadData encode(final DataBlock shard) throws N5IOException { - + + // concatenate slices for all non-null segments in shard.getData().index() + final NDArray index = shard.getData().index(); + final List readDatas = new ArrayList<>(); + for (Segment segment : index.data) { + if (segment != null ) { + readDatas.add(segment.source().slice(segment)); + } + } + final SegmentedReadData data = SegmentedReadData.concatenate(readDatas); + + if (indexLocation == START) { + data.materialize(); + final NDArray locations = ShardIndex.locations(index, data); + final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, indexBlockSizeInBytes); + final ReadData indexReadData = indexCodec.encode(indexDataBlock); + final OutputStreamWriter writer = out -> { + indexReadData.writeTo(out); + data.writeTo(out); + }; + return ReadData.from(writer); + } else { + + } // TODO: - // [ ] concatenate slices for all non-null segments in shard.getData().index() + // [+] concatenate slices for all non-null segments in shard.getData().index() + // [+] if IndexLocation == START: + // [+] materialize concatenated ReadData + // [+] get SegmentLocations in concatenated ReadData, adding indexBlockSizeInBytes as offset. + // [+] transform to DataBlock + // [+] write index + // [+] write concatenated ReadData + // [+] package this whole sequence as OutputStreamWriter (or just the final writing steps) // [ ] if IndexLocation == END: // [ ] write concatenated ReadData // [ ] get SegmentLocations in concatenated ReadData // [ ] transform to DataBlock // [ ] write index // [ ] package this whole sequence as OutputStreamWriter - // [ ] if IndexLocation == START: - // [ ] materialize concatenated ReadData - // [ ] get SegmentLocations in concatenated ReadData, adding indexBlockSizeInBytes as offset. - // [ ] transform to DataBlock - // [ ] write index - // [ ] write concatenated ReadData - // [ ] package this whole sequence as OutputStreamWriter (or just the final writing steps) // [ ] return ReadData.from(OutputStreamWriter) throw new UnsupportedOperationException(); From 926348ce13d86b8792582bf6ee2b496eaa80127a Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 6 Sep 2025 22:56:48 +0200 Subject: [PATCH 311/423] WIP RawShard encoding --- .../n5/shardstuff/RawShardStuff.java | 48 +++++-------------- 1 file changed, 13 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java index 564ba1911..50d580790 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java @@ -107,17 +107,7 @@ public RawShardData getData() { } - - - - - public interface RawShardCodec extends BlockCodec { - - @Override - RawShard decode(ReadData readData, long[] gridPosition) throws N5IOException; - } - - static class BasicRawShardCodec implements RawShardCodec { + public static class RawShardCodec implements BlockCodec { /** * Number of elements (DataBlocks, nested shards) in each dimension per shard. @@ -127,7 +117,7 @@ static class BasicRawShardCodec implements RawShardCodec { private final BlockCodec indexCodec; private final long indexBlockSizeInBytes; - BasicRawShardCodec(final int[] size, final IndexLocation indexLocation, final BlockCodec indexCodec) { + RawShardCodec(final int[] size, final IndexLocation indexLocation, final BlockCodec indexCodec) { this.size = size; this.indexLocation = indexLocation; @@ -147,38 +137,26 @@ public ReadData encode(final DataBlock shard) throws N5IOException } } final SegmentedReadData data = SegmentedReadData.concatenate(readDatas); - + final OutputStreamWriter writer; if (indexLocation == START) { data.materialize(); final NDArray locations = ShardIndex.locations(index, data); final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, indexBlockSizeInBytes); final ReadData indexReadData = indexCodec.encode(indexDataBlock); - final OutputStreamWriter writer = out -> { + writer = out -> { indexReadData.writeTo(out); data.writeTo(out); }; - return ReadData.from(writer); - } else { - + } else { // indexLocation == END + writer = out -> { + data.writeTo(out); + final NDArray locations = ShardIndex.locations(index, data); + final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, 0); + final ReadData indexReadData = indexCodec.encode(indexDataBlock); + indexReadData.writeTo(out); + }; } - // TODO: - // [+] concatenate slices for all non-null segments in shard.getData().index() - // [+] if IndexLocation == START: - // [+] materialize concatenated ReadData - // [+] get SegmentLocations in concatenated ReadData, adding indexBlockSizeInBytes as offset. - // [+] transform to DataBlock - // [+] write index - // [+] write concatenated ReadData - // [+] package this whole sequence as OutputStreamWriter (or just the final writing steps) - // [ ] if IndexLocation == END: - // [ ] write concatenated ReadData - // [ ] get SegmentLocations in concatenated ReadData - // [ ] transform to DataBlock - // [ ] write index - // [ ] package this whole sequence as OutputStreamWriter - // [ ] return ReadData.from(OutputStreamWriter) - - throw new UnsupportedOperationException(); + return ReadData.from(writer); } @Override From 2b1a38d38a9b1306052dea1f7eaba820fd519cad Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 6 Sep 2025 22:57:46 +0200 Subject: [PATCH 312/423] clean up --- .../org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java index 50d580790..01ca5d1c0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java @@ -14,11 +14,10 @@ import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; -import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.NDArray; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; +import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.NDArray; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.SegmentIndexAndData; -import static org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation.END; import static org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation.START; public class RawShardStuff { From 9b501dd383704a140d0d48536a5f0fa00bd086cd Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 7 Sep 2025 20:33:12 +0200 Subject: [PATCH 313/423] refactor --- .../n5/shardstuff/RawShardStuff.java | 69 ++++++++++--------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java index 01ca5d1c0..d0e9f0fe3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.List; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; @@ -23,27 +24,26 @@ public class RawShardStuff { - public static class RawShardData { + public static class RawShard { private final SegmentedReadData sourceData; private final NDArray index; - RawShardData(final int[] size) { + RawShard(final int[] size) { sourceData = null; index = new NDArray<>(size, Segment[]::new); } - RawShardData(final SegmentedReadData sourceData, final NDArray index) { + RawShard(final SegmentedReadData sourceData, final NDArray index) { this.sourceData = sourceData; this.index = index; } - RawShardData(final SegmentIndexAndData segmentIndexAndData) { + RawShard(final SegmentIndexAndData segmentIndexAndData) { this(segmentIndexAndData.data(), segmentIndexAndData.index()); } - /** * The ReadData from which the shard was constructed, or {@code null} * for a new empty shard. @@ -58,35 +58,33 @@ public SegmentedReadData sourceData() { public NDArray index() { return index; } - } - - - public static class RawShard implements DataBlock { - - private final long[] gridPosition; - - private final RawShardData shardData; - - RawShard(final long[] gridPosition, final RawShardData shardData) { - this.gridPosition = gridPosition; - this.shardData = shardData; - } public ReadData getElementData(final long[] pos) { - final Segment segment = shardData.index().get(pos); + final Segment segment = index.get(pos); return segment == null ? null : segment.source().slice(segment); } public void setElementData(final ReadData data, final long[] pos) { final Segment segment = SegmentedReadData.wrap(data).segments().getFirst(); - shardData.index().set(segment, pos); + index.set(segment, pos); } + } - // --- DataBlock --- + + public static class RawShardDataBlock implements DataBlock { + + private final long[] gridPosition; + + private final RawShard shard; + + RawShardDataBlock(final long[] gridPosition, final RawShard shard) { + this.gridPosition = gridPosition; + this.shard = shard; + } @Override public int[] getSize() { - return shardData.index().size(); + return shard.index().size(); } @Override @@ -96,17 +94,17 @@ public long[] getGridPosition() { @Override public int getNumElements() { - return shardData.index().numElements(); + return shard.index().numElements(); } @Override - public RawShardData getData() { - return shardData; + public RawShard getData() { + return shard; } } - public static class RawShardCodec implements BlockCodec { + public static class RawShardCodec implements BlockCodec { /** * Number of elements (DataBlocks, nested shards) in each dimension per shard. @@ -125,7 +123,7 @@ public static class RawShardCodec implements BlockCodec { } @Override - public ReadData encode(final DataBlock shard) throws N5IOException { + public ReadData encode(final DataBlock shard) throws N5IOException { // concatenate slices for all non-null segments in shard.getData().index() final NDArray index = shard.getData().index(); @@ -159,14 +157,14 @@ public ReadData encode(final DataBlock shard) throws N5IOException } @Override - public RawShard decode(final ReadData readData, final long[] gridPosition) throws N5IOException { + public DataBlock decode(final ReadData readData, final long[] gridPosition) throws N5IOException { final long indexOffset = (indexLocation == START) ? 0 : (readData.requireLength() - indexBlockSizeInBytes); final ReadData indexReadData = readData.slice(indexOffset, indexBlockSizeInBytes); final DataBlock indexDataBlock = indexCodec.decode(indexReadData, new long[size.length]); final NDArray locations = ShardIndex.fromDataBlock(indexDataBlock); final SegmentIndexAndData segments = ShardIndex.segments(locations, readData); - return new RawShard(gridPosition, new RawShardData(segments)); + return new RawShardDataBlock(gridPosition, new RawShard(segments)); } } @@ -182,7 +180,8 @@ public interface Sharding { ReadData writeBlocks(ReadData readData, List> blocks); } - public interface ShardCodecInfo extends CodecInfo { + // TODO: this could be a class instead of an interface + public interface ShardCodecInfo extends BlockCodecInfo { /** * Chunk size of the elements in this block. * That is, (1, ...) for DataBlockCodecInfo, respectively inner block size for ShardCodecInfo. @@ -190,7 +189,7 @@ public interface ShardCodecInfo extends CodecInfo { int[] getInnerBlockSize(); /** - * Nested BlockCodec. + * Nested Codec. * ({@code null} for DataBlockCodec. */ BlockCodecInfo getInnerBlockCodecInfo(); @@ -199,7 +198,13 @@ public interface ShardCodecInfo extends CodecInfo { // TODO: IndexCodec - RawShardCodec createRaw(int[] blockSize, DataCodecInfo... codecs); + @SuppressWarnings("unchecked") + @Override + default RawShardCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs) { + return create(blockSize, codecs); + } + + RawShardCodec create(int[] blockSize, DataCodecInfo... codecs); // TODO: not sure about this one. // This could recursively built the whole codec list? From d3cae93f63732c5dfdfc3e5af653348a8ab3365f Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 7 Sep 2025 22:02:35 +0200 Subject: [PATCH 314/423] WIP index CodecInfo and location --- .../n5/shardstuff/RawShardStuff.java | 40 ++++++++----------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java index d0e9f0fe3..2160f658a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java @@ -169,34 +169,34 @@ public DataBlock decode(final ReadData readData, final long[] gridPosi } - - - - // TODO: rename? - public interface Sharding { - - List> readBlocks(ReadData readData, List positions); - - ReadData writeBlocks(ReadData readData, List> blocks); - } - - // TODO: this could be a class instead of an interface public interface ShardCodecInfo extends BlockCodecInfo { + /** - * Chunk size of the elements in this block. - * That is, (1, ...) for DataBlockCodecInfo, respectively inner block size for ShardCodecInfo. + * Chunk size of each shard element (either nested shard or DataBlock) */ int[] getInnerBlockSize(); /** - * Nested Codec. - * ({@code null} for DataBlockCodec. + * BlockCodec for shard elements (either nested shard or DataBlock) */ BlockCodecInfo getInnerBlockCodecInfo(); + /** + * DataCodecs for inner BlockCodec + */ DataCodecInfo[] getInnerDataCodecInfos(); - // TODO: IndexCodec + /** + * BlockCodec for shard index + */ + BlockCodecInfo getIndexBlockCodecInfo(); + + /** + * Deterministic-size DataCodecs for index BlockCodec + */ + DataCodecInfo[] getIndexDataCodecInfos(); + + IndexLocation getIndexLocation(); @SuppressWarnings("unchecked") @Override @@ -205,11 +205,5 @@ default RawShardCodec create(DataType dataType, int[] blockSize, DataCodecInfo.. } RawShardCodec create(int[] blockSize, DataCodecInfo... codecs); - - // TODO: not sure about this one. - // This could recursively built the whole codec list? - // The result would have to be a BlockCodec. - // -// ??? create(DataType dataType, int[] blockSize, DataCodecInfo... codecs) {} } } From 524b2e75055d6d3c71a98031a9a7a623fbb9d5bf Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 7 Sep 2025 22:02:53 +0200 Subject: [PATCH 315/423] WIP dummy CodecInfos --- .../n5/shardstuff/RawShardStuff2.java | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java new file mode 100644 index 000000000..f0a5ecfe9 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java @@ -0,0 +1,115 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import java.util.List; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.shardstuff.RawShardStuff.RawShardCodec; +import org.janelia.saalfeldlab.n5.shardstuff.RawShardStuff.ShardCodecInfo; +import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; + +public class RawShardStuff2 { + + + + + + + + + // TODO: rename? + public interface Sharding { + + List> readBlocks(ReadData readData, List positions); + + ReadData writeBlocks(ReadData readData, List> blocks); + } + + + + + public static class DummyShardCodecInfo implements ShardCodecInfo { + + private final int[] innerBlockSize; + private final BlockCodecInfo innerBlockCodecInfo; + private final DataCodecInfo[] innerDataCodecInfos; + private final BlockCodecInfo indexBlockCodecInfo; + private final DataCodecInfo[] indexDataCodecInfos; + private final IndexLocation indexLocation; + + public DummyShardCodecInfo( + final int[] innerBlockSize, + final BlockCodecInfo innerBlockCodecInfo, + final DataCodecInfo[] innerDataCodecInfos, + final BlockCodecInfo indexBlockCodecInfo, + final DataCodecInfo[] indexDataCodecInfos, + final IndexLocation indexLocation) { + this.innerBlockSize = innerBlockSize; + this.innerBlockCodecInfo = innerBlockCodecInfo; + this.innerDataCodecInfos = innerDataCodecInfos; + this.indexBlockCodecInfo = indexBlockCodecInfo; + this.indexDataCodecInfos = indexDataCodecInfos; + this.indexLocation = indexLocation; + } + + @Override + public int[] getInnerBlockSize() { + return innerBlockSize; + } + + @Override + public BlockCodecInfo getInnerBlockCodecInfo() { + return innerBlockCodecInfo; + } + + @Override + public DataCodecInfo[] getInnerDataCodecInfos() { + return innerDataCodecInfos; + } + + @Override + public BlockCodecInfo getIndexBlockCodecInfo() { + return indexBlockCodecInfo; + } + + @Override + public DataCodecInfo[] getIndexDataCodecInfos() { + return indexDataCodecInfos; + } + + @Override + public IndexLocation getIndexLocation() { + return indexLocation; + } + + @Override + public RawShardCodec create(final int[] blockSize, final DataCodecInfo... codecs) { + return null; + } + + @Override + public String getType() { + return "DummyShardCodec"; + } + } + + + + public static class DummyDataBlockCodecInfo implements BlockCodecInfo { + + @Override + public BlockCodec create(final DataType dataType, final int[] blockSize, final DataCodecInfo... codecs) { + return null; + } + + @Override + public String getType() { + return "DummyDataBlockCodecInfo"; + } + } + + +} From a9d9500f230cc28bb935508eca1643a6b5fc6e50 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sun, 7 Sep 2025 22:12:10 +0200 Subject: [PATCH 316/423] DefaultShardCodecInfo --- .../n5/shardstuff/RawShardStuff2.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java index f0a5ecfe9..14d968298 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.shardstuff; +import java.util.Arrays; import java.util.List; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; @@ -31,7 +32,12 @@ public interface Sharding { - public static class DummyShardCodecInfo implements ShardCodecInfo { + public static class DefaultShardCodecInfo implements ShardCodecInfo { + + @Override + public String getType() { + return "ShardingCodec"; + } private final int[] innerBlockSize; private final BlockCodecInfo innerBlockCodecInfo; @@ -40,7 +46,7 @@ public static class DummyShardCodecInfo implements ShardCodecInfo { private final DataCodecInfo[] indexDataCodecInfos; private final IndexLocation indexLocation; - public DummyShardCodecInfo( + public DefaultShardCodecInfo( final int[] innerBlockSize, final BlockCodecInfo innerBlockCodecInfo, final DataCodecInfo[] innerDataCodecInfos, @@ -87,12 +93,19 @@ public IndexLocation getIndexLocation() { @Override public RawShardCodec create(final int[] blockSize, final DataCodecInfo... codecs) { - return null; - } - @Override - public String getType() { - return "DummyShardCodec"; + // Number of elements (DataBlocks, nested shards) in each dimension per shard. + final int[] size = new int[blockSize.length]; + // blockSize argument is number of pixels in the shard + // innerBlockSize is number of pixels in each shard element (nested shard or DataBlock) + Arrays.setAll(size, d -> blockSize[d] / innerBlockSize[d]); + + final BlockCodec indexCodec = indexBlockCodecInfo.create( + DataType.UINT64, + ShardIndex.blockSizeFromIndexSize(size), + indexDataCodecInfos); + + return new RawShardCodec(size, indexLocation, indexCodec); } } From 353d2ac314c077c5696780c8c7f10b046a6ea12f Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 8 Sep 2025 10:28:07 +0200 Subject: [PATCH 317/423] WIP --- .../n5/shardstuff/RawShardStuff2.java | 67 +++++++++++++++++-- 1 file changed, 61 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java index 14d968298..5802e3453 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java @@ -1,13 +1,14 @@ package org.janelia.saalfeldlab.n5.shardstuff; import java.util.Arrays; -import java.util.List; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; import org.janelia.saalfeldlab.n5.shardstuff.RawShardStuff.RawShardCodec; import org.janelia.saalfeldlab.n5.shardstuff.RawShardStuff.ShardCodecInfo; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; @@ -21,16 +22,70 @@ public class RawShardStuff2 { - // TODO: rename? - public interface Sharding { + public interface DatasetAccess { - List> readBlocks(ReadData readData, List positions); + DataBlock readBlock(KeyValueAccess kva, long[] gridPosition) throws N5IOException; - ReadData writeBlocks(ReadData readData, List> blocks); + void writeBlock(KeyValueAccess kva, DataBlock dataBlock) throws N5IOException; + +// List> readBlocks(List positions); +// void writeBlocks(List> blocks); } + public static class ShardedDatasetAccess implements DatasetAccess { + + private final NestedGrid grid; + public ShardedDatasetAccess(final NestedGrid grid, final BlockCodec[] blockCodecs) { + this.grid = grid; + } + @Override + public DataBlock readBlock(final KeyValueAccess kva, final long[] gridPosition) throws N5IOException { + throw new UnsupportedOperationException("TODO"); + } + + @Override + public void writeBlock(final KeyValueAccess kva, final DataBlock dataBlock) throws N5IOException { + throw new UnsupportedOperationException("TODO"); + } + } + + + static DatasetAccess create( + final DataType dataType, + int[] blockSize, + BlockCodecInfo blockCodecInfo, + DataCodecInfo[] dataCodecInfos + ) { + final int m = nestingDepth(blockCodecInfo); + + // There are m codecs: 1 DataBlock codecs, and m-1 shard codecs. + // The inner-most codec (the DataBlock codec) is at index 0. + final BlockCodec[] blockCodecs = new BlockCodec[m]; + final int[][] blockSizes = new int[m][]; + + for (int l = m - 1; l >= 0; --l) { + blockCodecs[l] = blockCodecInfo.create(dataType, blockSize, dataCodecInfos); + blockSizes[l] = blockSize; + if ( l > 0 ) { + final ShardCodecInfo info = (ShardCodecInfo) blockCodecInfo; + blockCodecInfo = info.getInnerBlockCodecInfo(); + dataCodecInfos = info.getInnerDataCodecInfos(); + blockSize = info.getInnerBlockSize(); + } + } + + return new ShardedDatasetAccess<>(new NestedGrid(blockSizes), blockCodecs); + } + + private static int nestingDepth(BlockCodecInfo info) { + if (info instanceof ShardCodecInfo) { + return 1 + nestingDepth(((ShardCodecInfo) info).getInnerBlockCodecInfo()); + } else { + return 1; + } + } public static class DefaultShardCodecInfo implements ShardCodecInfo { From b482a4b320bf1b6d417b92872830cb94b45d4c69 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 8 Sep 2025 13:24:16 +0200 Subject: [PATCH 318/423] WIP DatasetAccess --- .../saalfeldlab/n5/shardstuff/Nesting.java | 22 ++-- .../n5/shardstuff/RawShardStuff2.java | 110 +++++++++++++----- .../n5/shardstuff/ShardStuff2.java | 12 +- 3 files changed, 97 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java index 08d204b74..866682b43 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java @@ -33,7 +33,7 @@ public static void main(String[] args) { final NestedGrid grid = new NestedGrid(blockSizes); final NestedPosition pos = new NestedPosition(grid, new long[] {38, 7, 129}); System.out.println("pos = " + pos); - System.out.println("key = " + Arrays.toString(pos.keyPosition())); + System.out.println("key = " + Arrays.toString(pos.key())); } @@ -81,8 +81,7 @@ public int numDimensions() { * * @return relative grid position */ - // TODO: rename to relative()? reads nicer in position.relative(l) ... - public long[] relativePosition(final int level) { + public long[] relative(final int level) { return grid.relativePosition(position, level); } @@ -94,14 +93,12 @@ public long[] relativePosition(final int level) { * * @return absolute grid position */ - // TODO: rename to absolute()? reads nicer in position.absolute(l) ... - public long[] absolutePosition(final int level) { + public long[] absolute(final int level) { return grid.relativePosition(position, level); } - // TODO: rename to key()? to match absolute() and relative() ... - public long[] keyPosition() { - return relativePosition(grid.numLevels() - 1); + public long[] key() { + return relative(grid.numLevels() - 1); } @Override @@ -112,7 +109,7 @@ public String toString() { if ( l > level ) { sb.append(" / "); } - sb.append(Arrays.toString(relativePosition(l))); + sb.append(Arrays.toString(relative(l))); } sb.append(" (level ").append(level).append(")}"); return sb.toString(); @@ -140,9 +137,6 @@ public static class NestedGrid { // r[i][d] is block size at level i relative to level i-1 private final int[][] r; - // NB: as usual, level 0 is full resolution. - // TODO: BlockCodec stuff does this differently at the moment, but that should be changed ... - /** * {@code blockSizes[l][d]} is the block size at level {@code l} in dimension {@code d}. * Level 0 is the highest resolution (smallest block sizes). @@ -234,6 +228,10 @@ public long[] relativePosition( final int targetLevel) { return relativePosition(sourcePos, 0, targetLevel); } + + public int[] relativeBlockSize(final int level) { + return r[level]; + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java index 5802e3453..902a5c8cc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java @@ -3,13 +3,16 @@ import java.util.Arrays; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; +import org.janelia.saalfeldlab.n5.shardstuff.RawShardStuff.RawShard; import org.janelia.saalfeldlab.n5.shardstuff.RawShardStuff.RawShardCodec; +import org.janelia.saalfeldlab.n5.shardstuff.RawShardStuff.RawShardDataBlock; import org.janelia.saalfeldlab.n5.shardstuff.RawShardStuff.ShardCodecInfo; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; @@ -21,38 +24,104 @@ public class RawShardStuff2 { + public interface PositionValueAccess { + + /** + * @return ReadData for the given key or {@code null} if the key doesn't exist + */ + ReadData get(long[] key) throws N5IOException; + + void put(long[] key, ReadData data) throws N5IOException; + + void remove(long[] key) throws N5IOException; + } public interface DatasetAccess { - DataBlock readBlock(KeyValueAccess kva, long[] gridPosition) throws N5IOException; + DataBlock readBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; - void writeBlock(KeyValueAccess kva, DataBlock dataBlock) throws N5IOException; + void writeBlock(PositionValueAccess kva, DataBlock dataBlock) throws N5IOException; -// List> readBlocks(List positions); -// void writeBlocks(List> blocks); + // TODO: batch read/write methods +// List> readBlocks(PositionValueAccess kva, List positions); +// void writeBlocks(PositionValueAccess kva, List> blocks); } - public static class ShardedDatasetAccess implements DatasetAccess { + static class ShardedDatasetAccess implements DatasetAccess { private final NestedGrid grid; + private final BlockCodec[] codecs; - public ShardedDatasetAccess(final NestedGrid grid, final BlockCodec[] blockCodecs) { + public ShardedDatasetAccess(final NestedGrid grid, final BlockCodec[] codecs) { this.grid = grid; + this.codecs = codecs; } @Override - public DataBlock readBlock(final KeyValueAccess kva, final long[] gridPosition) throws N5IOException { - throw new UnsupportedOperationException("TODO"); + public DataBlock readBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { + final NestedPosition position = new NestedPosition(grid, gridPosition); + return readBlockRecursive(kva.get(position.key()), position, grid.numLevels() - 1); + } + + private DataBlock readBlockRecursive( + final ReadData readData, + final NestedPosition position, + final int level) + { + if (readData == null) { + return null; + } else if (level == 0) { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[0]; + return codec.decode(readData, position.absolute(0)); + } else { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[level]; + final RawShard shard = codec.decode(readData, position.absolute(level)).getData(); + return readBlockRecursive(shard.getElementData(position.relative(level - 1)), position, level - 1); + + } } @Override - public void writeBlock(final KeyValueAccess kva, final DataBlock dataBlock) throws N5IOException { - throw new UnsupportedOperationException("TODO"); + public void writeBlock(final PositionValueAccess kva, final DataBlock dataBlock) throws N5IOException { + final NestedPosition position = new NestedPosition(grid, dataBlock.getGridPosition()); + final long[] key = position.key(); + final ReadData existingData = kva.get(key); + final ReadData modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1); + kva.put(key, modifiedData); + } + + private ReadData writeBlockRecursive( + final ReadData existingReadData, + final DataBlock dataBlock, + final NestedPosition position, + final int level) + { + if ( level == 0 ) { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[0]; + return codec.encode(dataBlock); + } else { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[level]; + final long[] gridPos = position.absolute(level); + final RawShard shard = existingReadData == null ? + new RawShard(grid.relativeBlockSize(level)) : + codec.decode(existingReadData, gridPos).getData(); + final long[] elementPos = position.relative(level - 1); + final ReadData existingElementData = (level == 1) + ? null // if level == 1, we don't need to extract the nested (DataBlock) ReadData because it will be overridden anyway + : shard.getElementData(elementPos); + final ReadData modifiedElementData = writeBlockRecursive(existingElementData, dataBlock, position, level - 1); + shard.setElementData(modifiedElementData, elementPos); + return codec.encode(new RawShardDataBlock(gridPos, shard)); + } } } - static DatasetAccess create( + public static DatasetAccess create( final DataType dataType, int[] blockSize, BlockCodecInfo blockCodecInfo, @@ -163,21 +232,4 @@ public RawShardCodec create(final int[] blockSize, final DataCodecInfo... codecs return new RawShardCodec(size, indexLocation, indexCodec); } } - - - - public static class DummyDataBlockCodecInfo implements BlockCodecInfo { - - @Override - public BlockCodec create(final DataType dataType, final int[] blockSize, final DataCodecInfo... codecs) { - return null; - } - - @Override - public String getType() { - return "DummyDataBlockCodecInfo"; - } - } - - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java index 80c321667..f8b193e38 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java @@ -271,10 +271,10 @@ public DataBlock decode(ReadData readData, final NestedPosition position) thr // be verified, at least during development. for ( int l = grid.numLevels() - 1; l > 0; --l ) { - final Shard shard = shardCodecs[l].decode(readData, position.absolutePosition(l)); - readData = shard.getElementData(position.relativePosition(l - 1)); + final Shard shard = shardCodecs[l].decode(readData, position.absolute(l)); + readData = shard.getElementData(position.relative(l - 1)); } - return dataBlockCodec.decode(readData, position.absolutePosition(0)); + return dataBlockCodec.decode(readData, position.absolute(0)); } // TODO: alternative implementation that uses nestedCodec @@ -286,11 +286,11 @@ public DataBlock decodeNested(final ReadData readData, final NestedPosition p if ( nestedCodec != null ) { final int l = shardCodecs.length - 1; - final Shard shard = shardCodecs[l].decode(readData, position.absolutePosition(l)); - final ReadData nestedData = shard.getElementData(position.relativePosition(l - 1)); + final Shard shard = shardCodecs[l].decode(readData, position.absolute(l)); + final ReadData nestedData = shard.getElementData(position.relative(l - 1)); return nestedCodec.decodeNested(nestedData, position); } else { - return dataBlockCodec.decode(readData, position.absolutePosition(0)); + return dataBlockCodec.decode(readData, position.absolute(0)); } } From 8fc92447f9121bf35ce300b6edc663ad85f46054 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 8 Sep 2025 16:40:37 +0200 Subject: [PATCH 319/423] WIP --- .../saalfeldlab/n5/codec/BlockCodec.java | 10 ++- .../n5/shardstuff/RawShardStuff.java | 2 +- .../n5/shardstuff/RawShardStuff2.java | 65 +++++++++++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java index 4e564a40e..da54b5d11 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java @@ -30,6 +30,7 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** @@ -62,7 +63,12 @@ public interface BlockCodec { * if this {@code DataBlockSerializer} cannot determine encoded size independent of block content */ default long encodedSize(int[] blockSize) throws UnsupportedOperationException { - // TODO: adapt https://github.com/saalfeldlab/n5/pull/165 to new naming - throw new UnsupportedOperationException("TODO: not implemented. See https://github.com/saalfeldlab/n5/pull/165."); + + // TODO: Adapt https://github.com/saalfeldlab/n5/pull/165 to new naming! + + // TODO: REPLACE! This is a dirty hack, assuming this is only called for + // ShardIndex (UINT64) and the ShardIndex uses RawBlockCodec and + // RawCompression. + return DataBlock.getNumElements(blockSize) * 8L; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java index 2160f658a..1be890d96 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java @@ -65,7 +65,7 @@ public ReadData getElementData(final long[] pos) { } public void setElementData(final ReadData data, final long[] pos) { - final Segment segment = SegmentedReadData.wrap(data).segments().getFirst(); + final Segment segment = SegmentedReadData.wrap(data).segments().get(0); index.set(segment, pos); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java index 902a5c8cc..cb452f258 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java @@ -4,9 +4,12 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; @@ -19,6 +22,68 @@ public class RawShardStuff2 { + public static void main(String[] args) { + // DataBlocks are 3x3x3 + // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) + // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) + + + final BlockCodecInfo c0 = new N5BlockCodecInfo(); + final ShardCodecInfo c1 = new DefaultShardCodecInfo( + new int[] {3, 3, 3}, + c0, + new DataCodecInfo[] {new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[] {new RawCompression()}, + IndexLocation.END + ); + final ShardCodecInfo c2 = new DefaultShardCodecInfo( + new int[] {6, 6, 6}, + c1, + new DataCodecInfo[] {new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[] {new RawCompression()}, + IndexLocation.START + ); + + final DatasetAccess datasetAccess = create(DataType.INT8, + new int[] {24, 24, 24}, + c2, + new DataCodecInfo[] {new RawCompression()}); + + System.out.println("datasetAccess = " + datasetAccess); + + + + // TODO: + // [ ] implement TestPositionValueAccess that wraps a Map + // where Position wraps long[] + // [ ] create {3, 3, 3} INT8 DataBlock + // [ ] write the DataBlock + } + + + + static class TestPositionValueAccess implements PositionValueAccess { + + Map + + + @Override + public ReadData get(final long[] key) throws N5IOException { + return null; + } + + @Override + public void put(final long[] key, final ReadData data) throws N5IOException { + + } + + @Override + public void remove(final long[] key) throws N5IOException { + + } + } From 0e00feaae6bebc2845de6bb27be09c28ffc549e1 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 8 Sep 2025 17:23:59 +0200 Subject: [PATCH 320/423] WIP testing --- .../n5/shardstuff/RawShardStuff2.java | 64 ++++++++++++++----- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java index cb452f258..dc9fd24a0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java @@ -1,6 +1,9 @@ package org.janelia.saalfeldlab.n5.shardstuff; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; @@ -46,49 +49,76 @@ public static void main(String[] args) { IndexLocation.START ); - final DatasetAccess datasetAccess = create(DataType.INT8, + final DatasetAccess datasetAccess = create(DataType.INT8, new int[] {24, 24, 24}, c2, new DataCodecInfo[] {new RawCompression()}); - System.out.println("datasetAccess = " + datasetAccess); + final PositionValueAccess store = new TestPositionValueAccess(); + final int[] dataBlockSize = c1.getInnerBlockSize(); + datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 0, 0}, 1)); + datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 0, 0}, 2)); + datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 1, 0}, 3)); + datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 1, 0}, 4)); + datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {3, 2, 1}, 5)); + datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {8, 4, 1}, 6)); - // TODO: - // [ ] implement TestPositionValueAccess that wraps a Map - // where Position wraps long[] - // [ ] create {3, 3, 3} INT8 DataBlock - // [ ] write the DataBlock } - + private static DataBlock createDataBlock(int[] size, long[] gridPosition, int fillValue) { + final byte[] bytes = new byte[DataBlock.getNumElements(size)]; + Arrays.fill(bytes, (byte) fillValue); + return new ByteArrayDataBlock(size, gridPosition, bytes); + } static class TestPositionValueAccess implements PositionValueAccess { - Map + private static class Key { + private final long[] data; - @Override - public ReadData get(final long[] key) throws N5IOException { - return null; + Key(long[] data) { + this.data = data; + } + + @Override + public final boolean equals(final Object o) { + if (!(o instanceof Key)) { + return false; + } + final Key key = (Key) o; + return Arrays.equals(data, key.data); + } + + @Override + public int hashCode() { + return Arrays.hashCode(data); + } } + private final Map map = new HashMap<>(); + @Override - public void put(final long[] key, final ReadData data) throws N5IOException { + public ReadData get(final long[] key) { + final byte[] bytes = map.get(new Key(key)); + return bytes == null ? null : ReadData.from(bytes); + } + @Override + public void put(final long[] key, final ReadData data) { + final byte[] bytes = data == null ? null : data.allBytes(); + map.put(new Key(key), bytes); } @Override public void remove(final long[] key) throws N5IOException { - + map.remove(new Key(key)); } } - - - public interface PositionValueAccess { /** From 09000189aadddce45b6176035a0cc72898fb6c93 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 8 Sep 2025 17:27:49 +0200 Subject: [PATCH 321/423] bugfix: ReadData should know its length() after writeTo(OutputStream) --- .../saalfeldlab/n5/readdata/LazyReadData.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java index 044194deb..8c76dbbdf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import org.apache.commons.io.output.CountingOutputStream; import org.apache.commons.io.output.ProxyOutputStream; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; @@ -62,19 +63,25 @@ class LazyReadData implements ReadData { private ByteArrayReadData bytes; + private long length = -1; + @Override public ReadData materialize() throws N5IOException { if (bytes == null) { - final ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); - writeTo(baos); - bytes = new ByteArrayReadData(baos.toByteArray()); + try { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); + writer.writeTo(baos); + bytes = new ByteArrayReadData(baos.toByteArray()); + } catch (IOException e) { + throw new N5IOException(e); + } } return this; } @Override public long length() { - return (bytes != null) ? bytes.length() : -1; + return (bytes != null) ? bytes.length() : length; } // TODO: remove? Should this be the default implementation? @@ -113,7 +120,9 @@ public void writeTo(final OutputStream outputStream) throws N5IOException, Illeg if (bytes != null) { outputStream.write(bytes.allBytes()); } else { - writer.writeTo(outputStream); + final CountingOutputStream cos = new CountingOutputStream(outputStream); + writer.writeTo(cos); + length = cos.getByteCount(); } } catch (IOException e) { throw new N5IOException(e); From e233eab29c7cfd2ca86ade6336b283ddd60354c7 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 8 Sep 2025 18:10:47 +0200 Subject: [PATCH 322/423] WIP testing --- .../n5/shardstuff/RawShardStuff2.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java index dc9fd24a0..c6a282790 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java @@ -65,6 +65,26 @@ public static void main(String[] args) { datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {3, 2, 1}, 5)); datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {8, 4, 1}, 6)); + checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), true, 1); + checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); + checkBlock(datasetAccess.readBlock(store, new long[] {0, 1, 0}), true, 3); + checkBlock(datasetAccess.readBlock(store, new long[] {1, 1, 0}), true, 4); + checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5); + checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6); + } + + private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { + + if (dataBlock == null && expectedNonNull) { + throw new IllegalStateException("expected non-null dataBlock"); + } + + final byte[] bytes = dataBlock.getData(); + for (byte b : bytes) { + if (b != (byte) expectedFillValue) { + throw new IllegalStateException("expected all values to be " + expectedFillValue); + } + } } private static DataBlock createDataBlock(int[] size, long[] gridPosition, int fillValue) { @@ -137,6 +157,8 @@ public interface DatasetAccess { void writeBlock(PositionValueAccess kva, DataBlock dataBlock) throws N5IOException; + void deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; + // TODO: batch read/write methods // List> readBlocks(PositionValueAccess kva, List positions); // void writeBlocks(PositionValueAccess kva, List> blocks); @@ -187,6 +209,22 @@ public void writeBlock(final PositionValueAccess kva, final DataBlock dataBlo kva.put(key, modifiedData); } + @Override + public void deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { + // TODO + // [ ] private ReadData deleteBlockRecursive(...) + // [ ] in principle similar to writeBlockRecursive --> we need to decode/modify/re-encode shards + // [ ] level == 0 should return null (no block to encode) + // [ ] when receiving null for non-sharded dataset (probably sharded too) deleteBlock() should remove the key + // [ ] if at any point existingReadData is already null, do nothing + // --> maybe signal that by returning the same existing ReadData? + // [ ] when removing the last remaining block in a Shard, remove the shard + // --> this is signalled by returning null from nested call + // if this is a Shard, we will setElementData(null) + // We then need to check whether the shard became empty by inspecting the shard index and counting non-null values + // [ ] for sharded datasets, we don't even need to recursively descend to level 0. just immediately setElementData(null) + } + private ReadData writeBlockRecursive( final ReadData existingReadData, final DataBlock dataBlock, From 35166d8980a9adccafd88f0772edfb51d9e30d22 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 8 Sep 2025 21:50:46 +0200 Subject: [PATCH 323/423] WIP deleteBlock --- .../n5/shardstuff/RawShardStuff.java | 2 +- .../n5/shardstuff/RawShardStuff2.java | 104 ++++++++++++++---- .../saalfeldlab/n5/shardstuff/ShardIndex.java | 9 ++ 3 files changed, 92 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java index 1be890d96..46a94d654 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java @@ -65,7 +65,7 @@ public ReadData getElementData(final long[] pos) { } public void setElementData(final ReadData data, final long[] pos) { - final Segment segment = SegmentedReadData.wrap(data).segments().get(0); + final Segment segment = data == null ? null : SegmentedReadData.wrap(data).segments().get(0); index.set(segment, pos); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java index c6a282790..9aeb28012 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java @@ -29,8 +29,6 @@ public static void main(String[] args) { // DataBlocks are 3x3x3 // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) - - final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( new int[] {3, 3, 3}, @@ -71,18 +69,39 @@ public static void main(String[] args) { checkBlock(datasetAccess.readBlock(store, new long[] {1, 1, 0}), true, 4); checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5); checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6); - } - private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { + datasetAccess.deleteBlock(store, new long[] {0, 0, 0}); + checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); + checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); - if (dataBlock == null && expectedNonNull) { - throw new IllegalStateException("expected non-null dataBlock"); + // if a shard becomes empty the corresponding key should be deleted + if ( store.get(new long[] {1, 0, 0}) == null ) { + throw new IllegalStateException("expected non-null readData"); } + datasetAccess.deleteBlock(store, new long[] {8, 4, 1}); + if ( store.get(new long[] {1, 0, 0}) != null ) { + throw new IllegalStateException("expected null readData"); + } + + // deleting a non-existent block should not fail + datasetAccess.deleteBlock(store, new long[] {0, 0, 8}); + } - final byte[] bytes = dataBlock.getData(); - for (byte b : bytes) { - if (b != (byte) expectedFillValue) { - throw new IllegalStateException("expected all values to be " + expectedFillValue); + private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { + + if (dataBlock == null) { + if (expectedNonNull) { + throw new IllegalStateException("expected non-null dataBlock"); + } + } else { + if (!expectedNonNull) { + throw new IllegalStateException("expected null dataBlock"); + } + final byte[] bytes = dataBlock.getData(); + for (byte b : bytes) { + if (b != (byte) expectedFillValue) { + throw new IllegalStateException("expected all values to be " + expectedFillValue); + } } } } @@ -211,18 +230,59 @@ public void writeBlock(final PositionValueAccess kva, final DataBlock dataBlo @Override public void deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { - // TODO - // [ ] private ReadData deleteBlockRecursive(...) - // [ ] in principle similar to writeBlockRecursive --> we need to decode/modify/re-encode shards - // [ ] level == 0 should return null (no block to encode) - // [ ] when receiving null for non-sharded dataset (probably sharded too) deleteBlock() should remove the key - // [ ] if at any point existingReadData is already null, do nothing - // --> maybe signal that by returning the same existing ReadData? - // [ ] when removing the last remaining block in a Shard, remove the shard - // --> this is signalled by returning null from nested call - // if this is a Shard, we will setElementData(null) - // We then need to check whether the shard became empty by inspecting the shard index and counting non-null values - // [ ] for sharded datasets, we don't even need to recursively descend to level 0. just immediately setElementData(null) + final NestedPosition position = new NestedPosition(grid, gridPosition); + final long[] key = position.key(); + if ( grid.numLevels() == 1 ) { + // for non-sharded dataset, don't bother getting the value, just remove the key. + kva.remove(key); + } else { + final ReadData existingData = kva.get(key); + final ReadData modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); + if ( modifiedData == null ) { + kva.remove(key); + } else if ( modifiedData != existingData ) { + kva.put(key, modifiedData); + } + } + } + + private ReadData deleteBlockRecursive( + final ReadData existingReadData, + final NestedPosition position, + final int level) + { + if ( level == 0 || existingReadData == null ) { + return null; + } else { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[level]; + final long[] gridPos = position.absolute(level); + final RawShard shard = codec.decode(existingReadData, gridPos).getData(); + final long[] elementPos = position.relative(level - 1); + final ReadData existingElementData = shard.getElementData(elementPos); + if ( existingElementData == null ) { + // The DataBlock (or the whole nested shard containing it) does not exist. + // This shard remains unchanged. + return existingReadData; + } else { + final ReadData modifiedElementData = deleteBlockRecursive(existingElementData, position, level - 1); + if ( modifiedElementData == existingElementData ) { + // The nested shard was not modified. + // This shard remains unchanged. + return existingReadData; + } + shard.setElementData(modifiedElementData, elementPos); + if ( modifiedElementData == null ) { + // The DataBlock or nested shard was removed. + // Check whether this shard becomes empty. + if ( shard.index().allElementsNull() ) { + // This shard is empty and should be removed. + return null; + } + } + return codec.encode(new RawShardDataBlock(gridPos, shard)); + } + } } private ReadData writeBlockRecursive( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java index 697b53d67..a4b0ac8d9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java @@ -70,6 +70,15 @@ public int[] size() { public int numElements() { return data.length; } + + public boolean allElementsNull() { + for (T t : data) { + if (t != null) { + return false; + } + } + return true; + } } static int getNumElements(final int[] size) { From 6531f00b3d878f19a3dd62faa9e706d20dccae4d Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 8 Sep 2025 21:59:17 +0200 Subject: [PATCH 324/423] Add some TODOs --- .../n5/shardstuff/RawShardStuff2.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java index 9aeb28012..af4012ef5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java @@ -47,15 +47,27 @@ public static void main(String[] args) { IndexLocation.START ); + // TODO: This should go into DatasetAttributes. + // DatasetAccess replaces DatasetAttributes.blockCodec + // DatasetAttributes.getDataAccess() replaces DatasetAttributes.getBlockCodec() + // All information required for create(...) is known to DatasetAttributes already. final DatasetAccess datasetAccess = create(DataType.INT8, new int[] {24, 24, 24}, c2, new DataCodecInfo[] {new RawCompression()}); + // TODO: N5Reader/Writer needs to provide a PositionValueAccess implementation on top of its KVA. + // The read/write/deleteBlock methods would getDataAccess() from the DatasetAttributes and call it with that PositionValueAccess. final PositionValueAccess store = new TestPositionValueAccess(); - final int[] dataBlockSize = c1.getInnerBlockSize(); + // --------------------------------------------------------------- + // Some "tests" + // TODO: Turn into unit tests + // --------------------------------------------------------------- + + // write some blocks, filled with constant values + final int[] dataBlockSize = c1.getInnerBlockSize(); datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 0, 0}, 1)); datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 0, 0}, 2)); datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 1, 0}, 3)); @@ -63,6 +75,7 @@ public static void main(String[] args) { datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {3, 2, 1}, 5)); datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {8, 4, 1}, 6)); + // verify that the written blocks can be read back with the correct values checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), true, 1); checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); checkBlock(datasetAccess.readBlock(store, new long[] {0, 1, 0}), true, 3); @@ -70,6 +83,7 @@ public static void main(String[] args) { checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5); checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6); + // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) datasetAccess.deleteBlock(store, new long[] {0, 0, 0}); checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); @@ -178,7 +192,7 @@ public interface DatasetAccess { void deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; - // TODO: batch read/write methods + // TODO: batch read/write/delete methods // List> readBlocks(PositionValueAccess kva, List positions); // void writeBlocks(PositionValueAccess kva, List> blocks); } From 9627749976b618f6fce798b80db42d63661ad3eb Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 8 Sep 2025 22:03:00 +0200 Subject: [PATCH 325/423] Clean up --- .../n5/shardstuff/RawShardStuff.java | 2 - .../saalfeldlab/n5/shardstuff/ShardStuff.java | 444 ------------------ .../n5/shardstuff/ShardStuff2.java | 363 -------------- 3 files changed, 809 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java index 46a94d654..556f80c05 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java @@ -7,14 +7,12 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.ReadData.OutputStreamWriter; import org.janelia.saalfeldlab.n5.readdata.segment.Segment; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.NDArray; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.SegmentIndexAndData; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff.java deleted file mode 100644 index 14a36432d..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff.java +++ /dev/null @@ -1,444 +0,0 @@ -package org.janelia.saalfeldlab.n5.shardstuff; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.N5URI; -import org.janelia.saalfeldlab.n5.codec.CodecInfo; -import org.janelia.saalfeldlab.n5.codec.DataCodec; -import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - -public class ShardStuff { - - public interface Block { - - /** - * Returns the number of elements in a box of given size. - * - * @param size - * the size - * - * @return the number of elements - */ - static int getNumElements(final int[] size) { - - int n = size[0]; - for (int i = 1; i < size.length; ++i) - n *= size[i]; - return n; - } - - /** - * Returns the size of this block. - *

      - * The size of a data block is expected to be smaller than or equal to the - * spacing of the block grid. The dimensionality of size is expected to be - * equal to the dimensionality of the dataset. Consistency is not enforced. - * - * @return size of the block - */ - int[] getSize(); - - /** - * Returns the grid position of this block. - *

      - * Grid position is with respect to the level of the block (Shard, nested Shard, DataBlock, ...). - * If a block has grid position (i, ...) the adjacent block (in X) has position (i+1, ...). - * Grid position (0, ...) is the origin of the dataset, that is, grid positions are not relative to the containing shard etc. - *

      - * The dimensionality of the grid position is expected to be equal to the - * dimensionality of the dataset. Consistency is not enforced. - * - * @return position on the block grid - */ - long[] getGridPosition(); - } - - - public interface Shard extends Block { - - // pos is relative to this shard - ReadData getElementData(long[] pos); - - // pos is relative to this shard - void setElementData(ReadData data, long[] pos); - - // TODO: removeElement(long[] pos) - } - - - public interface DataBlock extends Block { - - /** - * Returns the data object held by this data block. - * - * @return data object - */ - T getData(); - -// interface DataBlockFactory { -// DataBlock createDataBlock(int[] blockSize, long[] gridPosition, T data); -// } - } - - - - - - - - public interface BlockCodec { - - ReadData encode(B block) throws N5IOException; - - /** - * {@code gridPosition} is with respect to the level of the block (Shard, nested Shard, DataBlock, ...). - * If a block has gridPosition (i, ...), then the adjacent block (in X) has position (i+1, ...). - * Grid position (0, ...) is the origin of the dataset, that is, grid positions are not relative to the containing shard etc. - */ - B decode(ReadData readData, long[] gridPosition) throws N5IOException; - } - - public interface ShardCodec extends BlockCodec { - } - - public interface DataBlockCodec extends BlockCodec> { - } - - - - - - - public interface BlockCodecInfo extends CodecInfo { - - BlockCodecs createRecursive(DataType dataType, int[] blockSize, DataCodecInfo... codecs); - } - - public interface ShardCodecInfo extends BlockCodecInfo { - - /** - * Chunk size of the elements in this block. - * That is, (1, ...) for DataBlockCodecInfo, respectively inner block size for ShardCodecInfo. - */ - int[] getChunkSize(); - - /** - * Nested BlockCodec. - * ({@code null} for DataBlockCodec. - */ - BlockCodecInfo getInnerBlockCodecInfo(); - - DataCodecInfo[] getInnerDataCodecInfos(); - - // TODO: IndexCodec - - ShardCodec create(int[] blockSize, DataCodecInfo... codecs); - - @Override - default BlockCodecs createRecursive(DataType dataType, int[] blockSize, DataCodecInfo... codecs) { - final ShardCodec shardCodec = create(blockSize, codecs); - final BlockCodecs blockCodecs = getInnerBlockCodecInfo().createRecursive( - dataType, getChunkSize(), - getInnerDataCodecInfos()); - return new BlockCodecs<>(shardCodec, blockSize, blockCodecs); - } - } - - public interface DataBlockCodecInfo extends BlockCodecInfo { - - DataBlockCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs); - - @Override - default BlockCodecs createRecursive(DataType dataType, int[] blockSize, DataCodecInfo... codecs) { - return new BlockCodecs<>(create(dataType, blockSize, codecs), blockSize); - } - } - - - - - - - public static class BlockCodecs { - - private final List shardCodecs = new ArrayList<>(); - private final DataBlockCodec dataBlockCodec; - - - // size of shard at a given level in units of DataBlocks - private final List chunkSizesInDataBlocks = new ArrayList<>(); - - // TODO: remove? - // number of nested elements in shard at a given level - private final List relativeChunkSizes = new ArrayList<>(); - - private final int[] dataBlockSize; - - public BlockCodecs(final DataBlockCodec dataBlockCodec, final int[] dataBlockSize) { - this.dataBlockCodec = dataBlockCodec; - this.dataBlockSize = dataBlockSize; - - final int[] ones = new int[dataBlockSize.length]; - Arrays.fill(ones, 1); - chunkSizesInDataBlocks.add(ones); - } - - /** - * @param shardSize in pixels - */ - public BlockCodecs(final ShardCodec codec, final int[] shardSize, final BlockCodecs others) { - this(others.dataBlockCodec, others.dataBlockSize); - shardCodecs.add(codec); - shardCodecs.addAll(others.shardCodecs); - - // TODO: verify that shardSize is integer multiple of dataBlockSize. - // TODO: verify that shardSize is integer multiple of nested shardSize - - final int[] gridSize = new int[shardSize.length]; - Arrays.setAll(gridSize, d -> shardSize[d] / dataBlockSize[d]); - chunkSizesInDataBlocks.add(gridSize); - chunkSizesInDataBlocks.addAll(others.chunkSizesInDataBlocks); - - final int[] relativeChunkSize = new int[shardSize.length]; - final int[] nestedGridSize = others.chunkSizesInDataBlocks.get(0); - Arrays.setAll(relativeChunkSize, d -> gridSize[d] / nestedGridSize[d]); - relativeChunkSizes.add(relativeChunkSize); - relativeChunkSizes.addAll(others.relativeChunkSizes); - } - - public int[] getDataBlockSize() { - return dataBlockSize; - } - - - - // TODO: [+] Decide whether to use absolute of relative NestedBlockPosition. - // For decoding we need both: - // shard.getElementData(RELATIVE_GRID_POSITION), - // (shard|dataBlock)Codec.decode(data, ABSOLUTE_GRID_POSITION) - // Maybe NestedBlockPosition should just contain both? - // ==> Trying that now... - - // TODO: [ ] split into separate NestedGrid class - - public NestedBlockPosition getNestedDataBlockPosition(final long[] gridPos) { - if (shardCodecs.isEmpty()) { - final long[][] nested = new long[][] {gridPos}; - return new NestedBlockPosition(nested, nested); - } - - final int n = gridPos.length; - final int m = shardCodecs.size() + 1; - - final long[][] nestedAbsolute = new long[m][]; - for (int i = 0; i < m - 1; ++i) { - nestedAbsolute[i] = new long[n]; - final int[] gridSize = chunkSizesInDataBlocks.get(i); - for (int d = 0; d < n; ++d) { - nestedAbsolute[i][d] = gridPos[d] / gridSize[d]; - } - } - nestedAbsolute[m - 1] = gridPos; // TODO: defensive clone() or not? - - final long[][] nestedRelative = new long[m][]; - nestedRelative[0] = nestedAbsolute[0]; - for (int i = 1; i < m; ++i) { - nestedRelative[i] = new long[n]; - final int[] relativeGridSize = relativeChunkSizes.get(i - 1); - for (int d = 0; d < n; ++d) { - nestedRelative[i][d] = nestedAbsolute[i][d] / relativeGridSize[d]; - } - } - - return new NestedBlockPosition(nestedAbsolute, nestedRelative); - } - - - // TODO: - // - // [+] Rewrite extractDataBlock() to use relative/absolute coordinates form nestedPos - // [ ] Extract something like "get the readData for nestedPos" which - // can then be re-used to implement extractDataBlock() and extractShard() non-recursively - // [ ] See whether that helps to think about writing / updating blocks - - - public DataBlock extractDataBlock(ReadData data, final NestedBlockPosition nestedPos) { - - // TODO: validation: nestedPos should refer to a DataBlock (not a Shard) - - // TODO: handle missing shards / blocks - - final int depth = nestedPos.depth(); - - for (int i = 0; i < depth; ++i) { - final long[] gridPosition = nestedPos.absolutePosition(i); - - if (i == depth - 1) { - return dataBlockCodec.decode(data, gridPosition); - } else { - final Shard shard = shardCodecs.get(i).decode(data, gridPosition); - data = shard.getElementData(nestedPos.relativePosition(i + 1)); - } - } - - throw new IllegalStateException(); // we should never end up here - } - - - -// public Shard getShard(ReadData data, final NestedBlockPosition nestedPos) { -// -// // TODO: validation: nestedPos should refer to a Shard (not a DataBlock) -// -// // TODO: handle missing shards -// -// final int depth = nestedPos.depth(); -// final int n = nestedPos.numDimensions(); -// -// final long[] gridOffset = new long[n]; -// for (int i = 0; i < depth; ++i) { -// final long[] relativeGridPos = nestedPos.relativePosition(i); -// -// final long[] gridPosition = new long[n]; -// Arrays.setAll(gridPosition, d -> gridOffset[d] + relativeGridPos[d]); -// -// final Shard shard = shardCodecs.get(i).decode(data, gridPosition); -// if ( i == depth - 1 ) { -// return shard; -// } -// data = shard.getElementData(relativeGridPos); -// -// final int[] relativeGridSize = relativeChunkSizes.get(i); -// Arrays.setAll(gridOffset, d -> gridPosition[d] * relativeGridSize[d]); -// } -// -// throw new IllegalStateException(); // we should never end up here -// } - - } - - - - - - - - // TODO: - // [ ] Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...). - // For nested = {X,Y,Z} compare by X, then Y, then Z. - // For X = {x,y,z} compare by z, then y, then x. (flattening order) - // [ ] Split into interface and implementation - // [ ] Add implementation based on NestedGrid and long[] gridPosition - - - - public static class NestedBlockPosition { - - private final long[][] nestedRelative; - private final long[][] nestedAbsolute; - - NestedBlockPosition(final long[][] nestedAbsolute, final long[][] nestedRelative) { - // TODO: validation: nestedRelative != null, at least one level, nestedAbsolute.length == nestedRelative.length - this.nestedAbsolute = nestedAbsolute; - this.nestedRelative = nestedRelative; - } - - public int depth() { - return nestedRelative.length; - } - - public int numDimensions() { - return nestedRelative[0].length; - } - - public long[] relativePosition(final int level) { - return nestedRelative[level]; - } - - public long[] absolutePosition(final int level) { - return nestedAbsolute[level]; - } - - public long[] keyPosition() { - return relativePosition(0); - } - - public NestedBlockPosition prefix(final int depth) { - // TODO: implement - throw new UnsupportedOperationException("TODO"); - } - } - - - // TODO: Implement NestedBlockPositions that recursively groups blocks under the same prefix. - // This would be the input to - - - - - - // DUMMY - static abstract class N5ReaderImpl implements GsonKeyValueN5Reader { - - - /** - * Reads a {@link DataBlock}. - * - * @param pathName - * dataset path - * @param datasetAttributes - * the dataset attributes - * @param gridPosition - * the grid position - * - * @return the data block - * - * @throws N5Exception - * the exception - */ - DataBlock readBlock( - final String pathName, - final DatasetAttributes datasetAttributes, - final BlockCodecs blockCodecs, // TODO: get from DatasetAttributes - final long... gridPosition) throws N5Exception { - - final NestedBlockPosition pos = blockCodecs.getNestedDataBlockPosition(gridPosition); - final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), pos.keyPosition()); - - try { - final ReadData keyData = getKeyValueAccess().createReadData(path); - return blockCodecs.extractDataBlock(keyData, pos); - } catch (N5Exception.N5NoSuchKeyException e) { - return null; - } - } - - - } - - - - - - - - - - - - - - - private static DataCodec[] instantiate(final DataCodecInfo... codecInfos) { - final DataCodec[] codecs = new DataCodec[codecInfos.length]; - Arrays.setAll(codecs, i -> codecInfos[i].create()); - return codecs; - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java deleted file mode 100644 index f8b193e38..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardStuff2.java +++ /dev/null @@ -1,363 +0,0 @@ -package org.janelia.saalfeldlab.n5.shardstuff; - -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.RawCompression; -import org.janelia.saalfeldlab.n5.codec.CodecInfo; -import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; - -public class ShardStuff2 { - - public interface Block { - - /** - * Returns the number of elements in a box of given size. - * - * @param size - * the size - * - * @return the number of elements - */ - static int getNumElements(final int[] size) { - - int n = size[0]; - for (int i = 1; i < size.length; ++i) - n *= size[i]; - return n; - } - - /** - * Returns the size of this block. - *

      - * The size of a data block is expected to be smaller than or equal to the - * spacing of the block grid. The dimensionality of size is expected to be - * equal to the dimensionality of the dataset. Consistency is not enforced. - * - * @return size of the block - */ - int[] getSize(); - - /** - * Returns the grid position of this block. - *

      - * Grid position is with respect to the level of the block (Shard, nested Shard, DataBlock, ...). - * If a block has grid position (i, ...) the adjacent block (in X) has position (i+1, ...). - * Grid position (0, ...) is the origin of the dataset, that is, grid positions are not relative to the containing shard etc. - *

      - * The dimensionality of the grid position is expected to be equal to the - * dimensionality of the dataset. Consistency is not enforced. - * - * @return position on the block grid - */ - long[] getGridPosition(); - } - - - public interface Shard extends Block { - - // pos is relative to this shard - ReadData getElementData(long[] pos); - - // pos is relative to this shard - void setElementData(ReadData data, long[] pos); - - // TODO: removeElement(long[] pos) - } - - - public interface DataBlock extends Block { - - /** - * Returns the data object held by this data block. - * - * @return data object - */ - T getData(); - -// interface DataBlockFactory { -// DataBlock createDataBlock(int[] blockSize, long[] gridPosition, T data); -// } - } - - - - - - - - public interface BlockCodec { - - ReadData encode(B block) throws N5IOException; - - /** - * {@code gridPosition} is with respect to the level of the block (Shard, nested Shard, DataBlock, ...). - * If a block has gridPosition (i, ...), then the adjacent block (in X) has position (i+1, ...). - * Grid position (0, ...) is the origin of the dataset, that is, grid positions are not relative to the containing shard etc. - */ - B decode(ReadData readData, long[] gridPosition) throws N5IOException; - } - - public interface ShardCodec extends BlockCodec { - } - - public interface DataBlockCodec extends BlockCodec> { - } - - - public interface BlockCodecInfo extends CodecInfo { - // when moving to Java 17+: - // this should be sealed, permitting ShardCodecInfo and DataBlockCodecInfo - } - - public interface ShardCodecInfo extends BlockCodecInfo { - - /** - * Chunk size of the elements in this block. - * That is, (1, ...) for DataBlockCodecInfo, respectively inner block size for ShardCodecInfo. - */ - int[] getInnerBlockSize(); - - /** - * Nested BlockCodec. - * ({@code null} for DataBlockCodec. - */ - BlockCodecInfo getInnerBlockCodecInfo(); - - DataCodecInfo[] getInnerDataCodecInfos(); - - // TODO: IndexCodec - - ShardCodec create(int[] blockSize, DataCodecInfo... codecs); - } - - public interface DataBlockCodecInfo extends BlockCodecInfo { - DataBlockCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs); - } - - - - - // ----------------- - // --- "testing" --- - - public static void main(String[] args) { - - // DataBlocks are 3x3x3 - // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) - // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) - - final DataBlockCodecInfo c0 = new DummyDataBlockCodecInfo(); - final ShardCodecInfo c1 = new DummyShardCodecInfo( - new int[] {3, 3, 3}, - c0, - new DataCodecInfo[] {new RawCompression()} - ); - final ShardCodecInfo c2 = new DummyShardCodecInfo( - new int[] {6, 6, 6}, - c1, - new DataCodecInfo[] {new RawCompression()} - ); - - final WipShardingCodec wipShardingCodec = create(DataType.INT8, - new int[] {24, 24, 24}, - c2, - new DataCodecInfo[] {new RawCompression()}); - - System.out.println("wipShardingCodec = " + wipShardingCodec); - } - - public static class DummyShardCodecInfo implements ShardCodecInfo { - - private final int[] innerBlockSize; - private final BlockCodecInfo innerBlockCodecInfo; - private final DataCodecInfo[] innerDataCodecInfos; - - public DummyShardCodecInfo( - final int[] innerBlockSize, - final BlockCodecInfo innerBlockCodecInfo, - final DataCodecInfo[] innerDataCodecInfos) { - this.innerBlockSize = innerBlockSize; - this.innerBlockCodecInfo = innerBlockCodecInfo; - this.innerDataCodecInfos = innerDataCodecInfos; - } - - @Override - public int[] getInnerBlockSize() { - return innerBlockSize; - } - - @Override - public BlockCodecInfo getInnerBlockCodecInfo() { - return innerBlockCodecInfo; - } - - @Override - public DataCodecInfo[] getInnerDataCodecInfos() { - return innerDataCodecInfos; - } - - @Override - public ShardCodec create(final int[] blockSize, final DataCodecInfo... codecs) { - return null; - } - - @Override - public String getType() { - return "DummyShardCodec"; - } - } - - public static class DummyDataBlockCodecInfo implements DataBlockCodecInfo { - - @Override - public DataBlockCodec create(final DataType dataType, final int[] blockSize, final DataCodecInfo... codecs) { - return null; - } - - @Override - public String getType() { - return "DummyDataBlockCodecInfo"; - } - } - - // --- "testing" --- - // ----------------- - - - - - - - - - - - - // NB: Something needs to get the ReadData from the KVA. - // We cannot put readBlocks(...) logic into the (Data)BlockCodec, because that doesn't have access to the KVA. - // We could put readBlocks(ReadData, ...) into the ShardCodec, to read several blocks from the same Shard. - // Then the N5Reader only has to do one level of batching. - - // TODO: Consider making this really recursive. - // ... but postpone until the overall shape is a bit clearer. - - static class WipShardingCodec { - - private final NestedGrid grid; - private final DataBlockCodec dataBlockCodec; - private final ShardCodec[] shardCodecs; - - private final WipShardingCodec nestedCodec; // TODO: For exploring logic options, we create a nested WipShardingCodec. - - public WipShardingCodec(final NestedGrid grid, final DataBlockCodec dataBlockCodec, final ShardCodec[] shardCodecs, final WipShardingCodec nestedCodec) { - this.grid = grid; - this.dataBlockCodec = dataBlockCodec; - this.shardCodecs = shardCodecs; - this.nestedCodec = nestedCodec; - } - - public NestedPosition toNestedPosition(final long[] gridPosition) { - return new NestedPosition(grid, gridPosition); - } - - // TODO: implementation without nestedCodec - public DataBlock decode(ReadData readData, final NestedPosition position) throws N5IOException { - - // NB: Assuming position.level==0 (refers to a DataBlock) and has - // nesting corresponding (at least) to our grid. This should perhaps - // be verified, at least during development. - - for ( int l = grid.numLevels() - 1; l > 0; --l ) { - final Shard shard = shardCodecs[l].decode(readData, position.absolute(l)); - readData = shard.getElementData(position.relative(l - 1)); - } - return dataBlockCodec.decode(readData, position.absolute(0)); - } - - // TODO: alternative implementation that uses nestedCodec - public DataBlock decodeNested(final ReadData readData, final NestedPosition position) throws N5IOException { - - // NB: Assuming position.level==0 (refers to a DataBlock) and has - // nesting corresponding (at least) to our grid. This should perhaps - // be verified, at least during development. - - if ( nestedCodec != null ) { - final int l = shardCodecs.length - 1; - final Shard shard = shardCodecs[l].decode(readData, position.absolute(l)); - final ReadData nestedData = shard.getElementData(position.relative(l - 1)); - return nestedCodec.decodeNested(nestedData, position); - } else { - return dataBlockCodec.decode(readData, position.absolute(0)); - } - } - - - - - // --- DataBlockCodec --- - -// @Override -// public ReadData encode(final DataBlock block) throws N5IOException { -// } -// -// @Override -// public DataBlock decode(final ReadData readData, final long[] gridPosition) throws N5IOException { -// } - } - - static WipShardingCodec create( - final DataType dataType, - int[] blockSize, - BlockCodecInfo blockCodecInfo, - DataCodecInfo[] dataCodecInfos - ) { - final int m = nestingDepth(blockCodecInfo); - - // TODO: For exploring logic options, we create a nested WipShardingCodec. - // If that turns out to be useful, the below logic should be simplified - // to re-use info from the nested WipShardingCodec. - final WipShardingCodec nestedWipShardingCodec; - if ( m > 1 ) { - final ShardCodecInfo info = (ShardCodecInfo) blockCodecInfo; - nestedWipShardingCodec = create(dataType, info.getInnerBlockSize(), info.getInnerBlockCodecInfo(), info.getInnerDataCodecInfos()); - } else { - nestedWipShardingCodec = null; - } - - final int[][] blockSizes = new int[m][]; - - // There are m codecs: 1 DataBlockCodec, m-1 ShardCodecs. - // Outermost codec comes last. - // To make the index math easier, shardCodecs[0] = null (the dataBlockCodec is at level 0). - final DataBlockCodec dataBlockCodec; - final ShardCodec[] shardCodecs = new ShardCodec[m]; - - for (int l = m - 1; l > 0; --l) { - final ShardCodecInfo info = (ShardCodecInfo) blockCodecInfo; // TODO instanceof check - blockSizes[l] = blockSize; - shardCodecs[l] = info.create(blockSize, dataCodecInfos); - blockCodecInfo = info.getInnerBlockCodecInfo(); - dataCodecInfos = info.getInnerDataCodecInfos(); - blockSize = info.getInnerBlockSize(); - } - - final DataBlockCodecInfo info = (DataBlockCodecInfo) blockCodecInfo; // TODO instanceof check - blockSizes[0] = blockSize; - dataBlockCodec = info.create(dataType, blockSize, dataCodecInfos); - - final NestedGrid grid = new NestedGrid(blockSizes); - - return new WipShardingCodec<>(grid, dataBlockCodec, shardCodecs, nestedWipShardingCodec); - } - - private static int nestingDepth(BlockCodecInfo info) { - if (info instanceof ShardCodecInfo) { - return 1 + nestingDepth(((ShardCodecInfo) info).getInnerBlockCodecInfo()); - } else { - return 1; - } - } -} From dad4adc4ba1d8fe715458b1983a55afcd32076d8 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 8 Sep 2025 22:17:11 +0200 Subject: [PATCH 326/423] refactor --- .../saalfeldlab/n5/shardstuff/RawShard.java | 52 +++++ .../n5/shardstuff/RawShardCodec.java | 79 +++++++ .../n5/shardstuff/RawShardDataBlock.java | 39 ++++ .../n5/shardstuff/RawShardStuff.java | 207 ------------------ .../n5/shardstuff/RawShardStuff2.java | 4 - .../n5/shardstuff/ShardCodecInfo.java | 44 ++++ 6 files changed, 214 insertions(+), 211 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShard.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardDataBlock.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardCodecInfo.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShard.java new file mode 100644 index 000000000..98c8ea6ae --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShard.java @@ -0,0 +1,52 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.Segment; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; +import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.NDArray; + +public class RawShard { + + private final SegmentedReadData sourceData; + + private final NDArray index; + + RawShard(final int[] size) { + sourceData = null; + index = new NDArray<>(size, Segment[]::new); + } + + RawShard(final SegmentedReadData sourceData, final NDArray index) { + this.sourceData = sourceData; + this.index = index; + } + + RawShard(final ShardIndex.SegmentIndexAndData segmentIndexAndData) { + this(segmentIndexAndData.data(), segmentIndexAndData.index()); + } + + /** + * The ReadData from which the shard was constructed, or {@code null} + * for a new empty shard. + */ + public SegmentedReadData sourceData() { + return sourceData; + } + + /** + * Maps grid position of shard elements to {@link Segment}s. + */ + public NDArray index() { + return index; + } + + public ReadData getElementData(final long[] pos) { + final Segment segment = index.get(pos); + return segment == null ? null : segment.source().slice(segment); + } + + public void setElementData(final ReadData data, final long[] pos) { + final Segment segment = data == null ? null : SegmentedReadData.wrap(data).segments().get(0); + index.set(segment, pos); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardCodec.java new file mode 100644 index 000000000..7ee37079b --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardCodec.java @@ -0,0 +1,79 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import java.util.ArrayList; +import java.util.List; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.Segment; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; +import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; +import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.NDArray; + +import static org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation.START; + +public class RawShardCodec implements BlockCodec { + + /** + * Number of elements (DataBlocks, nested shards) in each dimension per shard. + */ + private final int[] size; + private final IndexLocation indexLocation; + private final BlockCodec indexCodec; + private final long indexBlockSizeInBytes; + + RawShardCodec(final int[] size, final IndexLocation indexLocation, final BlockCodec indexCodec) { + + this.size = size; + this.indexLocation = indexLocation; + this.indexCodec = indexCodec; + indexBlockSizeInBytes = indexCodec.encodedSize(ShardIndex.blockSizeFromIndexSize(size)); + } + + @Override + public ReadData encode(final DataBlock shard) throws N5Exception.N5IOException { + + // concatenate slices for all non-null segments in shard.getData().index() + final NDArray index = shard.getData().index(); + final List readDatas = new ArrayList<>(); + for (Segment segment : index.data) { + if (segment != null) { + readDatas.add(segment.source().slice(segment)); + } + } + final SegmentedReadData data = SegmentedReadData.concatenate(readDatas); + final ReadData.OutputStreamWriter writer; + if (indexLocation == START) { + data.materialize(); + final NDArray locations = ShardIndex.locations(index, data); + final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, indexBlockSizeInBytes); + final ReadData indexReadData = indexCodec.encode(indexDataBlock); + writer = out -> { + indexReadData.writeTo(out); + data.writeTo(out); + }; + } else { // indexLocation == END + writer = out -> { + data.writeTo(out); + final NDArray locations = ShardIndex.locations(index, data); + final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, 0); + final ReadData indexReadData = indexCodec.encode(indexDataBlock); + indexReadData.writeTo(out); + }; + } + return ReadData.from(writer); + } + + @Override + public DataBlock decode(final ReadData readData, final long[] gridPosition) throws N5Exception.N5IOException { + + final long indexOffset = (indexLocation == START) ? 0 : (readData.requireLength() - indexBlockSizeInBytes); + final ReadData indexReadData = readData.slice(indexOffset, indexBlockSizeInBytes); + final DataBlock indexDataBlock = indexCodec.decode(indexReadData, new long[size.length]); + final NDArray locations = ShardIndex.fromDataBlock(indexDataBlock); + final ShardIndex.SegmentIndexAndData segments = ShardIndex.segments(locations, readData); + return new RawShardDataBlock(gridPosition, new RawShard(segments)); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardDataBlock.java new file mode 100644 index 000000000..16071ce9b --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardDataBlock.java @@ -0,0 +1,39 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import org.janelia.saalfeldlab.n5.DataBlock; + +/** + * Wrap a RawShard as a DataBlock. + * This basically just adds a gridPosition for the shard. + */ +public class RawShardDataBlock implements DataBlock { + + private final long[] gridPosition; + + private final RawShard shard; + + RawShardDataBlock(final long[] gridPosition, final RawShard shard) { + this.gridPosition = gridPosition; + this.shard = shard; + } + + @Override + public int[] getSize() { + return shard.index().size(); + } + + @Override + public long[] getGridPosition() { + return gridPosition; + } + + @Override + public int getNumElements() { + return shard.index().numElements(); + } + + @Override + public RawShard getData() { + return shard; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java deleted file mode 100644 index 556f80c05..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff.java +++ /dev/null @@ -1,207 +0,0 @@ -package org.janelia.saalfeldlab.n5.shardstuff; - -import java.util.ArrayList; -import java.util.List; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.codec.BlockCodec; -import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.readdata.ReadData.OutputStreamWriter; -import org.janelia.saalfeldlab.n5.readdata.segment.Segment; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; -import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; -import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.NDArray; -import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.SegmentIndexAndData; - -import static org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation.START; - -public class RawShardStuff { - - - public static class RawShard { - - private final SegmentedReadData sourceData; - - private final NDArray index; - - RawShard(final int[] size) { - sourceData = null; - index = new NDArray<>(size, Segment[]::new); - } - - RawShard(final SegmentedReadData sourceData, final NDArray index) { - this.sourceData = sourceData; - this.index = index; - } - - RawShard(final SegmentIndexAndData segmentIndexAndData) { - this(segmentIndexAndData.data(), segmentIndexAndData.index()); - } - - /** - * The ReadData from which the shard was constructed, or {@code null} - * for a new empty shard. - */ - public SegmentedReadData sourceData() { - return sourceData; - } - - /** - * Maps grid position of shard elements to {@link Segment}s. - */ - public NDArray index() { - return index; - } - - public ReadData getElementData(final long[] pos) { - final Segment segment = index.get(pos); - return segment == null ? null : segment.source().slice(segment); - } - - public void setElementData(final ReadData data, final long[] pos) { - final Segment segment = data == null ? null : SegmentedReadData.wrap(data).segments().get(0); - index.set(segment, pos); - } - } - - - public static class RawShardDataBlock implements DataBlock { - - private final long[] gridPosition; - - private final RawShard shard; - - RawShardDataBlock(final long[] gridPosition, final RawShard shard) { - this.gridPosition = gridPosition; - this.shard = shard; - } - - @Override - public int[] getSize() { - return shard.index().size(); - } - - @Override - public long[] getGridPosition() { - return gridPosition; - } - - @Override - public int getNumElements() { - return shard.index().numElements(); - } - - @Override - public RawShard getData() { - return shard; - } - } - - - public static class RawShardCodec implements BlockCodec { - - /** - * Number of elements (DataBlocks, nested shards) in each dimension per shard. - */ - private final int[] size; - private final IndexLocation indexLocation; - private final BlockCodec indexCodec; - private final long indexBlockSizeInBytes; - - RawShardCodec(final int[] size, final IndexLocation indexLocation, final BlockCodec indexCodec) { - - this.size = size; - this.indexLocation = indexLocation; - this.indexCodec = indexCodec; - indexBlockSizeInBytes = indexCodec.encodedSize(ShardIndex.blockSizeFromIndexSize(size)); - } - - @Override - public ReadData encode(final DataBlock shard) throws N5IOException { - - // concatenate slices for all non-null segments in shard.getData().index() - final NDArray index = shard.getData().index(); - final List readDatas = new ArrayList<>(); - for (Segment segment : index.data) { - if (segment != null ) { - readDatas.add(segment.source().slice(segment)); - } - } - final SegmentedReadData data = SegmentedReadData.concatenate(readDatas); - final OutputStreamWriter writer; - if (indexLocation == START) { - data.materialize(); - final NDArray locations = ShardIndex.locations(index, data); - final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, indexBlockSizeInBytes); - final ReadData indexReadData = indexCodec.encode(indexDataBlock); - writer = out -> { - indexReadData.writeTo(out); - data.writeTo(out); - }; - } else { // indexLocation == END - writer = out -> { - data.writeTo(out); - final NDArray locations = ShardIndex.locations(index, data); - final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, 0); - final ReadData indexReadData = indexCodec.encode(indexDataBlock); - indexReadData.writeTo(out); - }; - } - return ReadData.from(writer); - } - - @Override - public DataBlock decode(final ReadData readData, final long[] gridPosition) throws N5IOException { - - final long indexOffset = (indexLocation == START) ? 0 : (readData.requireLength() - indexBlockSizeInBytes); - final ReadData indexReadData = readData.slice(indexOffset, indexBlockSizeInBytes); - final DataBlock indexDataBlock = indexCodec.decode(indexReadData, new long[size.length]); - final NDArray locations = ShardIndex.fromDataBlock(indexDataBlock); - final SegmentIndexAndData segments = ShardIndex.segments(locations, readData); - return new RawShardDataBlock(gridPosition, new RawShard(segments)); - } - } - - - public interface ShardCodecInfo extends BlockCodecInfo { - - /** - * Chunk size of each shard element (either nested shard or DataBlock) - */ - int[] getInnerBlockSize(); - - /** - * BlockCodec for shard elements (either nested shard or DataBlock) - */ - BlockCodecInfo getInnerBlockCodecInfo(); - - /** - * DataCodecs for inner BlockCodec - */ - DataCodecInfo[] getInnerDataCodecInfos(); - - /** - * BlockCodec for shard index - */ - BlockCodecInfo getIndexBlockCodecInfo(); - - /** - * Deterministic-size DataCodecs for index BlockCodec - */ - DataCodecInfo[] getIndexDataCodecInfos(); - - IndexLocation getIndexLocation(); - - @SuppressWarnings("unchecked") - @Override - default RawShardCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs) { - return create(blockSize, codecs); - } - - RawShardCodec create(int[] blockSize, DataCodecInfo... codecs); - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java index af4012ef5..b24ef4600 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java @@ -16,10 +16,6 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; -import org.janelia.saalfeldlab.n5.shardstuff.RawShardStuff.RawShard; -import org.janelia.saalfeldlab.n5.shardstuff.RawShardStuff.RawShardCodec; -import org.janelia.saalfeldlab.n5.shardstuff.RawShardStuff.RawShardDataBlock; -import org.janelia.saalfeldlab.n5.shardstuff.RawShardStuff.ShardCodecInfo; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; public class RawShardStuff2 { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardCodecInfo.java new file mode 100644 index 000000000..b819967b4 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardCodecInfo.java @@ -0,0 +1,44 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; + +public interface ShardCodecInfo extends BlockCodecInfo { + + /** + * Chunk size of each shard element (either nested shard or DataBlock) + */ + int[] getInnerBlockSize(); + + /** + * BlockCodec for shard elements (either nested shard or DataBlock) + */ + BlockCodecInfo getInnerBlockCodecInfo(); + + /** + * DataCodecs for inner BlockCodec + */ + DataCodecInfo[] getInnerDataCodecInfos(); + + /** + * BlockCodec for shard index + */ + BlockCodecInfo getIndexBlockCodecInfo(); + + /** + * Deterministic-size DataCodecs for index BlockCodec + */ + DataCodecInfo[] getIndexDataCodecInfos(); + + IndexLocation getIndexLocation(); + + @SuppressWarnings("unchecked") + @Override + default RawShardCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs) { + return create(blockSize, codecs); + } + + RawShardCodec create(int[] blockSize, DataCodecInfo... codecs); +} From 76c83c041a48823357b9fbed8304494c2fbf66c2 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 8 Sep 2025 22:37:28 +0200 Subject: [PATCH 327/423] refactor --- .../n5/shardstuff/DatasetAccess.java | 24 ++ .../n5/shardstuff/DefaultShardCodecInfo.java | 89 ++++++ .../n5/shardstuff/PositionValueAccess.java | 19 ++ .../n5/shardstuff/RawShardCodec.java | 4 + .../n5/shardstuff/RawShardStuff2.java | 281 +----------------- .../n5/shardstuff/ShardedDatasetAccess.java | 173 +++++++++++ 6 files changed, 317 insertions(+), 273 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultShardCodecInfo.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java new file mode 100644 index 000000000..768f9c5b2 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java @@ -0,0 +1,24 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; + +/** + * Wrap an instantiated DataBlock/shard codec hierarchy to implement (single and + * batch) DataBlock read/write methods. + * + * @param + * type of the data contained in the DataBlock + */ +public interface DatasetAccess { + + DataBlock readBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; + + void writeBlock(PositionValueAccess kva, DataBlock dataBlock) throws N5IOException; + + void deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; + + // TODO: batch read/write/delete methods +// List> readBlocks(PositionValueAccess kva, List positions); +// void writeBlocks(PositionValueAccess kva, List> blocks); +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultShardCodecInfo.java new file mode 100644 index 000000000..37e80f49b --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultShardCodecInfo.java @@ -0,0 +1,89 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import java.util.Arrays; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; + +/** + * Default (and probably only) implementation of {@link ShardCodecInfo}. + */ +// TODO rename? +public class DefaultShardCodecInfo implements ShardCodecInfo { + + @Override + public String getType() { + return "ShardingCodec"; + } + + private final int[] innerBlockSize; + private final BlockCodecInfo innerBlockCodecInfo; + private final DataCodecInfo[] innerDataCodecInfos; + private final BlockCodecInfo indexBlockCodecInfo; + private final DataCodecInfo[] indexDataCodecInfos; + private final IndexLocation indexLocation; + + public DefaultShardCodecInfo( + final int[] innerBlockSize, + final BlockCodecInfo innerBlockCodecInfo, + final DataCodecInfo[] innerDataCodecInfos, + final BlockCodecInfo indexBlockCodecInfo, + final DataCodecInfo[] indexDataCodecInfos, + final IndexLocation indexLocation) { + this.innerBlockSize = innerBlockSize; + this.innerBlockCodecInfo = innerBlockCodecInfo; + this.innerDataCodecInfos = innerDataCodecInfos; + this.indexBlockCodecInfo = indexBlockCodecInfo; + this.indexDataCodecInfos = indexDataCodecInfos; + this.indexLocation = indexLocation; + } + + @Override + public int[] getInnerBlockSize() { + return innerBlockSize; + } + + @Override + public BlockCodecInfo getInnerBlockCodecInfo() { + return innerBlockCodecInfo; + } + + @Override + public DataCodecInfo[] getInnerDataCodecInfos() { + return innerDataCodecInfos; + } + + @Override + public BlockCodecInfo getIndexBlockCodecInfo() { + return indexBlockCodecInfo; + } + + @Override + public DataCodecInfo[] getIndexDataCodecInfos() { + return indexDataCodecInfos; + } + + @Override + public IndexLocation getIndexLocation() { + return indexLocation; + } + + @Override + public RawShardCodec create(final int[] blockSize, final DataCodecInfo... codecs) { + + // Number of elements (DataBlocks, nested shards) in each dimension per shard. + final int[] size = new int[blockSize.length]; + // blockSize argument is number of pixels in the shard + // innerBlockSize is number of pixels in each shard element (nested shard or DataBlock) + Arrays.setAll(size, d -> blockSize[d] / innerBlockSize[d]); + + final BlockCodec indexCodec = indexBlockCodecInfo.create( + DataType.UINT64, + ShardIndex.blockSizeFromIndexSize(size), + indexDataCodecInfos); + + return new RawShardCodec(size, indexLocation, indexCodec); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java new file mode 100644 index 000000000..88be18c92 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java @@ -0,0 +1,19 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +/** + * Idea is to wrap a KeyValueAccess and a dataset URI to be able to get/put values (ReadData) by {@code long[]} key + */ +public interface PositionValueAccess { + + /** + * @return ReadData for the given key or {@code null} if the key doesn't exist + */ + ReadData get(long[] key) throws N5Exception.N5IOException; + + void put(long[] key, ReadData data) throws N5Exception.N5IOException; + + void remove(long[] key) throws N5Exception.N5IOException; +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardCodec.java index 7ee37079b..dfc9b1689 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardCodec.java @@ -38,12 +38,16 @@ public ReadData encode(final DataBlock shard) throws N5Exception.N5IOE // concatenate slices for all non-null segments in shard.getData().index() final NDArray index = shard.getData().index(); final List readDatas = new ArrayList<>(); + // TODO: Any clever ReadData grouping, slice merging, etc. should go here + // This basic implementation just slices ReadData for all non-null + // elements and concatenates in flat index order. for (Segment segment : index.data) { if (segment != null) { readDatas.add(segment.source().slice(segment)); } } final SegmentedReadData data = SegmentedReadData.concatenate(readDatas); + final ReadData.OutputStreamWriter writer; if (indexLocation == START) { data.materialize(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java index b24ef4600..4511f0cab 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java @@ -8,14 +8,11 @@ import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.RawCompression; -import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; public class RawShardStuff2 { @@ -46,8 +43,8 @@ public static void main(String[] args) { // TODO: This should go into DatasetAttributes. // DatasetAccess replaces DatasetAttributes.blockCodec // DatasetAttributes.getDataAccess() replaces DatasetAttributes.getBlockCodec() - // All information required for create(...) is known to DatasetAttributes already. - final DatasetAccess datasetAccess = create(DataType.INT8, + // All information required for ShardedDatasetAccess.create(...) is known to DatasetAttributes already. + final DatasetAccess datasetAccess = ShardedDatasetAccess.create(DataType.INT8, new int[] {24, 24, 24}, c2, new DataCodecInfo[] {new RawCompression()}); @@ -122,6 +119,12 @@ private static DataBlock createDataBlock(int[] size, long[] gridPosition return new ByteArrayDataBlock(size, gridPosition, bytes); } + + /** + * Dummy implementation of PositionValueAccess for testing. Just stores + * {@code byte[]} array values in a {@code Map} (instead of a {@code + * KeyValueAccess}. + */ static class TestPositionValueAccess implements PositionValueAccess { private static class Key { @@ -167,272 +170,4 @@ public void remove(final long[] key) throws N5IOException { } } - - public interface PositionValueAccess { - - /** - * @return ReadData for the given key or {@code null} if the key doesn't exist - */ - ReadData get(long[] key) throws N5IOException; - - void put(long[] key, ReadData data) throws N5IOException; - - void remove(long[] key) throws N5IOException; - } - - public interface DatasetAccess { - - DataBlock readBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; - - void writeBlock(PositionValueAccess kva, DataBlock dataBlock) throws N5IOException; - - void deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; - - // TODO: batch read/write/delete methods -// List> readBlocks(PositionValueAccess kva, List positions); -// void writeBlocks(PositionValueAccess kva, List> blocks); - } - - static class ShardedDatasetAccess implements DatasetAccess { - - private final NestedGrid grid; - private final BlockCodec[] codecs; - - public ShardedDatasetAccess(final NestedGrid grid, final BlockCodec[] codecs) { - this.grid = grid; - this.codecs = codecs; - } - - @Override - public DataBlock readBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { - final NestedPosition position = new NestedPosition(grid, gridPosition); - return readBlockRecursive(kva.get(position.key()), position, grid.numLevels() - 1); - } - - private DataBlock readBlockRecursive( - final ReadData readData, - final NestedPosition position, - final int level) - { - if (readData == null) { - return null; - } else if (level == 0) { - @SuppressWarnings("unchecked") - final BlockCodec codec = (BlockCodec) codecs[0]; - return codec.decode(readData, position.absolute(0)); - } else { - @SuppressWarnings("unchecked") - final BlockCodec codec = (BlockCodec) codecs[level]; - final RawShard shard = codec.decode(readData, position.absolute(level)).getData(); - return readBlockRecursive(shard.getElementData(position.relative(level - 1)), position, level - 1); - - } - } - - @Override - public void writeBlock(final PositionValueAccess kva, final DataBlock dataBlock) throws N5IOException { - final NestedPosition position = new NestedPosition(grid, dataBlock.getGridPosition()); - final long[] key = position.key(); - final ReadData existingData = kva.get(key); - final ReadData modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1); - kva.put(key, modifiedData); - } - - @Override - public void deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { - final NestedPosition position = new NestedPosition(grid, gridPosition); - final long[] key = position.key(); - if ( grid.numLevels() == 1 ) { - // for non-sharded dataset, don't bother getting the value, just remove the key. - kva.remove(key); - } else { - final ReadData existingData = kva.get(key); - final ReadData modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); - if ( modifiedData == null ) { - kva.remove(key); - } else if ( modifiedData != existingData ) { - kva.put(key, modifiedData); - } - } - } - - private ReadData deleteBlockRecursive( - final ReadData existingReadData, - final NestedPosition position, - final int level) - { - if ( level == 0 || existingReadData == null ) { - return null; - } else { - @SuppressWarnings("unchecked") - final BlockCodec codec = (BlockCodec) codecs[level]; - final long[] gridPos = position.absolute(level); - final RawShard shard = codec.decode(existingReadData, gridPos).getData(); - final long[] elementPos = position.relative(level - 1); - final ReadData existingElementData = shard.getElementData(elementPos); - if ( existingElementData == null ) { - // The DataBlock (or the whole nested shard containing it) does not exist. - // This shard remains unchanged. - return existingReadData; - } else { - final ReadData modifiedElementData = deleteBlockRecursive(existingElementData, position, level - 1); - if ( modifiedElementData == existingElementData ) { - // The nested shard was not modified. - // This shard remains unchanged. - return existingReadData; - } - shard.setElementData(modifiedElementData, elementPos); - if ( modifiedElementData == null ) { - // The DataBlock or nested shard was removed. - // Check whether this shard becomes empty. - if ( shard.index().allElementsNull() ) { - // This shard is empty and should be removed. - return null; - } - } - return codec.encode(new RawShardDataBlock(gridPos, shard)); - } - } - } - - private ReadData writeBlockRecursive( - final ReadData existingReadData, - final DataBlock dataBlock, - final NestedPosition position, - final int level) - { - if ( level == 0 ) { - @SuppressWarnings("unchecked") - final BlockCodec codec = (BlockCodec) codecs[0]; - return codec.encode(dataBlock); - } else { - @SuppressWarnings("unchecked") - final BlockCodec codec = (BlockCodec) codecs[level]; - final long[] gridPos = position.absolute(level); - final RawShard shard = existingReadData == null ? - new RawShard(grid.relativeBlockSize(level)) : - codec.decode(existingReadData, gridPos).getData(); - final long[] elementPos = position.relative(level - 1); - final ReadData existingElementData = (level == 1) - ? null // if level == 1, we don't need to extract the nested (DataBlock) ReadData because it will be overridden anyway - : shard.getElementData(elementPos); - final ReadData modifiedElementData = writeBlockRecursive(existingElementData, dataBlock, position, level - 1); - shard.setElementData(modifiedElementData, elementPos); - return codec.encode(new RawShardDataBlock(gridPos, shard)); - } - } - } - - - public static DatasetAccess create( - final DataType dataType, - int[] blockSize, - BlockCodecInfo blockCodecInfo, - DataCodecInfo[] dataCodecInfos - ) { - final int m = nestingDepth(blockCodecInfo); - - // There are m codecs: 1 DataBlock codecs, and m-1 shard codecs. - // The inner-most codec (the DataBlock codec) is at index 0. - final BlockCodec[] blockCodecs = new BlockCodec[m]; - final int[][] blockSizes = new int[m][]; - - for (int l = m - 1; l >= 0; --l) { - blockCodecs[l] = blockCodecInfo.create(dataType, blockSize, dataCodecInfos); - blockSizes[l] = blockSize; - if ( l > 0 ) { - final ShardCodecInfo info = (ShardCodecInfo) blockCodecInfo; - blockCodecInfo = info.getInnerBlockCodecInfo(); - dataCodecInfos = info.getInnerDataCodecInfos(); - blockSize = info.getInnerBlockSize(); - } - } - - return new ShardedDatasetAccess<>(new NestedGrid(blockSizes), blockCodecs); - } - - private static int nestingDepth(BlockCodecInfo info) { - if (info instanceof ShardCodecInfo) { - return 1 + nestingDepth(((ShardCodecInfo) info).getInnerBlockCodecInfo()); - } else { - return 1; - } - } - - public static class DefaultShardCodecInfo implements ShardCodecInfo { - - @Override - public String getType() { - return "ShardingCodec"; - } - - private final int[] innerBlockSize; - private final BlockCodecInfo innerBlockCodecInfo; - private final DataCodecInfo[] innerDataCodecInfos; - private final BlockCodecInfo indexBlockCodecInfo; - private final DataCodecInfo[] indexDataCodecInfos; - private final IndexLocation indexLocation; - - public DefaultShardCodecInfo( - final int[] innerBlockSize, - final BlockCodecInfo innerBlockCodecInfo, - final DataCodecInfo[] innerDataCodecInfos, - final BlockCodecInfo indexBlockCodecInfo, - final DataCodecInfo[] indexDataCodecInfos, - final IndexLocation indexLocation) { - this.innerBlockSize = innerBlockSize; - this.innerBlockCodecInfo = innerBlockCodecInfo; - this.innerDataCodecInfos = innerDataCodecInfos; - this.indexBlockCodecInfo = indexBlockCodecInfo; - this.indexDataCodecInfos = indexDataCodecInfos; - this.indexLocation = indexLocation; - } - - @Override - public int[] getInnerBlockSize() { - return innerBlockSize; - } - - @Override - public BlockCodecInfo getInnerBlockCodecInfo() { - return innerBlockCodecInfo; - } - - @Override - public DataCodecInfo[] getInnerDataCodecInfos() { - return innerDataCodecInfos; - } - - @Override - public BlockCodecInfo getIndexBlockCodecInfo() { - return indexBlockCodecInfo; - } - - @Override - public DataCodecInfo[] getIndexDataCodecInfos() { - return indexDataCodecInfos; - } - - @Override - public IndexLocation getIndexLocation() { - return indexLocation; - } - - @Override - public RawShardCodec create(final int[] blockSize, final DataCodecInfo... codecs) { - - // Number of elements (DataBlocks, nested shards) in each dimension per shard. - final int[] size = new int[blockSize.length]; - // blockSize argument is number of pixels in the shard - // innerBlockSize is number of pixels in each shard element (nested shard or DataBlock) - Arrays.setAll(size, d -> blockSize[d] / innerBlockSize[d]); - - final BlockCodec indexCodec = indexBlockCodecInfo.create( - DataType.UINT64, - ShardIndex.blockSizeFromIndexSize(size), - indexDataCodecInfos); - - return new RawShardCodec(size, indexLocation, indexCodec); - } - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java new file mode 100644 index 000000000..4dbbdcb56 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java @@ -0,0 +1,173 @@ +package org.janelia.saalfeldlab.n5.shardstuff; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; + +class ShardedDatasetAccess implements DatasetAccess { + + private final NestedGrid grid; + private final BlockCodec[] codecs; + + public ShardedDatasetAccess(final NestedGrid grid, final BlockCodec[] codecs) { + this.grid = grid; + this.codecs = codecs; + } + + @Override + public DataBlock readBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { + final NestedPosition position = new NestedPosition(grid, gridPosition); + return readBlockRecursive(kva.get(position.key()), position, grid.numLevels() - 1); + } + + private DataBlock readBlockRecursive( + final ReadData readData, + final NestedPosition position, + final int level) { + if (readData == null) { + return null; + } else if (level == 0) { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[0]; + return codec.decode(readData, position.absolute(0)); + } else { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[level]; + final RawShard shard = codec.decode(readData, position.absolute(level)).getData(); + return readBlockRecursive(shard.getElementData(position.relative(level - 1)), position, level - 1); + + } + } + + @Override + public void writeBlock(final PositionValueAccess kva, final DataBlock dataBlock) throws N5IOException { + final NestedPosition position = new NestedPosition(grid, dataBlock.getGridPosition()); + final long[] key = position.key(); + final ReadData existingData = kva.get(key); + final ReadData modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1); + kva.put(key, modifiedData); + } + + private ReadData writeBlockRecursive( + final ReadData existingReadData, + final DataBlock dataBlock, + final NestedPosition position, + final int level) { + if (level == 0) { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[0]; + return codec.encode(dataBlock); + } else { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[level]; + final long[] gridPos = position.absolute(level); + final RawShard shard = existingReadData == null ? + new RawShard(grid.relativeBlockSize(level)) : + codec.decode(existingReadData, gridPos).getData(); + final long[] elementPos = position.relative(level - 1); + final ReadData existingElementData = (level == 1) + ? null // if level == 1, we don't need to extract the nested (DataBlock) ReadData because it will be overridden anyway + : shard.getElementData(elementPos); + final ReadData modifiedElementData = writeBlockRecursive(existingElementData, dataBlock, position, level - 1); + shard.setElementData(modifiedElementData, elementPos); + return codec.encode(new RawShardDataBlock(gridPos, shard)); + } + } + + @Override + public void deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { + final NestedPosition position = new NestedPosition(grid, gridPosition); + final long[] key = position.key(); + if (grid.numLevels() == 1) { + // for non-sharded dataset, don't bother getting the value, just remove the key. + kva.remove(key); + } else { + final ReadData existingData = kva.get(key); + final ReadData modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); + if (modifiedData == null) { + kva.remove(key); + } else if (modifiedData != existingData) { + kva.put(key, modifiedData); + } + } + } + + private ReadData deleteBlockRecursive( + final ReadData existingReadData, + final NestedPosition position, + final int level) { + if (level == 0 || existingReadData == null) { + return null; + } else { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[level]; + final long[] gridPos = position.absolute(level); + final RawShard shard = codec.decode(existingReadData, gridPos).getData(); + final long[] elementPos = position.relative(level - 1); + final ReadData existingElementData = shard.getElementData(elementPos); + if (existingElementData == null) { + // The DataBlock (or the whole nested shard containing it) does not exist. + // This shard remains unchanged. + return existingReadData; + } else { + final ReadData modifiedElementData = deleteBlockRecursive(existingElementData, position, level - 1); + if (modifiedElementData == existingElementData) { + // The nested shard was not modified. + // This shard remains unchanged. + return existingReadData; + } + shard.setElementData(modifiedElementData, elementPos); + if (modifiedElementData == null) { + // The DataBlock or nested shard was removed. + // Check whether this shard becomes empty. + if (shard.index().allElementsNull()) { + // This shard is empty and should be removed. + return null; + } + } + return codec.encode(new RawShardDataBlock(gridPos, shard)); + } + } + } + + public static DatasetAccess create( + final DataType dataType, + int[] blockSize, + BlockCodecInfo blockCodecInfo, + DataCodecInfo[] dataCodecInfos + ) { + final int m = nestingDepth(blockCodecInfo); + + // There are m codecs: 1 DataBlock codecs, and m-1 shard codecs. + // The inner-most codec (the DataBlock codec) is at index 0. + final BlockCodec[] blockCodecs = new BlockCodec[m]; + final int[][] blockSizes = new int[m][]; + + for (int l = m - 1; l >= 0; --l) { + blockCodecs[l] = blockCodecInfo.create(dataType, blockSize, dataCodecInfos); + blockSizes[l] = blockSize; + if (l > 0) { + final ShardCodecInfo info = (ShardCodecInfo) blockCodecInfo; + blockCodecInfo = info.getInnerBlockCodecInfo(); + dataCodecInfos = info.getInnerDataCodecInfos(); + blockSize = info.getInnerBlockSize(); + } + } + + return new ShardedDatasetAccess<>(new NestedGrid(blockSizes), blockCodecs); + } + + private static int nestingDepth(BlockCodecInfo info) { + if (info instanceof ShardCodecInfo) { + return 1 + nestingDepth(((ShardCodecInfo) info).getInnerBlockCodecInfo()); + } else { + return 1; + } + } +} From cba45a4c97ca24a48cfc809c9db1131bcbaaf9b8 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 8 Sep 2025 22:39:33 +0200 Subject: [PATCH 328/423] refactor --- .../org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java} | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) rename src/{main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java => test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java} (99%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java similarity index 99% rename from src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java rename to src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java index 4511f0cab..8a6331ee7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardStuff2.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java @@ -15,7 +15,7 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; -public class RawShardStuff2 { +public class RawShardTest { public static void main(String[] args) { @@ -92,6 +92,8 @@ public static void main(String[] args) { // deleting a non-existent block should not fail datasetAccess.deleteBlock(store, new long[] {0, 0, 8}); + + System.out.println("all good"); } private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { From ca0c9f87f42b88ac38dab7108eaf3b6bca8c6ac9 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 12 Sep 2025 23:17:18 +0200 Subject: [PATCH 329/423] add TODO --- .../janelia/saalfeldlab/n5/shardstuff/RawShardDataBlock.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardDataBlock.java index 16071ce9b..7f794e7c1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardDataBlock.java @@ -17,6 +17,9 @@ public class RawShardDataBlock implements DataBlock { this.shard = shard; } + // TODO: should this be the number of elements in the Shard (number of + // sub-shards / datablock) along each dimension, or the number of + // pixels alon each dimension? @Override public int[] getSize() { return shard.index().size(); From f9645d8762d708e17956fa1eb6b14c1161aa0d61 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 12 Sep 2025 23:17:33 +0200 Subject: [PATCH 330/423] MAke ShardIndex package-private --- .../java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java index a4b0ac8d9..f8d1d1a61 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java @@ -13,7 +13,7 @@ import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData.SegmentsAndData; -public class ShardIndex { +class ShardIndex { private ShardIndex() { // utility class. should not be instantiated. From 500f600dcbf20e946baadebab430c9d3a0d13261 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 12 Sep 2025 23:17:40 +0200 Subject: [PATCH 331/423] clean up --- .../java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java index f8d1d1a61..e9e727dec 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java @@ -110,7 +110,6 @@ static int[] getStrides(final int[] size) { */ private static final int LONGS_PER_BLOCK = 2; - // TODO do we need additional offset here? static NDArray fromDataBlock( final DataBlock block ) { final long[] blockData = block.getData(); @@ -128,7 +127,6 @@ static NDArray fromDataBlock( final DataBlock block ) { return new NDArray<>(size, locations); } - // TODO do we use offset? If not, remove! static DataBlock toDataBlock( final NDArray locations, final long offset ) { final SegmentLocation[] data = locations.data; From ce464735a092f78ed4e9dbc61902777db38d9aa3 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 12 Sep 2025 23:41:14 +0200 Subject: [PATCH 332/423] Add BlockDodec.encodedSize(int[]) and implementations --- .../saalfeldlab/n5/codec/BlockCodec.java | 7 +--- .../n5/codec/DeterministicSizeDataCodec.java | 17 +++++++++ .../saalfeldlab/n5/codec/N5BlockCodecs.java | 36 +++++++++++++++++-- .../saalfeldlab/n5/codec/RawBlockCodecs.java | 11 ++++++ 4 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java index da54b5d11..269daa107 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java @@ -64,11 +64,6 @@ public interface BlockCodec { */ default long encodedSize(int[] blockSize) throws UnsupportedOperationException { - // TODO: Adapt https://github.com/saalfeldlab/n5/pull/165 to new naming! - - // TODO: REPLACE! This is a dirty hack, assuming this is only called for - // ShardIndex (UINT64) and the ShardIndex uses RawBlockCodec and - // RawCompression. - return DataBlock.getNumElements(blockSize) * 8L; + throw new UnsupportedOperationException(); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java new file mode 100644 index 000000000..214d37c49 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java @@ -0,0 +1,17 @@ +package org.janelia.saalfeldlab.n5.codec; + +/** + * A {@link DataCodec} that can deterministically determine the size of encoded + * data from the size of the raw data (i.e. encoding is data independent). + */ +public interface DeterministicSizeDataCodec extends DataCodec { + + /** + * Given {@code size} bytes of raw data, how many bytes will the encoded + * data have. + * + * @param size in bytes + * @return encoded size in bytes + */ + long encodedSize(long size); +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java index 03e058c6f..300190d6d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java @@ -51,6 +51,7 @@ import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.MODE_DEFAULT; import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.MODE_OBJECT; import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.MODE_VARLENGTH; +import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.headerSizeInBytes; public class N5BlockCodecs { @@ -120,9 +121,9 @@ private interface BlockCodecFactory { abstract static class N5AbstractBlockCodec implements BlockCodec { - private final FlatArrayCodec dataCodec; + final FlatArrayCodec dataCodec; private final DataBlockFactory dataBlockFactory; - private final DataCodec codec; + final DataCodec codec; N5AbstractBlockCodec(FlatArrayCodec dataCodec, DataBlockFactory dataBlockFactory, DataCodec codec) { this.dataCodec = dataCodec; @@ -189,6 +190,18 @@ protected BlockHeader decodeBlockHeader(final InputStream in) throws N5IOExcepti return BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); } + + @Override + public long encodedSize(final int[] blockSize) throws UnsupportedOperationException { + if (codec instanceof DeterministicSizeDataCodec) { + final int bytesPerElement = dataCodec.bytesPerElement(); + final int numElements = DataBlock.getNumElements(blockSize); + final int headerSize = headerSizeInBytes(MODE_DEFAULT, blockSize.length); + return headerSize + ((DeterministicSizeDataCodec) codec).encodedSize((long) numElements * bytesPerElement); + } else { + throw new UnsupportedOperationException(); + } + } } /** @@ -315,6 +328,25 @@ private static void writeBlockSize(final int[] blockSize, final DataOutputStream } } + static int headerSizeInBytes(final short mode, final int numDimensions) { + switch (mode) { + case MODE_DEFAULT: + return 2 + // 1 short for mode + 2 + // 1 short for blockSize.length + 4 * numDimensions; // 1 int for each blockSize element + case MODE_VARLENGTH: + return 2 +// 1 short for mode + 2 + // 1 short for blockSize.length + 4 * numDimensions + // 1 int for each blockSize dimension + 4; // 1 int for numElements + case MODE_OBJECT: + return 2 + // 1 short for mode + 4; // 1 int for numElements + default: + throw new N5Exception("unexpected mode: " + mode); + } + } + void writeTo(final OutputStream out) throws N5IOException { try { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java index 1fc606500..4a1ddb4ca 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java @@ -115,5 +115,16 @@ public DataBlock decode(ReadData readData, long[] gridPosition) { final T data = dataCodec.decode(decodeData, numElements); return dataBlockFactory.createDataBlock(blockSize, gridPosition, data); } + + @Override + public long encodedSize(final int[] blockSize) throws UnsupportedOperationException { + if (codec instanceof DeterministicSizeDataCodec) { + final int bytesPerElement = dataCodec.bytesPerElement(); + final int numElements = DataBlock.getNumElements(blockSize); + return ((DeterministicSizeDataCodec) codec).encodedSize((long) numElements * bytesPerElement); + } else { + throw new UnsupportedOperationException(); + } + } } } From 73042ccb6c4bd209335bd11d60d24a6e79f6a44b Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 13 Sep 2025 14:36:52 +0200 Subject: [PATCH 333/423] Concatenation of DeterministicSizeDataCodec --- ...oncatenatedDeterministicSizeDataCodec.java | 21 +++++++++++++++++++ .../saalfeldlab/n5/codec/DataCodec.java | 8 ++++++- .../n5/codec/DeterministicSizeDataCodec.java | 21 +++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDeterministicSizeDataCodec.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDeterministicSizeDataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDeterministicSizeDataCodec.java new file mode 100644 index 000000000..99febb924 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDeterministicSizeDataCodec.java @@ -0,0 +1,21 @@ +package org.janelia.saalfeldlab.n5.codec; + +class ConcatenatedDeterministicSizeDataCodec extends ConcatenatedDataCodec implements DeterministicSizeDataCodec { + + private final DeterministicSizeDataCodec[] codecs; + + ConcatenatedDeterministicSizeDataCodec(final DeterministicSizeDataCodec[] codecs) { + + super(codecs); + this.codecs = codecs; + } + + @Override + public long encodedSize(long size) { + + for (DeterministicSizeDataCodec codec : codecs) { + size = codec.encodedSize(size); + } + return size; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java index 5c7795f0f..50a11cbbf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -46,6 +46,9 @@ public interface DataCodec { /** * Create a {@code DataCodec} that sequentially applies {@code codecs} in * the given order for encoding, and in reverse order for decoding. + *

      + * If all {@code codecs} implement {@code DeterministicSizeDataCodec}, the + * returned {@code DataCodec} will also be a {@code DeterministicSizeDataCodec}. * * @param codecs * a list of DataCodecs @@ -59,7 +62,10 @@ static DataCodec concatenate(final DataCodec... codecs) { if (codecs.length == 1) return codecs[0]; - return new ConcatenatedDataCodec(codecs); + if (Arrays.stream(codecs).allMatch(DeterministicSizeDataCodec.class::isInstance)) + return new ConcatenatedDeterministicSizeDataCodec(Arrays.copyOf(codecs, codecs.length, DeterministicSizeDataCodec[].class)); + else + return new ConcatenatedDataCodec(codecs); } static DataCodec create(final DataCodecInfo... codecInfos) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java index 214d37c49..5c533b34a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java @@ -14,4 +14,25 @@ public interface DeterministicSizeDataCodec extends DataCodec { * @return encoded size in bytes */ long encodedSize(long size); + + /** + * Create a {@code DeterministicSizeDataCodec} that sequentially applies + * {@code codecs} in the given order for encoding, and in reverse order for + * decoding. + * + * @param codecs + * a list of DeterministicSizeDataCodec + * @return the concatenated DeterministicSizeDataCodec + */ + static DeterministicSizeDataCodec concatenate(final DeterministicSizeDataCodec... codecs) { + + if (codecs == null) + throw new NullPointerException(); + + if (codecs.length == 1) + return codecs[0]; + + return new ConcatenatedDeterministicSizeDataCodec(codecs); + } + } From 6b284e0746a47946df44fa0848cb9a39c2ab0b8c Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Sat, 13 Sep 2025 14:38:54 +0200 Subject: [PATCH 334/423] RawCompression is a DeterministicSizeDataCodec --- .../java/org/janelia/saalfeldlab/n5/RawCompression.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index bb4dfe51b..093c44017 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -54,10 +54,11 @@ package org.janelia.saalfeldlab.n5; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeDataCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; @CompressionType("raw") -public class RawCompression implements Compression { +public class RawCompression implements Compression, DeterministicSizeDataCodec { private static final long serialVersionUID = 7526445806847086477L; @@ -75,4 +76,9 @@ public ReadData encode(final ReadData readData) { public ReadData decode(final ReadData readData) { return readData; } + + @Override + public long encodedSize(final long size) { + return size; + } } From 373c0ecf3438e121b1e87ca202be1ec4272715de Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 17 Sep 2025 19:58:49 -0400 Subject: [PATCH 335/423] fix/test: ReadData test for length vs requiredLength --- .../org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index e62f9e2c1..e7fe3b9c5 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -88,7 +88,8 @@ public void testFileKvaReadData() throws IOException { final ReadData readData = new FileSystemKeyValueAccess(FileSystems.getDefault()) .createReadData(tmpF.getAbsolutePath()); - assertEquals("file read data length", 128, readData.length()); + assertEquals("file read data length", -1, readData.length()); + assertEquals("file read data length", 128, readData.requireLength()); sliceTestHelper(readData, N); } From 6dec87911fc7c7b9bc20727179ba020195409914 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 17 Sep 2025 20:00:46 -0400 Subject: [PATCH 336/423] feat!: use DatasetAccess instead of BlockCodec * rm getBlockCodecInfo and getBlockCodec from DatasetAttrubutes * add getDatasetAccess * use getDatasetAccess in readBlock / writeBlock * add PositionValueAccess wrapping KeyValueAccess --- .../saalfeldlab/n5/DatasetAttributes.java | 14 ++-- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 9 ++- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 14 ++-- .../n5/shardstuff/PositionValueAccess.java | 66 +++++++++++++++++++ .../n5/shardstuff/ShardedDatasetAccess.java | 3 +- 5 files changed, 84 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 1127c658a..166575a5b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -36,6 +36,8 @@ import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.shardstuff.DatasetAccess; +import org.janelia.saalfeldlab.n5.shardstuff.ShardedDatasetAccess; /** * Mandatory dataset attributes: @@ -135,15 +137,11 @@ public DataType getDataType() { * * @return the {@code BlockCodecInfo} for this dataset */ - public BlockCodecInfo getBlockCodecInfo() { + public DatasetAccess getDatasetAccess() { - return blockCodecInfo; - } - - @SuppressWarnings("unchecked") - BlockCodec getBlockCodec() { - - return (BlockCodec) blockCodec; + return ShardedDatasetAccess.create(getDataType(), + new int[] {24, 24, 24}, + blockCodecInfo, dataCodecInfos); } public HashMap asMap() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 714b86d37..d06839976 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -34,7 +34,7 @@ import java.io.UncheckedIOException; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.shardstuff.PositionValueAccess; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -96,11 +96,10 @@ default DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { - final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), gridPosition); - try { - final ReadData blockData = getKeyValueAccess().createReadData(path); - return datasetAttributes.getBlockCodec().decode(blockData, gridPosition); + final PositionValueAccess posKva = PositionValueAccess.fromKva( + getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName)); + return datasetAttributes.getDatasetAccess().readBlock(posKva, gridPosition); } catch (N5Exception.N5NoSuchKeyException e) { return null; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 97b9a745e..b59da02a4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -62,6 +62,7 @@ import com.google.gson.JsonSyntaxException; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.shardstuff.PositionValueAccess; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -241,17 +242,16 @@ default void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception { - final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition()); - try ( - final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath); - final OutputStream out = lock.newOutputStream() - ) { - datasetAttributes.getBlockCodec().encode(dataBlock).writeTo(out); - } catch (final IOException | UncheckedIOException e) { + try { + final PositionValueAccess posKva = PositionValueAccess.fromKva( + getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path)); + datasetAttributes.getDatasetAccess().writeBlock(posKva, dataBlock); + } catch (final UncheckedIOException e) { throw new N5IOException( "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, e); } + } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java index 88be18c92..ae5f91c69 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java @@ -1,6 +1,13 @@ package org.janelia.saalfeldlab.n5.shardstuff; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; + +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** @@ -16,4 +23,63 @@ public interface PositionValueAccess { void put(long[] key, ReadData data) throws N5Exception.N5IOException; void remove(long[] key) throws N5Exception.N5IOException; + + public static PositionValueAccess fromKva( + final KeyValueAccess kva, + final URI uri, + final String normalPath) { + + return new KvaPositionValueAccess(kva, uri, normalPath); + } + + class KvaPositionValueAccess implements PositionValueAccess { + + private final KeyValueAccess kva; + private final URI uri; + private final String normalPath; + + KvaPositionValueAccess(final KeyValueAccess kva, + final URI uri, + final String normalPath) { + this.kva = kva; + this.uri = uri; + this.normalPath = normalPath; + } + + // TODO this duplicates GsonKeyValueReader.absoluteDataBlockPath + // is this where we want the logic? + private String absolutePath( + final long... gridPosition) { + + final String[] components = new String[gridPosition.length + 1]; + components[0] = normalPath; + int i = 0; + for (final long p : gridPosition) + components[++i] = Long.toString(p); + + return kva.compose(uri, components); + } + + @Override + public ReadData get(long[] key) throws N5IOException { + return kva.createReadData(absolutePath(key)); + } + + @Override + public void put(long[] key, ReadData data) throws N5IOException { + + try ( final LockedChannel ch = kva.lockForWriting(absolutePath(key)); + final OutputStream outputStream = ch.newOutputStream();) { + data.writeTo(outputStream); + } catch (IOException e) { + throw new N5IOException(e); + } + } + + @Override + public void remove(long[] key) throws N5IOException { + kva.delete( absolutePath(key)); + } + + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java index 4dbbdcb56..a39ce61a5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java @@ -10,7 +10,7 @@ import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; -class ShardedDatasetAccess implements DatasetAccess { +public class ShardedDatasetAccess implements DatasetAccess { private final NestedGrid grid; private final BlockCodec[] codecs; @@ -41,7 +41,6 @@ private DataBlock readBlockRecursive( final BlockCodec codec = (BlockCodec) codecs[level]; final RawShard shard = codec.decode(readData, position.absolute(level)).getData(); return readBlockRecursive(shard.getElementData(position.relative(level - 1)), position, level - 1); - } } From db1928dbf6bef9240eaf2ab3d26a83ff521e4950 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 22 Sep 2025 21:47:32 +0200 Subject: [PATCH 337/423] Revise SegmentLocation Add TODO for potential renaming: SegmentLocation will be uased in other instances that require a (offset, length) pair. Therefore, it should probably be renamed to "Range" or similar. Add convenience method end() == offset() + length(). Add equals/hashCode for DefaultSegmentLocation. --- .../readdata/segment/DefaultSegmentLocation.java | 16 ++++++++++++++++ .../n5/readdata/segment/SegmentLocation.java | 6 ++++++ 2 files changed, 22 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentLocation.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentLocation.java index 3d63c6f55..dc05775f3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentLocation.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentLocation.java @@ -20,6 +20,22 @@ public long length() { return length; } + @Override + public final boolean equals(final Object o) { + if (!(o instanceof DefaultSegmentLocation)) + return false; + + final DefaultSegmentLocation that = (DefaultSegmentLocation) o; + return offset == that.offset && length == that.length; + } + + @Override + public int hashCode() { + int result = Long.hashCode(offset); + result = 31 * result + Long.hashCode(length); + return result; + } + @Override public String toString() { return "SegmentLocation{" + diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java index 6525b57cf..2c058667b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java @@ -2,6 +2,8 @@ import java.util.Comparator; +// TODO: If we use this to describe slices, it should be renamed probably!? +// Ideas: Range, SliceLocation public interface SegmentLocation { Comparator COMPARATOR = Comparator @@ -12,6 +14,10 @@ public interface SegmentLocation { long length(); + default long end() { + return offset() + length(); + } + static SegmentLocation at(final long offset, final long length) { return new DefaultSegmentLocation(offset, length); } From e8ce75be41fded5cb34c047758ac421222aba4bf Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 22 Sep 2025 21:51:05 +0200 Subject: [PATCH 338/423] Slice prefetching math and tests --- .../n5/readdata/segment/Slices.java | 76 ++++++++++ .../n5/readdata/segment/SlicesTest.java | 133 ++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Slices.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SlicesTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Slices.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Slices.java new file mode 100644 index 000000000..604d73847 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Slices.java @@ -0,0 +1,76 @@ +package org.janelia.saalfeldlab.n5.readdata.segment; + +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +class Slices { + + private Slices() { + // utility class. should not be instantiated. + } + + /** + * Pre-conditions: + *

        + *
      1. Slices are ordered by offset.
        2. + *
      2. If two slices overlap, no slice is fully contained within the other. (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
        3. + *
      + * + * @param slices + * ordered list of slices + * @param offset + * @param length + * + * @return + */ + static T findContainingSlice(final List slices, final long offset, final long length) { + // Find the slice s with largest s.offset <= offset. + + final int i = Collections.binarySearch(slices, SegmentLocation.at(offset, 0), Comparator.comparingLong(SegmentLocation::offset)); + + // Largest index of a slice with slice.offset <= offset. + final int index = i < 0 ? -i - 2 : i; + if (index < 0) { + // We find no overlapping slice, because + // slices[0].offset is already too large. + return null; + } + + final T slice = slices.get(index); + if (slice.end() < offset + length) { + return null; + } + + return slice; + } + + static T findContainingSlice(final List slices, final SegmentLocation range) { + return findContainingSlice(slices, range.offset(), range.length()); + } + + /** + * Add a new {@code slice} to the {@code slice} list. + * Note, that the new {@code slice} is expected to not be fully contained in an existing slice! + * This will insert {@code slice} into the list at the correct position ({@code slices} is ordered by slice offset), and remove all existing slices that are fully contained in the new {@code slice}. + */ + static void addSlice(final List slices, final T slice) { + + final int i = Collections.binarySearch(slices, slice, Comparator.comparingLong(SegmentLocation::offset)); + final int from = i < 0 ? -i - 1 : i; + + int to = from; + while (to < slices.size() && slices.get(to).end() <= slice.end()) { + ++to; + } + + if (from == to) { + // empty range: just insert + slices.add(from, slice); + } else { + // overwrite the first element in range, remove the rest + slices.set(from, slice); + slices.subList(from + 1, to).clear(); + } + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SlicesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SlicesTest.java new file mode 100644 index 000000000..a005ef34a --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SlicesTest.java @@ -0,0 +1,133 @@ +package org.janelia.saalfeldlab.n5.readdata.segment; + +import java.util.ArrayList; +import java.util.List; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + + +public class SlicesTest { + + private List createSlices(final long[] offsets, final long[] lengths) { + final List slices = new ArrayList<>(); + for (int i = 0; i < offsets.length; ++i) { + slices.add(SegmentLocation.at(offsets[i], lengths[i])); + } + return slices; + } + + @Test + public void testFindContaining() { + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,6) [-----------] + // (6,4) [---------] + // (8,6) [-----------] + + final List slices = createSlices( + new long[] {2, 6, 8}, + new long[] {6, 4, 6}); + SegmentLocation slice; + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (1,1) [-] + slice = Slices.findContainingSlice(slices, 1, 1); + assertEquals(null, slice); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,1) [-] + slice = Slices.findContainingSlice(slices, 2, 1); + assertEquals(2, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,6) [-----------] + slice = Slices.findContainingSlice(slices, 2, 6); + assertEquals(2, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,7) [-------------] + slice = Slices.findContainingSlice(slices, 2, 7); + assertEquals(null, slice); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (6,4) [-------] + slice = Slices.findContainingSlice(slices, 6, 4); + assertEquals(6, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (8,2) [---] + slice = Slices.findContainingSlice(slices, 8, 2); + assertEquals(8, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (12,2) [---] + slice = Slices.findContainingSlice(slices, 12, 2); + assertEquals(8, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (12,3) [-----] + slice = Slices.findContainingSlice(slices, 12, 3); + assertEquals(null, slice); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (14,1) [-] + slice = Slices.findContainingSlice(slices, 14, 1); + assertEquals(null, slice); + } + + + @Test + public void testAddSlice() { + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,6) [-----------] + // (6,4) [---------] + // (8,6) [-----------] + final List initial = createSlices( + new long[] {2, 6, 8}, + new long[] {6, 4, 6}); + List slices; + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, SegmentLocation.at(0, 1)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (0,1) [-] + // (2,6) [-----------] + // (6,4) [---------] + // (8,6) [-----------] + assertEquals(createSlices( + new long[] {0, 2, 6, 8}, + new long[] {1, 6, 4, 6}), slices); + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, SegmentLocation.at(0, 16)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (0,16)[-------------------------------] + assertEquals(createSlices( + new long[] {0}, + new long[] {16}), slices); + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, SegmentLocation.at(2, 8)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,8) [-----------------] + // (8,6) [-----------] + assertEquals(createSlices( + new long[] {2, 8}, + new long[] {8, 6}), slices); + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, SegmentLocation.at(1, 10)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (1,10) [---------------------] + // (8,6) [-----------] + assertEquals(createSlices( + new long[] {1, 8}, + new long[] {10, 6}), slices); + } +} From 2082f86e564970e0769f4334146b3f67d43709df Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 22 Sep 2025 21:51:29 +0200 Subject: [PATCH 339/423] WIP: Add SliceTrackingReadData --- .../segment/SliceTrackingReadData.java | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java new file mode 100644 index 000000000..4b895ea8d --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java @@ -0,0 +1,112 @@ +package org.janelia.saalfeldlab.n5.readdata.segment; + +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +class SliceTrackingReadData implements ReadData { + + private static class Slice implements SegmentLocation { + + private final long offset; + private final long length; + private final ReadData data; + + Slice(final long offset, final long length, final ReadData data) { + this.offset = offset; + this.length = length; + this.data = data; + } + + @Override + public long offset() { + return offset; + } + + @Override + public long length() { + return length; + } + + @Override + public String toString() { + return "{" + offset + ", " + length + '}'; + } + } + + private final List slices = new ArrayList<>(); + + /** + * The {@code ReadData} providing our data. + */ + private final ReadData delegate; + + private SliceTrackingReadData(final ReadData delegate) { + this.delegate = delegate; + } + + static ReadData wrap(final ReadData readData) { + return new SliceTrackingReadData(readData); + } + + @Override + public long length() { + return delegate.length(); + } + + @Override + public long requireLength() throws N5IOException { + return delegate.requireLength(); + } + + @Override + public ReadData slice(final long offset, final long length) throws N5IOException { + final Slice containing = Slices.findContainingSlice(slices, offset, length); + if (containing != null) { + return containing.data.slice(offset - containing.offset, length); + } else { + final ReadData data = delegate.slice(offset, length); + Slices.addSlice(slices, new Slice(offset, length, data)); + return data; + } + } + + @Override + public InputStream inputStream() throws N5IOException, IllegalStateException { + return delegate.inputStream(); + } + + @Override + public byte[] allBytes() throws N5IOException, IllegalStateException { + return delegate.allBytes(); + } + + @Override + public ByteBuffer toByteBuffer() throws N5IOException, IllegalStateException { + return delegate.toByteBuffer(); + } + + @Override + public ReadData materialize() throws N5IOException { + delegate.materialize(); + if (slices.size() != 1 || slices.get(0).data != delegate) { + slices.clear(); + slices.add(new Slice(0, length(), delegate)); + } + return this; + } + + @Override + public void writeTo(final OutputStream outputStream) throws N5IOException, IllegalStateException { + delegate.writeTo(outputStream); + } + + @Override + public ReadData encode(final OutputStreamOperator encoder) { + return delegate.encode(encoder); + } +} From cc1312a808543b01b06b5025f4e8a0c53e3a3ee6 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 22 Sep 2025 11:31:52 +0200 Subject: [PATCH 340/423] WIP: Always use SliceTrackingReadData in SegmentedReadData wrapper --- .../saalfeldlab/n5/readdata/segment/SegmentedReadData.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java index 5745f3701..687e70194 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java @@ -18,7 +18,7 @@ interface SegmentsAndData { * of {@link SegmentedReadData#segments()}. */ static SegmentedReadData wrap(ReadData readData) { - return DefaultSegmentedReadData.wrap(readData); + return DefaultSegmentedReadData.wrap(SliceTrackingReadData.wrap(readData)); } /** @@ -38,7 +38,7 @@ static SegmentsAndData wrap(ReadData readData, SegmentLocation... locations) { * {@link SegmentsAndData#data()} are ordered by offset). */ static SegmentsAndData wrap(ReadData readData, List locations) { - return DefaultSegmentedReadData.wrap(readData, locations); + return DefaultSegmentedReadData.wrap(SliceTrackingReadData.wrap(readData), locations); } static SegmentedReadData concatenate(List readDatas) { From 97098890e371a5bf166c5fd76596bc56e43ab2a3 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 22 Sep 2025 22:36:09 +0200 Subject: [PATCH 341/423] Add (empty) prefetch method --- .../n5/readdata/segment/SliceTrackingReadData.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java index 4b895ea8d..113cca7a2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java @@ -4,6 +4,7 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -109,4 +110,17 @@ public void writeTo(final OutputStream outputStream) throws N5IOException, Illeg public ReadData encode(final OutputStreamOperator encoder) { return delegate.encode(encoder); } + + + // --- -- - prefetching - -- --- + + /** + * Indicates that the given slices will be subsequently read. + * {@code ReadData} implementations (optionally) may take steps to prepare + * for these subsequent slices. + */ + // TODO: where to put this? Could be in ReadData interface with empty default implementation? + public void prefetch(final Collection slices) { + + } } From 39f1c8a4c69acef654c21180f2899e7ad29c3fd5 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 22 Sep 2025 22:45:21 +0200 Subject: [PATCH 342/423] minimal prefetch implementation --- .../segment/SliceTrackingReadData.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java index 113cca7a2..2410353c1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java @@ -120,7 +120,27 @@ public ReadData encode(final OutputStreamOperator encoder) { * for these subsequent slices. */ // TODO: where to put this? Could be in ReadData interface with empty default implementation? - public void prefetch(final Collection slices) { + public void prefetch(final Collection ranges) { + + // Minimal implementation: Find offset and length covering all ranges + // that are not yet fully covered by existing slices. Then materialize + // the slice covering that range. + + long fromIndex = Long.MAX_VALUE; + long toIndex = Long.MIN_VALUE; + for (final SegmentLocation slice : ranges) { + if (!isCovered(slice)) { + fromIndex = Math.min(fromIndex, slice.offset()); + toIndex = Math.max(toIndex, slice.end()); + } + } + + if (fromIndex < toIndex) { + slice(fromIndex, toIndex - fromIndex).materialize(); + } + } + private boolean isCovered(final SegmentLocation slice) { + return Slices.findContainingSlice(slices, slice) != null; } } From 552d7123332f94819aea30b78fdd5d4c9fbc2edb Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Sep 2025 11:58:00 -0400 Subject: [PATCH 343/423] fix: NestedGrid validates blockSizes --- .../saalfeldlab/n5/shardstuff/Nesting.java | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java index 866682b43..fcbb0f223 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java @@ -146,11 +146,11 @@ public static class NestedGrid { */ public NestedGrid(int[][] blockSizes) { - // TODO: validate - // [ ] not null - // [ ] nesteds not null - // [ ] nesteds same length - // [ ] sizes match up to integer multiples + if (blockSizes == null) + throw new IllegalArgumentException("blockSizes is null"); + + if (blockSizes[0] == null) + throw new IllegalArgumentException("blockSizes[0] is null"); m = blockSizes.length; n = blockSizes[0].length; @@ -158,7 +158,36 @@ public NestedGrid(int[][] blockSizes) { r = new int[m][n]; for (int l = 0; l < m; ++l) { final int k = Math.max(0, l - 1); + + if (blockSizes[l] == null) + throw new IllegalArgumentException("blockSizes[" + l + "] null"); + + if (blockSizes[l].length != n) + throw new IllegalArgumentException( + String.format("Block size at level %d has a different length (%d vs %d)", l, n, blockSizes[l].length)); + for (int d = 0; d < n; ++d) { + + if (blockSizes[l][d] <= 0 ) { + throw new IllegalArgumentException( + String.format("Block sizes at level %d (%d) is negative for dimension %d.", + l, blockSizes[l][d], d)); + } + + if (blockSizes[l][d] > blockSizes[k][d]) { + throw new IllegalArgumentException( + String.format("Block sizes at level %d (%d) is larger than previous level (%d) " + + " for dimension %d.", + l, blockSizes[l][d], blockSizes[k][d], d)); + } + + if (blockSizes[k][d] % blockSizes[l][d] != 0) { + throw new IllegalArgumentException( + String.format("Block sizes at level %d (%d) not a multiple of previous level (%d) " + + " for dimension %d.", + l, blockSizes[l][d], blockSizes[k][d], d)); + } + s[l][d] = blockSizes[l][d] / blockSizes[0][d]; r[l][d] = blockSizes[l][d] / blockSizes[k][d]; } From 1861cc1891ca3bd4b79c3dc80dc0ae5263918024 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Sep 2025 12:02:42 -0400 Subject: [PATCH 344/423] fix: use NestedGrid for error checking * before building BlockCodecs --- .../n5/shardstuff/ShardedDatasetAccess.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java index a39ce61a5..75eb1d8e9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java @@ -20,6 +20,10 @@ public ShardedDatasetAccess(final NestedGrid grid, final BlockCodec[] codecs) this.codecs = codecs; } + public NestedGrid getGrid() { + return grid; + } + @Override public DataBlock readBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { final NestedPosition position = new NestedPosition(grid, gridPosition); @@ -145,11 +149,10 @@ public static DatasetAccess create( // There are m codecs: 1 DataBlock codecs, and m-1 shard codecs. // The inner-most codec (the DataBlock codec) is at index 0. - final BlockCodec[] blockCodecs = new BlockCodec[m]; final int[][] blockSizes = new int[m][]; + for (int l = m - 1; l >= 0; --l) { - blockCodecs[l] = blockCodecInfo.create(dataType, blockSize, dataCodecInfos); blockSizes[l] = blockSize; if (l > 0) { final ShardCodecInfo info = (ShardCodecInfo) blockCodecInfo; @@ -159,7 +162,15 @@ public static DatasetAccess create( } } - return new ShardedDatasetAccess<>(new NestedGrid(blockSizes), blockCodecs); + // NestedGrid validates block sizes, so instantiate it before creating the blockCodecs + final NestedGrid grid = new NestedGrid(blockSizes); + + final BlockCodec[] blockCodecs = new BlockCodec[m]; + for (int l = m - 1; l >= 0; --l) { + blockCodecs[l] = blockCodecInfo.create(dataType, blockSize, dataCodecInfos); + } + + return new ShardedDatasetAccess<>(grid, blockCodecs); } private static int nestingDepth(BlockCodecInfo info) { From ec27cb595471dd146ba319260deca06245c4b4be Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Sep 2025 15:38:06 -0400 Subject: [PATCH 345/423] fix: Nesting blockSize size checks were backwards --- .../java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java index fcbb0f223..03e9ba14e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java @@ -174,14 +174,14 @@ public NestedGrid(int[][] blockSizes) { l, blockSizes[l][d], d)); } - if (blockSizes[l][d] > blockSizes[k][d]) { + if (blockSizes[l][d] < blockSizes[k][d]) { throw new IllegalArgumentException( - String.format("Block sizes at level %d (%d) is larger than previous level (%d) " + String.format("Block sizes at level %d (%d) is smaller than previous level (%d) " + " for dimension %d.", l, blockSizes[l][d], blockSizes[k][d], d)); } - if (blockSizes[k][d] % blockSizes[l][d] != 0) { + if (blockSizes[l][d] % blockSizes[k][d] != 0) { throw new IllegalArgumentException( String.format("Block sizes at level %d (%d) not a multiple of previous level (%d) " + " for dimension %d.", From 10b87c2166f1499f4fe10bc4cb2003463f4bf75d Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Sep 2025 15:42:00 -0400 Subject: [PATCH 346/423] feat: add NestedGrid.getBlockSize --- .../org/janelia/saalfeldlab/n5/shardstuff/Nesting.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java index 03e9ba14e..332107ed5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java @@ -137,6 +137,8 @@ public static class NestedGrid { // r[i][d] is block size at level i relative to level i-1 private final int[][] r; + private final int[][] blockSizes; + /** * {@code blockSizes[l][d]} is the block size at level {@code l} in dimension {@code d}. * Level 0 is the highest resolution (smallest block sizes). @@ -152,6 +154,8 @@ public NestedGrid(int[][] blockSizes) { if (blockSizes[0] == null) throw new IllegalArgumentException("blockSizes[0] is null"); + this.blockSizes = blockSizes; + m = blockSizes.length; n = blockSizes[0].length; s = new int[m][n]; @@ -202,6 +206,10 @@ public int numDimensions() { return n; } + public int[] getBlockSize(int level) { + return blockSizes[level]; + } + public void absolutePosition( final long[] sourcePos, final int sourceLevel, From 07b4b811cbcf7d809a97c2e4e3ac1697d65a19a6 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Sep 2025 15:50:01 -0400 Subject: [PATCH 347/423] fix: DatasetAttributes build dataCodecInfos from shard size * tmp commenting out some shard methods --- .../saalfeldlab/n5/DatasetAttributes.java | 420 +++++++++--------- .../saalfeldlab/n5/DatasetAttributesTest.java | 206 ++++----- 2 files changed, 324 insertions(+), 302 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 5a835c9cd..1337e8604 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -39,11 +39,14 @@ import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.shard.BlockAsShardCodec; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.util.Position; import org.janelia.saalfeldlab.n5.shardstuff.DatasetAccess; +import org.janelia.saalfeldlab.n5.shardstuff.DefaultShardCodecInfo; import org.janelia.saalfeldlab.n5.shardstuff.ShardCodecInfo; +import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; import org.janelia.saalfeldlab.n5.shardstuff.ShardedDatasetAccess; import java.io.Serializable; @@ -94,72 +97,33 @@ public class DatasetAttributes implements Serializable { // number of samples per block per dimension private final int[] blockSize; - // number of samples per shard per dimension - private final int[] shardSize; - private final DataType dataType; - private final ShardingCodec shardingCodec; private final BlockCodecInfo blockCodecInfo; private final DataCodecInfo[] dataCodecInfos; - - private final BlockCodec blockCodec; + private final DatasetAccess access; public DatasetAttributes( final long[] dimensions, - final int[] shardSize, - final int[] blockSize, + final int[] outerBlockSize, final DataType dataType, final BlockCodecInfo blockCodecInfo, final DataCodecInfo... dataCodecInfos) { - validateBlockShardSizes(dimensions, shardSize, blockSize); - this.dimensions = dimensions; - this.blockSize = blockSize; - this.shardSize = shardSize; this.dataType = dataType; this.blockCodecInfo = blockCodecInfo == null ? defaultBlockCodecInfo() : blockCodecInfo; - this.dataCodecInfos = Arrays.stream(dataCodecInfos).filter(it -> !(it instanceof RawCompression)).toArray(DataCodecInfo[]::new); - if (this.blockCodecInfo instanceof ShardingCodec) - shardingCodec = (ShardingCodec)this.blockCodecInfo; - else - shardingCodec = new BlockAsShardCodec(this.blockCodecInfo); - blockCodec = this.shardingCodec.create(this, dataCodecInfos); + this.dataCodecInfos = Arrays.stream(dataCodecInfos) + .filter(it -> it != null && !(it instanceof RawCompression)) + .toArray(DataCodecInfo[]::new); - } - - private void validateBlockShardSizes(long[] dimensions, int[] shardSize, int[] blockSize) { - - final int nd = dimensions.length; + final ShardedDatasetAccess shardAccess = ShardedDatasetAccess.create( + getDataType(), outerBlockSize, + this.blockCodecInfo, this.dataCodecInfos); + access = shardAccess; - if (blockSize.length != nd) - throw new IllegalArgumentException(String.format("Number of block dimensions (%d) must equal number of dimensions (%d).", - blockSize.length, nd)); - - if (shardSize.length != nd) - throw new IllegalArgumentException(String.format("Number of shard dimensions (%d) must equal number of dimensions (%d).", - shardSize.length, nd)); - - for (int i = 0; i < blockSize.length; i++) { - - if (blockSize[i] <= 0) - throw new IllegalArgumentException(String.format("Block size in dimension %d (%d) is <= 0", - i, blockSize[i])); - - if (shardSize[i] < blockSize[i]) - throw new IllegalArgumentException(String.format("Shard size in dimension %d (%d) is larger than the block size (%d)", - i, shardSize[i], blockSize[i])); - else if (shardSize[i] % blockSize[i] != 0) - throw new IllegalArgumentException(String.format("Shard size in dimension %d (%d) not a multiple of the block size (%d)", - i, shardSize[i], blockSize[i])); - } - } - - protected BlockCodecInfo defaultArrayCodec() { - - return new N5BlockCodecInfo(); + this.blockSize = shardAccess.getGrid().getBlockSize(0); } /** @@ -177,7 +141,7 @@ public DatasetAttributes( final DataType dataType, final DataCodecInfo compression) { - this(dimensions, blockSize, blockSize, dataType, null, compression); + this(dimensions, blockSize, dataType, null, compression); } /** @@ -192,189 +156,250 @@ public DatasetAttributes( final int[] blockSize, final DataType dataType) { - this(dimensions, blockSize, blockSize, dataType, null); - } - - protected BlockCodecInfo defaultBlockCodecInfo() { - - return new N5BlockCodecInfo(); + this(dimensions, blockSize, dataType, null); } - public long[] getDimensions() { + /** + * Constructs a DatasetAttributes instance with specified dimensions, block size, data type, and default codecs + * + * @param dimensions the dimensions of the dataset + * @param blockSize the size of the blocks in the dataset + * @param dataType the data type of the dataset + */ + public DatasetAttributes( + final long[] dimensions, + final int[] shardSize, + final int[] blockSize, + final DataType dataType) { - return dimensions; + this(dimensions, shardSize, dataType, + defaultShardCodecInfo(blockSize)); } - public int getNumDimensions() { + protected static BlockCodecInfo defaultShardCodecInfo(int[] innerBlockSize) { - return dimensions.length; + return new DefaultShardCodecInfo( + innerBlockSize, + new N5BlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + IndexLocation.END); } - public int[] getShardSize() { + private void validateBlockShardSizes(long[] dimensions, int[] shardSize, int[] blockSize) { - return shardSize; - } + final int nd = dimensions.length; - public int[] getBlockSize() { + if (blockSize.length != nd) + throw new IllegalArgumentException(String.format("Number of block dimensions (%d) must equal number of dimensions (%d).", + blockSize.length, nd)); - return blockSize; - } + if (shardSize.length != nd) + throw new IllegalArgumentException(String.format("Number of shard dimensions (%d) must equal number of dimensions (%d).", + shardSize.length, nd)); - /** - * Returns the number of blocks per dimension for each shard. - * - * @return the blocks per shard - */ - public int[] getBlocksPerShard() { + for (int i = 0; i < blockSize.length; i++) { - final int[] shardSize = getShardSize(); - final int nd = getNumDimensions(); - final int[] blocksPerShard = new int[nd]; - final int[] blockSize = getBlockSize(); - for (int i = 0; i < nd; i++) - blocksPerShard[i] = shardSize[i] / blockSize[i]; + if (blockSize[i] <= 0) + throw new IllegalArgumentException(String.format("Block size in dimension %d (%d) is <= 0", + i, blockSize[i])); - return blocksPerShard; + if (shardSize[i] < blockSize[i]) + throw new IllegalArgumentException(String.format("Shard size in dimension %d (%d) is larger than the block size (%d)", + i, shardSize[i], blockSize[i])); + else if (shardSize[i] % blockSize[i] != 0) + throw new IllegalArgumentException(String.format("Shard size in dimension %d (%d) not a multiple of the block size (%d)", + i, shardSize[i], blockSize[i])); + } } - /** - * Returns the number of blocks per dimension for this dataset. - * - * @return blocks per dataset - */ - public long[] blocksPerDataset() { + protected BlockCodecInfo defaultArrayCodec() { - return IntStream.range(0, getNumDimensions()) - .mapToLong(i -> (long)Math.ceil((double)getDimensions()[i] / getBlockSize()[i])) - .toArray(); + return new N5BlockCodecInfo(); } - /** - * Returns the number of shards per dimension for this dataset. - * - * @return shards per dataset - */ - public long[] shardsPerDataset() { - return IntStream.range(0, getNumDimensions()) - .mapToLong(i -> (long)Math.ceil((double)getDimensions()[i] / getShardSize()[i])) - .toArray(); - } - - /** - * Returns the total number of blocks in each shard. - * - * @return number of blocks in a shard - */ - public long getNumBlocksPerShard() { + protected BlockCodecInfo defaultBlockCodecInfo() { - return Arrays.stream(getBlocksPerShard()).reduce(1, (x, y) -> x * y); + return new N5BlockCodecInfo(); } - /** - * Given a block's position relative to the dataset, returns the position of - * the shard containing that block. - * - * @param blockGridPosition position of a block relative to the dataset - * @return the position of the containing shard in the shard grid - */ - public long[] getShardPositionForBlock(final long... blockGridPosition) { - - final int[] blocksPerShard = getBlocksPerShard(); - final long[] shardGridPosition = new long[blockGridPosition.length]; - for (int i = 0; i < shardGridPosition.length; i++) { - shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / blocksPerShard[i]); - } + public long[] getDimensions() { - return shardGridPosition; + return dimensions; } - /** - * Given a {@code datasetRelativeBlockPosition} returns the position - * relative the the shard at position {@code shardPosition}. - * - * @param shardPosition position of the shard - * @param datasetRelativeBlockPosition position of the block relative to the dataset - * @return position of the block relative to the shard - */ - public int[] getShardRelativeBlockPosition(final long[] shardPosition, final long[] datasetRelativeBlockPosition) { - - final long[] shardPos = getShardPositionForBlock(datasetRelativeBlockPosition); - if (!Arrays.equals(shardPosition, shardPos)) - return null; + public int getNumDimensions() { - final int[] shardSize = getBlocksPerShard(); - final int[] shardRelativeBlockPosition = new int[shardSize.length]; - for (int i = 0; i < shardSize.length; i++) { - shardRelativeBlockPosition[i] = (int)(datasetRelativeBlockPosition[i] % shardSize[i]); - } - return shardRelativeBlockPosition; + return dimensions.length; } - /** - * Given a {@code shardRelativeBlockPosition} relative to the shard at - * position {@code shardPosition}, returns the block' position relative the - * dataset. - * - * @param shardPosition position of the shard - * @param shardRelativeBlockPosition position of the block relative to the shard - * @return position of the block relative to the dataset - */ - public long[] getBlockPositionFromShardPosition(final long[] shardPosition, final int[] shardRelativeBlockPosition) { +// public int[] getShardSize() { +// +// return shardSize; +// } - final int[] shardBlockSize = getBlocksPerShard(); - final long[] datasetRelativeBlockPosition = new long[getNumDimensions()]; - for (int i = 0; i < getNumDimensions(); i++) { - datasetRelativeBlockPosition[i] = (shardPosition[i] * shardBlockSize[i]) + (shardRelativeBlockPosition[i]); - } + public int[] getBlockSize() { - return datasetRelativeBlockPosition; + return blockSize; } +// /** +// * Returns the number of blocks per dimension for each shard. +// * +// * @return the blocks per shard +// */ +// public int[] getBlocksPerShard() { +// +// final int[] shardSize = getShardSize(); +// final int nd = getNumDimensions(); +// final int[] blocksPerShard = new int[nd]; +// final int[] blockSize = getBlockSize(); +// for (int i = 0; i < nd; i++) +// blocksPerShard[i] = shardSize[i] / blockSize[i]; +// +// return blocksPerShard; +// } + /** - * Returns the number of shards per dimension for the dataset. + * Returns the number of blocks per dimension for this dataset. * - * @return the size of the shard grid of a dataset + * @return blocks per dataset */ - public int[] getShardBlockGridSize() { - - final int nd = getNumDimensions(); - final int[] shardBlockGridSize = new int[nd]; - final int[] blockSize = getBlockSize(); - for (int i = 0; i < nd; i++) - shardBlockGridSize[i] = (int)(Math.ceil((double)getDimensions()[i] / blockSize[i])); - - return shardBlockGridSize; - } - - public Map> groupBlockPositions(final List blockPositions) { - - final TreeMap> map = new TreeMap<>(); - for (final long[] blockPos : blockPositions) { - Position shardPos = Position.wrap(getShardPositionForBlock(blockPos)); - if (!map.containsKey(shardPos)) { - map.put(shardPos, new ArrayList<>()); - } - map.get(shardPos).add(blockPos); - } + public long[] blocksPerDataset() { - return map; + return IntStream.range(0, getNumDimensions()) + .mapToLong(i -> (long)Math.ceil((double)getDimensions()[i] / getBlockSize()[i])) + .toArray(); } - public Map>> groupBlocks(final List> blocks) { - - // figure out how to re-use groupBlockPositions here? - final TreeMap>> map = new TreeMap<>(); - for (final DataBlock block : blocks) { - Position shardPos = Position.wrap(getShardPositionForBlock(block.getGridPosition())); - if (!map.containsKey(shardPos)) { - map.put(shardPos, new ArrayList<>()); - } - map.get(shardPos).add(block); - } - - return map; - } +// /** +// * Returns the number of shards per dimension for this dataset. +// * +// * @return shards per dataset +// */ +// public long[] shardsPerDataset() { +// +// return IntStream.range(0, getNumDimensions()) +// .mapToLong(i -> (long)Math.ceil((double)getDimensions()[i] / getShardSize()[i])) +// .toArray(); +// } +// +// /** +// * Returns the total number of blocks in each shard. +// * +// * @return number of blocks in a shard +// */ +// public long getNumBlocksPerShard() { +// +// return Arrays.stream(getBlocksPerShard()).reduce(1, (x, y) -> x * y); +// } +// +// /** +// * Given a block's position relative to the dataset, returns the position of +// * the shard containing that block. +// * +// * @param blockGridPosition position of a block relative to the dataset +// * @return the position of the containing shard in the shard grid +// */ +// public long[] getShardPositionForBlock(final long... blockGridPosition) { +// +// final int[] blocksPerShard = getBlocksPerShard(); +// final long[] shardGridPosition = new long[blockGridPosition.length]; +// for (int i = 0; i < shardGridPosition.length; i++) { +// shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / blocksPerShard[i]); +// } +// +// return shardGridPosition; +// } +// +// /** +// * Given a {@code datasetRelativeBlockPosition} returns the position +// * relative the the shard at position {@code shardPosition}. +// * +// * @param shardPosition position of the shard +// * @param datasetRelativeBlockPosition position of the block relative to the dataset +// * @return position of the block relative to the shard +// */ +// public int[] getShardRelativeBlockPosition(final long[] shardPosition, final long[] datasetRelativeBlockPosition) { +// +// final long[] shardPos = getShardPositionForBlock(datasetRelativeBlockPosition); +// if (!Arrays.equals(shardPosition, shardPos)) +// return null; +// +// final int[] shardSize = getBlocksPerShard(); +// final int[] shardRelativeBlockPosition = new int[shardSize.length]; +// for (int i = 0; i < shardSize.length; i++) { +// shardRelativeBlockPosition[i] = (int)(datasetRelativeBlockPosition[i] % shardSize[i]); +// } +// return shardRelativeBlockPosition; +// } +// +// /** +// * Given a {@code shardRelativeBlockPosition} relative to the shard at +// * position {@code shardPosition}, returns the block' position relative the +// * dataset. +// * +// * @param shardPosition position of the shard +// * @param shardRelativeBlockPosition position of the block relative to the shard +// * @return position of the block relative to the dataset +// */ +// public long[] getBlockPositionFromShardPosition(final long[] shardPosition, final int[] shardRelativeBlockPosition) { +// +// final int[] shardBlockSize = getBlocksPerShard(); +// final long[] datasetRelativeBlockPosition = new long[getNumDimensions()]; +// for (int i = 0; i < getNumDimensions(); i++) { +// datasetRelativeBlockPosition[i] = (shardPosition[i] * shardBlockSize[i]) + (shardRelativeBlockPosition[i]); +// } +// +// return datasetRelativeBlockPosition; +// } +// +// /** +// * Returns the number of shards per dimension for the dataset. +// * +// * @return the size of the shard grid of a dataset +// */ +// public int[] getShardBlockGridSize() { +// +// final int nd = getNumDimensions(); +// final int[] shardBlockGridSize = new int[nd]; +// final int[] blockSize = getBlockSize(); +// for (int i = 0; i < nd; i++) +// shardBlockGridSize[i] = (int)(Math.ceil((double)getDimensions()[i] / blockSize[i])); +// +// return shardBlockGridSize; +// } +// +// public Map> groupBlockPositions(final List blockPositions) { +// +// final TreeMap> map = new TreeMap<>(); +// for (final long[] blockPos : blockPositions) { +// Position shardPos = Position.wrap(getShardPositionForBlock(blockPos)); +// if (!map.containsKey(shardPos)) { +// map.put(shardPos, new ArrayList<>()); +// } +// map.get(shardPos).add(blockPos); +// } +// +// return map; +// } +// +// public Map>> groupBlocks(final List> blocks) { +// +// // figure out how to re-use groupBlockPositions here? +// final TreeMap>> map = new TreeMap<>(); +// for (final DataBlock block : blocks) { +// Position shardPos = Position.wrap(getShardPositionForBlock(block.getGridPosition())); +// if (!map.containsKey(shardPos)) { +// map.put(shardPos, new ArrayList<>()); +// } +// map.get(shardPos).add(block); +// } +// +// return map; +// } public boolean isSharded() { @@ -412,9 +437,7 @@ public DataType getDataType() { */ public DatasetAccess getDatasetAccess() { - return ShardedDatasetAccess.create(getDataType(), - new int[] {24, 24, 24}, - blockCodecInfo, dataCodecInfos); + return (DatasetAccess)access; } public HashMap asMap() { @@ -454,7 +477,6 @@ public static class DatasetAttributesAdapter implements JsonSerializer new DatasetAttributes(dimensions, new int[]{1, 1, 1}, new int[]{1, 0, -1}, dataType, null)); - assertTrue(ex0.getMessage().contains("<= 0")); + () -> new DatasetAttributes(dimensions, new int[]{1, 1, 1}, new int[]{1, 0, -1}, dataType)); + assertTrue(ex0.getMessage().contains("negative")); // Different number of dimensions IllegalArgumentException ex1 = assertThrows( IllegalArgumentException.class, - () -> new DatasetAttributes(dimensions, new int[]{64, 64}, new int[]{32, 32, 32}, dataType, null)); - assertTrue(ex1.getMessage().contains("must equal number of dimensions")); + () -> new DatasetAttributes(dimensions, new int[]{64, 64}, new int[]{32, 32, 32}, dataType)); + assertTrue(ex1.getMessage().contains("different length")); // Shard size smaller than block size IllegalArgumentException ex2 = assertThrows( IllegalArgumentException.class, - () -> new DatasetAttributes(dimensions, new int[]{32, 64, 64}, new int[]{64, 64, 64}, dataType, null)); - assertTrue(ex2.getMessage().contains("is larger than the block size")); + () -> new DatasetAttributes(dimensions, new int[]{32, 64, 64}, new int[]{64, 64, 64}, dataType)); + assertTrue(ex2.getMessage().contains("is smaller than previous")); // Shard size not a multiple of block size IllegalArgumentException ex3 = assertThrows( IllegalArgumentException.class, - () -> new DatasetAttributes(dimensions, new int[]{100, 100, 100}, new int[]{64, 64, 64}, dataType, null)); - assertTrue(ex3.getMessage().contains("not a multiple of the block size")); + () -> new DatasetAttributes(dimensions, new int[]{100, 100, 100}, new int[]{64, 64, 64}, dataType)); + assertTrue(ex3.getMessage().contains("not a multiple of previous level")); // Multiple violations - shard smaller than block in one dimension IllegalArgumentException ex4 = assertThrows( IllegalArgumentException.class, - () -> new DatasetAttributes(dimensions, new int[]{128, 32, 128}, new int[]{64, 64, 64}, dataType, null)); - assertTrue(ex4.getMessage().contains("is larger than the block size")); + () -> new DatasetAttributes(dimensions, new int[]{128, 32, 128}, new int[]{64, 64, 64}, dataType)); + assertTrue(ex4.getMessage().contains("is smaller than previous")); // Edge case - shard size of 0 assertThrows( IllegalArgumentException.class, - () -> new DatasetAttributes(dimensions, new int[]{0, 64, 64}, new int[]{64, 64, 64}, dataType, null)); + () -> new DatasetAttributes(dimensions, new int[]{0, 64, 64}, new int[]{64, 64, 64}, dataType)); } - @Test - public void testShardProperties() { - - final long[] arraySize = new long[]{16, 16}; - final int[] shardSize = new int[]{16, 16}; - final long[] shardPosition = new long[]{1, 1}; - final int[] blkSize = new int[]{4, 4}; - - final DatasetAttributes dsetAttrs = new DatasetAttributes( - arraySize, - shardSize, - blkSize, - DataType.UINT8, - new ShardingCodec( - blkSize, - new CodecInfo[]{new N5BlockCodecInfo()}, - new DeterministicSizeCodecInfo[]{new RawBlockCodecInfo()}, - IndexLocation.END - ) - ); - - final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); - - assertArrayEquals(new int[]{4, 4}, shard.getBlockGridSize()); - - assertArrayEquals(new long[]{0, 0}, dsetAttrs.getShardPositionForBlock(0, 0)); - assertArrayEquals(new long[]{1, 1}, dsetAttrs.getShardPositionForBlock(5, 5)); - assertArrayEquals(new long[]{1, 0}, dsetAttrs.getShardPositionForBlock(5, 0)); - assertArrayEquals(new long[]{0, 1}, dsetAttrs.getShardPositionForBlock(0, 5)); - - assertArrayEquals(new int[]{0, 0}, shard.getRelativeBlockPosition(4, 4)); - assertArrayEquals(new int[]{1, 1}, shard.getRelativeBlockPosition(5, 5)); - assertArrayEquals(new int[]{2, 2}, shard.getRelativeBlockPosition(6, 6)); - assertArrayEquals(new int[]{3, 3}, shard.getRelativeBlockPosition(7, 7)); - } - - @Test - public void testShardGrouping() { - - final long[] arraySize = new long[]{8, 12}; - final int[] shardSize = new int[]{4, 6}; - final int[] blkSize = new int[]{2, 3}; - - final DatasetAttributes attrs = new DatasetAttributes( - arraySize, - shardSize, - blkSize, - DataType.UINT8, - new ShardingCodec( - blkSize, - new CodecInfo[]{ new N5BlockCodecInfo() }, - new DeterministicSizeCodecInfo[]{new RawBlockCodecInfo()}, - IndexLocation.END - ) - ); - - List blockPositions = blockPositions(attrs).collect(Collectors.toList()); - final Map> result = attrs.groupBlockPositions(blockPositions); - - // there are four shards in this image - assertEquals(4, result.size()); - - // there are four blocks per shard in this image - result.values().forEach(x -> assertEquals(4, x.size())); - } - - private static Stream blockPositions( final DatasetAttributes attrs ) { - - final int nd = attrs.getNumDimensions(); - final int[] blocksPerShard = attrs.getBlocksPerShard(); - return toStream( new GridIterator(attrs.shardsPerDataset())) - .flatMap( shardPosition -> { - final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]); - return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); - }); - } +// @Test +// public void testShardProperties() { +// +// final long[] arraySize = new long[]{16, 16}; +// final int[] shardSize = new int[]{16, 16}; +// final long[] shardPosition = new long[]{1, 1}; +// final int[] blkSize = new int[]{4, 4}; +// +// final DatasetAttributes dsetAttrs = new DatasetAttributes( +// arraySize, +// shardSize, +// blkSize, +// DataType.UINT8, +// new ShardingCodec( +// blkSize, +// new CodecInfo[]{new N5BlockCodecInfo()}, +// new DeterministicSizeCodecInfo[]{new RawBlockCodecInfo()}, +// IndexLocation.END +// ) +// ); +// +// final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); +// +// assertArrayEquals(new int[]{4, 4}, shard.getBlockGridSize()); +// +// assertArrayEquals(new long[]{0, 0}, dsetAttrs.getShardPositionForBlock(0, 0)); +// assertArrayEquals(new long[]{1, 1}, dsetAttrs.getShardPositionForBlock(5, 5)); +// assertArrayEquals(new long[]{1, 0}, dsetAttrs.getShardPositionForBlock(5, 0)); +// assertArrayEquals(new long[]{0, 1}, dsetAttrs.getShardPositionForBlock(0, 5)); +// +// assertArrayEquals(new int[]{0, 0}, shard.getRelativeBlockPosition(4, 4)); +// assertArrayEquals(new int[]{1, 1}, shard.getRelativeBlockPosition(5, 5)); +// assertArrayEquals(new int[]{2, 2}, shard.getRelativeBlockPosition(6, 6)); +// assertArrayEquals(new int[]{3, 3}, shard.getRelativeBlockPosition(7, 7)); +// } + +// @Test +// public void testShardGrouping() { +// +// final long[] arraySize = new long[]{8, 12}; +// final int[] shardSize = new int[]{4, 6}; +// final int[] blkSize = new int[]{2, 3}; +// +// final DatasetAttributes attrs = new DatasetAttributes( +// arraySize, +// shardSize, +// blkSize, +// DataType.UINT8, +// new ShardingCodec( +// blkSize, +// new CodecInfo[]{ new N5BlockCodecInfo() }, +// new DeterministicSizeCodecInfo[]{new RawBlockCodecInfo()}, +// IndexLocation.END +// ) +// ); +// +// List blockPositions = blockPositions(attrs).collect(Collectors.toList()); +// final Map> result = attrs.groupBlockPositions(blockPositions); +// +// // there are four shards in this image +// assertEquals(4, result.size()); +// +// // there are four blocks per shard in this image +// result.values().forEach(x -> assertEquals(4, x.size())); +// } + +// private static Stream blockPositions( final DatasetAttributes attrs ) { +// +// final int nd = attrs.getNumDimensions(); +// final int[] blocksPerShard = attrs.getBlocksPerShard(); +// return toStream( new GridIterator(attrs.shardsPerDataset())) +// .flatMap( shardPosition -> { +// final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]); +// return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); +// }); +// } private static Stream toStream( final Iterator it ) { - return StreamSupport.stream( Spliterators.spliteratorUnknownSize( - it, Spliterator.ORDERED), - false); + + return StreamSupport.stream( + Spliterators.spliteratorUnknownSize(it, Spliterator.ORDERED), + false); } } \ No newline at end of file From 3d97e94131531eead8604acb21af6ac89fe25401 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Sep 2025 15:50:17 -0400 Subject: [PATCH 348/423] wip: tmp comment shard related things --- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 25 +- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 2 +- .../org/janelia/saalfeldlab/n5/N5Writer.java | 2 +- .../saalfeldlab/n5/shard/AbstractShard.java | 78 +- .../n5/shard/BlockAsShardCodec.java | 182 ++--- .../saalfeldlab/n5/shard/InMemoryShard.java | 148 ++-- .../janelia/saalfeldlab/n5/shard/Shard.java | 413 +++++------ .../saalfeldlab/n5/shard/ShardIndex.java | 692 +++++++++--------- .../saalfeldlab/n5/shard/ShardingCodec.java | 337 ++++----- .../saalfeldlab/n5/shard/VirtualShard.java | 210 +++--- .../saalfeldlab/n5/shardstuff/ShardIndex.java | 2 +- .../n5/shardstuff/ShardedDatasetAccess.java | 2 +- .../saalfeldlab/n5/AbstractN5Test.java | 79 +- .../saalfeldlab/n5/codec/BlockCodecTests.java | 4 - .../saalfeldlab/n5/demo/BlockIterators.java | 123 ++-- .../saalfeldlab/n5/shard/ShardIndexTest.java | 1 - .../saalfeldlab/n5/shard/ShardTest.java | 1 - 17 files changed, 1148 insertions(+), 1153 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index b3e0ec810..2d06d5b7d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -223,7 +223,6 @@ default void writeBlocks( final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { - // TODO // if (datasetAttributes.isSharded()) { // /* Group blocks by shard index */ // final Map>> shardBlockMap = datasetAttributes.groupBlocks( @@ -252,8 +251,6 @@ default void writeBlocks( // } } - - @Override default void writeBlock( final String path, @@ -278,16 +275,18 @@ default void writeShard( final DatasetAttributes datasetAttributes, final Shard shard) throws N5Exception { - final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shard.getGridPosition()); - try ( - final LockedChannel channel = getKeyValueAccess().lockForWriting(shardPath); - final OutputStream shardOut = channel.newOutputStream() - ) { - shard.createReadData().writeTo(shardOut); - } catch (final IOException | UncheckedIOException e) { - throw new N5IOException( - "Failed to write shard " + Arrays.toString(shard.getGridPosition()) + " into dataset " + path, e); - } + // TODO + +// final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shard.getGridPosition()); +// try ( +// final LockedChannel channel = getKeyValueAccess().lockForWriting(shardPath); +// final OutputStream shardOut = channel.newOutputStream() +// ) { +// shard.createReadData().writeTo(shardOut); +// } catch (final IOException | UncheckedIOException e) { +// throw new N5IOException( +// "Failed to write shard " + Arrays.toString(shard.getGridPosition()) + " into dataset " + path, e); +// } } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index fc514a2c8..125002ff6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -68,7 +68,7 @@ static Gson registerGson(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, DatasetAttributes.getJsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBlockCodecInfo.byteOrderAdapter); - gsonBuilder.registerTypeHierarchyAdapter(ShardingCodec.IndexLocation.class, ShardingCodec.indexLocationAdapter); +// gsonBuilder.registerTypeHierarchyAdapter(ShardingCodec.IndexLocation.class, ShardingCodec.indexLocationAdapter); gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.disableHtmlEscaping(); return gsonBuilder.create(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index d0afacfc2..cf85bce56 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -248,7 +248,7 @@ default void createDataset( System.arraycopy(codecs, 0, dataCodecs, 0, dataCodecs.length); } - createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, blockSize, dataType, blockCodecInfo, dataCodecs)); + createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, blockCodecInfo, dataCodecs)); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java index 9aa81b740..7d7d4511c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java @@ -5,43 +5,43 @@ public abstract class AbstractShard implements Shard { - protected final DatasetAttributes datasetAttributes; - - protected ShardIndex index; - - private final long[] gridPosition; - - public AbstractShard( - final DatasetAttributes datasetAttributes, - final long[] gridPosition, - final ShardIndex index) { - - this.datasetAttributes = datasetAttributes; - this.gridPosition = gridPosition; - this.index = index; - } - - @Override - public DatasetAttributes getDatasetAttributes() { - - return datasetAttributes; - } - - @Override - public int[] getSize() { - - return getDatasetAttributes().getShardSize(); - } - - @Override - public int[] getBlockSize() { - - return datasetAttributes.getBlockSize(); - } - - @Override - public long[] getGridPosition() { - - return gridPosition; - } +// protected final DatasetAttributes datasetAttributes; +// +// protected ShardIndex index; +// +// private final long[] gridPosition; +// +// public AbstractShard( +// final DatasetAttributes datasetAttributes, +// final long[] gridPosition, +// final ShardIndex index) { +// +// this.datasetAttributes = datasetAttributes; +// this.gridPosition = gridPosition; +// this.index = index; +// } +// +// @Override +// public DatasetAttributes getDatasetAttributes() { +// +// return datasetAttributes; +// } +// +// @Override +// public int[] getSize() { +// +// return getDatasetAttributes().getShardSize(); +// } +// +// @Override +// public int[] getBlockSize() { +// +// return datasetAttributes.getBlockSize(); +// } +// +// @Override +// public long[] getGridPosition() { +// +// return gridPosition; +// } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java index d8e9fe86a..c8e2d136d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java @@ -17,95 +17,95 @@ @NameConfig.Serialize(false) public class BlockAsShardCodec extends ShardingCodec { - private static final LongArrayDataBlock BLOCK_AS_SHARD_INDEX = new LongArrayDataBlock(new int[0], new long[0], new long[]{0, -1}); - private static final BlockCodec BLOCK_AS_SHARD_BLOCK_SERIALIZER = new BlockCodec() { - - @Override public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { - - return ReadData.from(new byte[0]); - } - - @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException { - - return BLOCK_AS_SHARD_INDEX; - } - }; - - private static final RawBlockCodecInfo VIRTUAL_SHARD_INDEX_CODEC = new RawBlockCodecInfo() { - - public BlockCodec create(DatasetAttributes attributes, DataCodec... dataCodec) { - - return BLOCK_AS_SHARD_BLOCK_SERIALIZER; - } - }; - private static final DataCodecInfo[] EMPTY_SHARD_CODECS = new DataCodecInfo[0]; - - private static final DeterministicSizeCodecInfo[] NO_OP_INDEX_CODECS = new DeterministicSizeCodecInfo[0]; - private static final IndexCodecAdapter BLOCK_AS_SHARD_INDEX_CODEC_ADAPTER = new IndexCodecAdapter(VIRTUAL_SHARD_INDEX_CODEC, NO_OP_INDEX_CODECS) { - - @Override public long encodedSize(long initialSize) { - - return 0; - } - }; - - final BlockCodecInfo datasetArrayCodec; - - public BlockAsShardCodec(BlockCodecInfo datasetArrayCodec) { - - super(null, EMPTY_SHARD_CODECS, BLOCK_AS_SHARD_INDEX_CODEC_ADAPTER, IndexLocation.START); - this.datasetArrayCodec = datasetArrayCodec; - } - - @Override - public ShardIndex createIndex(DatasetAttributes attributes) { - - return new ShardIndex(attributes.getBlocksPerShard(), getIndexLocation(), BLOCK_AS_SHARD_INDEX_CODEC_ADAPTER); - } - - @Override - public BlockCodecInfo getBlockCodecInfo() { - - return datasetArrayCodec; - } - - @Override - public long[] getKeyPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { - - return datasetArrayCodec.getKeyPositionForBlock(attributes, datablock); - } - - @Override - public long[] getKeyPositionForBlock(DatasetAttributes attributes, long... blockPosition) { - - return datasetArrayCodec.getKeyPositionForBlock(attributes, blockPosition); - } - - @Override - public BlockCodec create(DatasetAttributes attributes, DataCodec... codecs) { - - // TODO - return null; - -// dataBlockSerializer = datasetArrayCodec.create(attributes, codecs); -// return (BlockCodec)dataBlockSerializer; - } - - @Override - public long encodedSize(long size) { - - return datasetArrayCodec.encodedSize(size); - } - - @Override - public long decodedSize(long size) { - - return datasetArrayCodec.decodedSize(size); - } - - @Override - public String getType() { - //TODO Caleb: can we ensure this is never called? - return datasetArrayCodec.getType(); - } +// private static final LongArrayDataBlock BLOCK_AS_SHARD_INDEX = new LongArrayDataBlock(new int[0], new long[0], new long[]{0, -1}); +// private static final BlockCodec BLOCK_AS_SHARD_BLOCK_SERIALIZER = new BlockCodec() { +// +// @Override public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { +// +// return ReadData.from(new byte[0]); +// } +// +// @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException { +// +// return BLOCK_AS_SHARD_INDEX; +// } +// }; +// +// private static final RawBlockCodecInfo VIRTUAL_SHARD_INDEX_CODEC = new RawBlockCodecInfo() { +// +// public BlockCodec create(DatasetAttributes attributes, DataCodec... dataCodec) { +// +// return BLOCK_AS_SHARD_BLOCK_SERIALIZER; +// } +// }; +// private static final DataCodecInfo[] EMPTY_SHARD_CODECS = new DataCodecInfo[0]; +// +// private static final DeterministicSizeCodecInfo[] NO_OP_INDEX_CODECS = new DeterministicSizeCodecInfo[0]; +// private static final IndexCodecAdapter BLOCK_AS_SHARD_INDEX_CODEC_ADAPTER = new IndexCodecAdapter(VIRTUAL_SHARD_INDEX_CODEC, NO_OP_INDEX_CODECS) { +// +// @Override public long encodedSize(long initialSize) { +// +// return 0; +// } +// }; +// +// final BlockCodecInfo datasetArrayCodec; +// +// public BlockAsShardCodec(BlockCodecInfo datasetArrayCodec) { +// +// super(null, EMPTY_SHARD_CODECS, BLOCK_AS_SHARD_INDEX_CODEC_ADAPTER, IndexLocation.START); +// this.datasetArrayCodec = datasetArrayCodec; +// } +// +// @Override +// public ShardIndex createIndex(DatasetAttributes attributes) { +// +// return new ShardIndex(attributes.getBlocksPerShard(), getIndexLocation(), BLOCK_AS_SHARD_INDEX_CODEC_ADAPTER); +// } +// +// @Override +// public BlockCodecInfo getBlockCodecInfo() { +// +// return datasetArrayCodec; +// } +// +// @Override +// public long[] getKeyPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { +// +// return datasetArrayCodec.getKeyPositionForBlock(attributes, datablock); +// } +// +// @Override +// public long[] getKeyPositionForBlock(DatasetAttributes attributes, long... blockPosition) { +// +// return datasetArrayCodec.getKeyPositionForBlock(attributes, blockPosition); +// } +// +// @Override +// public BlockCodec create(DatasetAttributes attributes, DataCodec... codecs) { +// +// // TODO +// return null; +// +//// dataBlockSerializer = datasetArrayCodec.create(attributes, codecs); +//// return (BlockCodec)dataBlockSerializer; +// } +// +// @Override +// public long encodedSize(long size) { +// +// return datasetArrayCodec.encodedSize(size); +// } +// +// @Override +// public long decodedSize(long size) { +// +// return datasetArrayCodec.decodedSize(size); +// } +// +// @Override +// public String getType() { +// //TODO Caleb: can we ensure this is never called? +// return datasetArrayCodec.getType(); +// } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java index d9ff655d7..0e97124cf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java @@ -12,78 +12,78 @@ public class InMemoryShard extends AbstractShard { - /** Map {@link DataBlock#getGridPosition} as hashable {@link Position} to the block */ - private final Map> blocks; - - public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] shardPosition) { - - this(datasetAttributes, shardPosition, null); - } - - public InMemoryShard( - final DatasetAttributes datasetAttributes, - final long[] gridPosition, - ShardIndex index) { - - super(datasetAttributes, gridPosition, index); - blocks = new TreeMap<>(); - } - - private void storeBlock(DataBlock block) { - - blocks.put(Position.wrap(block.getGridPosition()), block); - } - - @Override - public DataBlock getBlock(int... blockGridPosition) { - - return blocks.get(Position.wrap(blockGridPosition)); - } - - /** - * Add the {@code block} to this shard. If the block is not contained in this shard, do not add it. - * - * @param block to add the shard - * @return whether the block was added - */ - public boolean addBlock(DataBlock block) { - - final long[] shardPositionForBlock = datasetAttributes.getShardPositionForBlock(block.getGridPosition()); - if (!Arrays.equals(shardPositionForBlock, getGridPosition())) - return false; - - storeBlock(block); - return true; - } - - @Override - public List> getBlocks() { - - return new ArrayList<>(blocks.values()); - } - - @Override - public ShardIndex getIndex() { - -// return index = index != null ? index : createIndex(); - - // TODO - return index; - } - - public static InMemoryShard fromShard(Shard shard) { - - if (shard == null) - return null; - - if (shard instanceof InMemoryShard) - return (InMemoryShard)shard; - - final InMemoryShard inMemoryShard = new InMemoryShard( - shard.getDatasetAttributes(), - shard.getGridPosition()); - - shard.forEach(inMemoryShard::addBlock); - return inMemoryShard; - } +// /** Map {@link DataBlock#getGridPosition} as hashable {@link Position} to the block */ +// private final Map> blocks; +// +// public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] shardPosition) { +// +// this(datasetAttributes, shardPosition, null); +// } +// +// public InMemoryShard( +// final DatasetAttributes datasetAttributes, +// final long[] gridPosition, +// ShardIndex index) { +// +// super(datasetAttributes, gridPosition, index); +// blocks = new TreeMap<>(); +// } +// +// private void storeBlock(DataBlock block) { +// +// blocks.put(Position.wrap(block.getGridPosition()), block); +// } +// +// @Override +// public DataBlock getBlock(int... blockGridPosition) { +// +// return blocks.get(Position.wrap(blockGridPosition)); +// } +// +// /** +// * Add the {@code block} to this shard. If the block is not contained in this shard, do not add it. +// * +// * @param block to add the shard +// * @return whether the block was added +// */ +// public boolean addBlock(DataBlock block) { +// +// final long[] shardPositionForBlock = datasetAttributes.getShardPositionForBlock(block.getGridPosition()); +// if (!Arrays.equals(shardPositionForBlock, getGridPosition())) +// return false; +// +// storeBlock(block); +// return true; +// } +// +// @Override +// public List> getBlocks() { +// +// return new ArrayList<>(blocks.values()); +// } +// +// @Override +// public ShardIndex getIndex() { +// +//// return index = index != null ? index : createIndex(); +// +// // TODO +// return index; +// } +// +// public static InMemoryShard fromShard(Shard shard) { +// +// if (shard == null) +// return null; +// +// if (shard instanceof InMemoryShard) +// return (InMemoryShard)shard; +// +// final InMemoryShard inMemoryShard = new InMemoryShard( +// shard.getDatasetAttributes(), +// shard.getGridPosition()); +// +// shard.forEach(inMemoryShard::addBlock); +// return inMemoryShard; +// } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java index c19b34335..e9972c590 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java @@ -16,213 +16,214 @@ import static org.janelia.saalfeldlab.n5.N5Exception.*; -public interface Shard extends Iterable> { - - /** - * Returns the number of blocks this shard contains along all dimensions. - * - * The size of a shard expected to be smaller than or equal to the spacing of the shard grid. The dimensionality of size is expected to be equal to the dimensionality of the - * dataset. Consistency is not enforced. - * - * @return size of the shard in units of blocks - */ - default int[] getBlockGridSize() { - - return getDatasetAttributes().getBlocksPerShard(); - } - - DatasetAttributes getDatasetAttributes(); - - /** - * Returns the size of shards in pixel units. - * - * @return shard size - */ - default int[] getSize() { - return getDatasetAttributes().getShardSize(); - } - - /** - * Returns the size of blocks in pixel units. - * - * @return block size - */ - default int[] getBlockSize() { - return getDatasetAttributes().getBlockSize(); - } - - /** - * Returns the position of this shard on the shard grid. - * - * The dimensionality of the grid position is expected to be equal to the dimensionality of the dataset. Consistency is not enforced. - * - * @return position on the shard grid - */ - long[] getGridPosition(); - - /** - * Given a {@code blockPosition} relative to the dataset, return its position relative to this - * shard. - * - * @param blockPositionInDataset dataset-relative block position - * @return the shard-relative block positiorelativeBlockPositionn - * @see {@link DatasetAttributes#getBlockPositionInShard(long[], long[])} - * @see {@link #getBlockPositionFromShardPosition(long[], long[])} - */ - default int[] getRelativeBlockPosition(long... datasetBlockPosition) { - - return getDatasetAttributes().getShardRelativeBlockPosition( - getGridPosition(), - datasetBlockPosition); - } - - /** - * Tests whether the block at the {@code relativeBlockPosition} (relative to - * this shard) exists. - *

      - * Avoids reading the block data, if possible. - * - * @return true of the block exists in this shard - */ - default boolean blockExists(int... relativeBlockPosition) { - - return getIndex().exists(relativeBlockPosition); - } - - /** - * Retrieve the DataBlock at {@code blockGridPosition} relative to this shard if it exists and is - * a member of this shard. - *

      - * If needed, use {@code getRelativeBlockPosition} to convert block positions relative to the dataset into - * positions relative to this shard. - * - * @param blockGridPosition position of the desired block relative to this shard - * @return the block if it exists and is part of this shard, otherwise null - */ - DataBlock getBlock(int... blockGridPosition); - - default Iterator> iterator() { - - return new DataBlockIterator<>(this); - } - - default int getNumBlocks() { - - return Arrays.stream(getBlockGridSize()).reduce(1, (x, y) -> x * y); - } - - default List> getBlocks() { - - final List> blocks = new ArrayList<>(); - for (DataBlock block : this) { - blocks.add(block); - } - return blocks; - } - - - /** - * @return the ShardIndex for this shard, or a new ShardIndex if the Shard is non-existent - */ - ShardIndex getIndex(); - - default ReadData createReadData() throws N5IOException { - - final DatasetAttributes datasetAttributes = getDatasetAttributes(); - DatasetAccess access = datasetAttributes.getDatasetAccess(); - - // TODO make a PositionValueAccess that just stores the ReadData - // and returns it? - for( DataBlock b : this ) { - access.writeBlock(null, b); - } - - return null; - -// ShardingCodec shardingCodec = datasetAttributes.getShardingCodec(); -// final BlockCodec blockCodecInfo = shardingCodec.getBlockCodec(); - -// final ShardIndex index = createIndex(); -// final long indexSize = index.numBytes(); -// long blocksStartBytes = index.getLocation() == ShardingCodec.IndexLocation.START ? indexSize : 0; -// final AtomicLong blockOffset = new AtomicLong(blocksStartBytes); -// -// /* isIndexEmpty is true when writing to a non-sharded dataset through the Shard API. */ -// final boolean isIndexEmpty = indexSize == 0; -// if (index.getLocation() == ShardingCodec.IndexLocation.END || isIndexEmpty) { -// return ReadData.from(out -> { -// try (final CountingOutputStream countOut = new CountingOutputStream(out)) { -// long prevCount = 0; -// for (DataBlock block : getBlocks()) { -// blockCodecInfo.encode(block).writeTo(countOut); -// final int[] blockPosition = getRelativeBlockPosition(block.getGridPosition()); -// final long curCount = countOut.getByteCount(); -// final long blockWrittenSize = curCount - prevCount; -// prevCount = curCount; -// if (!isIndexEmpty) -// synchronized (index) { -// index.set(blockOffset.getAndAdd(blockWrittenSize), blockWrittenSize, blockPosition); -// } -// } -// if (!isIndexEmpty) -// synchronized (index) { -// ShardIndex.write(out, index); -// } -// } -// }); -// } else { -// final ArrayList blocksData = new ArrayList<>(); -// for (DataBlock dataBlock : getBlocks()) { -// ReadData readDataBlock = ReadData.from(out -> blockCodecInfo.encode(dataBlock).writeTo(out)); -// blocksData.add(readDataBlock); -// final long length = readDataBlock.length(); -// synchronized (index) { -// index.set(blockOffset.getAndAdd(length), length, getRelativeBlockPosition(dataBlock.getGridPosition())); -// } +public interface Shard { +//extends Iterable> { + +// /** +// * Returns the number of blocks this shard contains along all dimensions. +// * +// * The size of a shard expected to be smaller than or equal to the spacing of the shard grid. The dimensionality of size is expected to be equal to the dimensionality of the +// * dataset. Consistency is not enforced. +// * +// * @return size of the shard in units of blocks +// */ +// default int[] getBlockGridSize() { +// +// return getDatasetAttributes().getBlocksPerShard(); +// } +// +// DatasetAttributes getDatasetAttributes(); +// +// /** +// * Returns the size of shards in pixel units. +// * +// * @return shard size +// */ +// default int[] getSize() { +// return getDatasetAttributes().getShardSize(); +// } +// +// /** +// * Returns the size of blocks in pixel units. +// * +// * @return block size +// */ +// default int[] getBlockSize() { +// return getDatasetAttributes().getBlockSize(); +// } +// +// /** +// * Returns the position of this shard on the shard grid. +// * +// * The dimensionality of the grid position is expected to be equal to the dimensionality of the dataset. Consistency is not enforced. +// * +// * @return position on the shard grid +// */ +// long[] getGridPosition(); +// +// /** +// * Given a {@code blockPosition} relative to the dataset, return its position relative to this +// * shard. +// * +// * @param blockPositionInDataset dataset-relative block position +// * @return the shard-relative block positiorelativeBlockPositionn +// * @see {@link DatasetAttributes#getBlockPositionInShard(long[], long[])} +// * @see {@link #getBlockPositionFromShardPosition(long[], long[])} +// */ +// default int[] getRelativeBlockPosition(long... datasetBlockPosition) { +// +// return getDatasetAttributes().getShardRelativeBlockPosition( +// getGridPosition(), +// datasetBlockPosition); +// } +// +// /** +// * Tests whether the block at the {@code relativeBlockPosition} (relative to +// * this shard) exists. +// *

      +// * Avoids reading the block data, if possible. +// * +// * @return true of the block exists in this shard +// */ +// default boolean blockExists(int... relativeBlockPosition) { +// +// return getIndex().exists(relativeBlockPosition); +// } +// +// /** +// * Retrieve the DataBlock at {@code blockGridPosition} relative to this shard if it exists and is +// * a member of this shard. +// *

      +// * If needed, use {@code getRelativeBlockPosition} to convert block positions relative to the dataset into +// * positions relative to this shard. +// * +// * @param blockGridPosition position of the desired block relative to this shard +// * @return the block if it exists and is part of this shard, otherwise null +// */ +// DataBlock getBlock(int... blockGridPosition); +// +// default Iterator> iterator() { +// +// return new DataBlockIterator<>(this); +// } +// +// default int getNumBlocks() { +// +// return Arrays.stream(getBlockGridSize()).reduce(1, (x, y) -> x * y); +// } +// +// default List> getBlocks() { +// +// final List> blocks = new ArrayList<>(); +// for (DataBlock block : this) { +// blocks.add(block); +// } +// return blocks; +// } +// +// +// /** +// * @return the ShardIndex for this shard, or a new ShardIndex if the Shard is non-existent +// */ +// ShardIndex getIndex(); +// +// default ReadData createReadData() throws N5IOException { +// +// final DatasetAttributes datasetAttributes = getDatasetAttributes(); +// DatasetAccess access = datasetAttributes.getDatasetAccess(); +// +// // TODO make a PositionValueAccess that just stores the ReadData +// // and returns it? +// for( DataBlock b : this ) { +// access.writeBlock(null, b); +// } +// +// return null; +// +//// ShardingCodec shardingCodec = datasetAttributes.getShardingCodec(); +//// final BlockCodec blockCodecInfo = shardingCodec.getBlockCodec(); +// +//// final ShardIndex index = createIndex(); +//// final long indexSize = index.numBytes(); +//// long blocksStartBytes = index.getLocation() == ShardingCodec.IndexLocation.START ? indexSize : 0; +//// final AtomicLong blockOffset = new AtomicLong(blocksStartBytes); +//// +//// /* isIndexEmpty is true when writing to a non-sharded dataset through the Shard API. */ +//// final boolean isIndexEmpty = indexSize == 0; +//// if (index.getLocation() == ShardingCodec.IndexLocation.END || isIndexEmpty) { +//// return ReadData.from(out -> { +//// try (final CountingOutputStream countOut = new CountingOutputStream(out)) { +//// long prevCount = 0; +//// for (DataBlock block : getBlocks()) { +//// blockCodecInfo.encode(block).writeTo(countOut); +//// final int[] blockPosition = getRelativeBlockPosition(block.getGridPosition()); +//// final long curCount = countOut.getByteCount(); +//// final long blockWrittenSize = curCount - prevCount; +//// prevCount = curCount; +//// if (!isIndexEmpty) +//// synchronized (index) { +//// index.set(blockOffset.getAndAdd(blockWrittenSize), blockWrittenSize, blockPosition); +//// } +//// } +//// if (!isIndexEmpty) +//// synchronized (index) { +//// ShardIndex.write(out, index); +//// } +//// } +//// }); +//// } else { +//// final ArrayList blocksData = new ArrayList<>(); +//// for (DataBlock dataBlock : getBlocks()) { +//// ReadData readDataBlock = ReadData.from(out -> blockCodecInfo.encode(dataBlock).writeTo(out)); +//// blocksData.add(readDataBlock); +//// final long length = readDataBlock.length(); +//// synchronized (index) { +//// index.set(blockOffset.getAndAdd(length), length, getRelativeBlockPosition(dataBlock.getGridPosition())); +//// } +//// } +//// return ReadData.from(out -> { +//// ShardIndex.write(out, index); +//// for (ReadData blockData : blocksData) { +//// blockData.writeTo(out); +//// } +//// }); +//// } +// } +// +// class DataBlockIterator implements Iterator> { +// +// private final GridIterator it; +// private final Shard shard; +// private final ShardIndex index; +// private final DatasetAttributes attributes; +// private int blockIndex = 0; +// +// public DataBlockIterator(final Shard shard) { +// +// this.shard = shard; +// this.index = shard.getIndex(); +// this.attributes = shard.getDatasetAttributes(); +// this.blockIndex = 0; +// it = new GridIterator(shard.getBlockGridSize()); +// } +// +// @Override +// public boolean hasNext() { +// +// for (int i = blockIndex; i < attributes.getNumBlocksPerShard(); i++) { +// if (index.exists(i)) +// return true; // } -// return ReadData.from(out -> { -// ShardIndex.write(out, index); -// for (ReadData blockData : blocksData) { -// blockData.writeTo(out); -// } -// }); +// return false; // } - } - - class DataBlockIterator implements Iterator> { - - private final GridIterator it; - private final Shard shard; - private final ShardIndex index; - private final DatasetAttributes attributes; - private int blockIndex = 0; - - public DataBlockIterator(final Shard shard) { - - this.shard = shard; - this.index = shard.getIndex(); - this.attributes = shard.getDatasetAttributes(); - this.blockIndex = 0; - it = new GridIterator(shard.getBlockGridSize()); - } - - @Override - public boolean hasNext() { - - for (int i = blockIndex; i < attributes.getNumBlocksPerShard(); i++) { - if (index.exists(i)) - return true; - } - return false; - } - - @Override - public DataBlock next() { - while (!index.exists(blockIndex++)) - it.fwd(); - - return shard.getBlock(it.nextInt()); - } - } +// +// @Override +// public DataBlock next() { +// while (!index.exists(blockIndex++)) +// it.fwd(); +// +// return shard.getBlock(it.nextInt()); +// } +// } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index df5745268..09eae5498 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -5,7 +5,6 @@ import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import java.util.Arrays; import java.util.stream.IntStream; @@ -29,349 +28,350 @@ * "https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/index.html#binary-shard-format">The * Zarr V3 specification for the binary shard format */ -public class ShardIndex extends LongArrayDataBlock { - - /** - * Special value indicating an empty block entry in the index. - * Used for both offset and length when a block doesn't exist. - */ - public static final long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; - private static final int BYTES_PER_LONG = 8; - private static final int LONGS_PER_BLOCK = 2; - private static final long[] DUMMY_GRID_POSITION = null; - - private final IndexLocation location; - private final ShardIndexAttributes indexAttributes; - private final IndexCodecAdapter indexCodexAdapter; - - /** - * Creates a ShardIndex with specified data. - * - * @param shardBlockGridSize the dimensions of the block grid within the shard - * @param data the raw index data containing offsets and lengths - * @param location where the index is stored (START or END of shard) - * @param indexCodecAdapter data object for Shard Index codecs. - */ - public ShardIndex(int[] shardBlockGridSize, long[] data, IndexLocation location, final IndexCodecAdapter indexCodecAdapter) { - - // prepend the number of longs per block to the shard block grid size - super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, data); - this.indexCodexAdapter = indexCodecAdapter; - this.location = location; - this.indexAttributes = new ShardIndexAttributes(this); - } - - /** - * Creates an empty ShardIndex at the specified location. - * - * @param shardBlockGridSize the dimensions of the block grid within the shard - * @param location where the index is stored (START or END of shard) - * @param indexCodecAdapter data object for idnex codecs - */ - public ShardIndex(int[] shardBlockGridSize, IndexLocation location, final IndexCodecAdapter indexCodecAdapter) { - - this(shardBlockGridSize, emptyIndexData(shardBlockGridSize), location, indexCodecAdapter); - } - - /** - * Creates an empty ShardIndex at the default location (END). - * - * @param shardBlockGridSize the dimensions of the block grid within the shard - * @param indexCodecAdapter data object for idnex codecs - */ - public ShardIndex(int[] shardBlockGridSize, final IndexCodecAdapter indexCodecAdapter) { - - this(shardBlockGridSize, IndexLocation.END, indexCodecAdapter); - } - - /** - * Creates an empty ShardIndex at the specified location. - * - * @param shardBlockGridSize the dimensions of the block grid within the shard - * @param location where the index is stored (START or END of shard) - * @param blockCodecInfo blockCodecInfo for the IndexCodecAdapter - */ - public ShardIndex(int[] shardBlockGridSize, IndexLocation location, final BlockCodecInfo blockCodecInfo) { - - this(shardBlockGridSize, location, new IndexCodecAdapter(blockCodecInfo)); - } - - /** - * Creates an empty ShardIndex at the default location (END). - * - * @param shardBlockGridSize the dimensions of the block grid within the shard - * @param blockCodecInfo blockCodecInfo for the IndexCodecAdapter - */ - public ShardIndex(int[] shardBlockGridSize, final BlockCodecInfo blockCodecInfo) { - - this(shardBlockGridSize, IndexLocation.END, blockCodecInfo); - } - - public IndexCodecAdapter getIndexCodexAdapter() { - - return indexCodexAdapter; - } - - /** - * Checks existence of the block at a given grid position. - * - * @param gridPosition the n-dimensional position of the block in the shard grid - * @return true if the block exists, false otherwise - */ - public boolean exists(int[] gridPosition) { - - return getOffset(gridPosition) != EMPTY_INDEX_NBYTES || - getNumBytes(gridPosition) != EMPTY_INDEX_NBYTES; - } - - /** - * Checks existence of the block at a given flat index. - * - * @param index the flattened index of the block - * @return true if the block exists, false otherwise - */ - public boolean exists(int index) { - - return data[index * 2] != EMPTY_INDEX_NBYTES || - data[index * 2 + 1] != EMPTY_INDEX_NBYTES; - } - - /** - * Gets the total number of blocks that can be stored in this index. - * - * @return the total number of blocks in the shard grid - */ - public int getNumBlocks() { - - /* getSize() is the number of data entries; each block takes 2 entries (offset and length) - * so the product of the dimension sizes, divided by 2, is the number of blocks. */ - return Arrays.stream(getSize()).reduce(1, (x, y) -> x * y) / 2; - } - - /** - * Checks if the index is completely empty (no blocks exist). - * - * @return true if no blocks exist in the index, false otherwise - */ - public boolean isEmpty() { - - return !IntStream.range(0, getNumBlocks()).anyMatch(this::exists); - } - - /** - * Gets the location of this index within the shard. - * - * @return the index location (START or END) - */ - public IndexLocation getLocation() { - - return location; - } - - /** - * Gets the offset in this shard in bytes for the block at a grid position. - * - * @param gridPosition the n-dimensional position of the block in the shard grid - * @return the offset in bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist - */ - public long getOffset(int... gridPosition) { - - return data[getOffsetIndex(gridPosition)]; - } - - /** - * Gets the offset in this shard in bytes for the block at a given index. - * - * @param index the flattened index of the block - * @return the offset in bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist - */ - public long getOffsetByBlockIndex(int index) { - - return data[index * 2]; - } - - /** - * Gets the number of bytes for the block at a grid position. - * - * @param gridPosition the n-dimensional position of the block in the shard grid - * @return the number of bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist - */ - public long getNumBytes(int... gridPosition) { - - return data[getNumBytesIndex(gridPosition)]; - } - - /** - * Gets the number of bytes for the block at a given index. - * - * @param index the flattened index of the block - * @return the number of bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist - */ - public long getNumBytesByBlockIndex(int index) { - - return data[index * 2 + 1]; - } - - /** - * Sets the offset and number of bytes for a block at the specified position. - * - * @param offset the byte offset of the block in the shard - * @param nbytes the number of bytes the block occupies - * @param gridPosition the n-dimensional position of the block in the shard grid - */ - public void set(long offset, long nbytes, int[] gridPosition) { - - final int i = getOffsetIndex(gridPosition); - data[i] = offset; - data[i + 1] = nbytes; - } - - /** - * Marks a block position as empty. - * - * @param gridPosition the n-dimensional position of the block to mark as empty - */ - public void setEmpty(int[] gridPosition) { - - set(EMPTY_INDEX_NBYTES, EMPTY_INDEX_NBYTES, gridPosition); - } - - /** - * Calculates the flattened array index for the offset value of a block. - * - * @param gridPosition the n-dimensional position of the block - * @return the index in the data array where the offset is stored - */ - protected int getOffsetIndex(int... gridPosition) { - - int idx = (int) gridPosition[0]; - int cumulativeSize = 1; - for (int i = 1; i < gridPosition.length; i++) { - cumulativeSize *= size[i]; - idx += gridPosition[i] * cumulativeSize; - } - return idx * 2; - } - - /** - * Calculates the flattened array index for the number of bytes value of a block. - * - * @param gridPosition the n-dimensional position of the block - * @return the index in the data array where the number of bytes is stored - */ - protected int getNumBytesIndex(int... gridPosition) { - - return getOffsetIndex(gridPosition) + 1; - } - - /** - * Calculates the total size of the index in bytes after compression. - * - * @return the total number of bytes the index occupies after applying all codecs - */ - public long numBytes() { - - final long numEntries = Arrays.stream(getSize()).reduce(1, (x, y) -> x * y); - return getIndexCodexAdapter().encodedSize(numEntries * BYTES_PER_LONG); - } - - - /** - * DatasetAttributes for the ShardIndex, used for codec operations. - */ - private static class ShardIndexAttributes extends DatasetAttributes { - - /** - * Creates attributes for the given ShardIndex. - * - * @param index the ShardIndex - */ - public ShardIndexAttributes(ShardIndex index) { - super( - Arrays.stream(index.getSize()).mapToLong(it -> it).toArray(), - index.getSize(), - index.getSize(), - DataType.UINT64, - index.indexCodexAdapter.getBlockCodecInfo(), - index.indexCodexAdapter.getDataCodecs() - ); - } - } - - /** - * Calculates the start byte of the index within a shard. - * - * @param index the ShardIndex - * @param objectSize the total size of the shard in bytes - * @return the start byte of the index - */ - public static long indexStartByte(final ShardIndex index, long objectSize) { - - return indexStartByte(index.numBytes(), index.location, objectSize); - } - - /** - * Calculates the start byte an index within a shard. - * - * @param indexSize the size of the index in bytes - * @param indexLocation the location of the index (START or END) - * @param objectSize the total size of the shard in bytes - * @return the start byte of the index - */ - public static long indexStartByte(final long indexSize, final IndexLocation indexLocation, final long objectSize) { - - if (indexLocation == IndexLocation.START) { - return 0L; - } else { - return objectSize - indexSize; - } - } - - /** - * Creates an empty index data array filled with {@link #EMPTY_INDEX_NBYTES}. - * - * @param size the dimensions of the block grid - * @return an array filled with empty values - */ - private static long[] emptyIndexData(final int[] size) { - - final int N = 2 * Arrays.stream(size).reduce(1, (x, y) -> x * y); - final long[] data = new long[N]; - Arrays.fill(data, EMPTY_INDEX_NBYTES); - return data; - } - - /** - * Prepends a value to an array. - * - * @param value the value to prepend - * @param array the original array - * @return a new array with the value prepended - */ - private static int[] prepend(final int value, final int[] array) { - - final int[] indexBlockSize = new int[array.length + 1]; - indexBlockSize[0] = value; - System.arraycopy(array, 0, indexBlockSize, 1, array.length); - return indexBlockSize; - } - - @Override - public boolean equals(Object other) { - - if (other instanceof ShardIndex) { - - final ShardIndex index = (ShardIndex)other; - if (this.location != index.location) - return false; - - if (!Arrays.equals(this.size, index.size)) - return false; - - if (!Arrays.equals(this.data, index.data)) - return false; - - } - return true; - } +public class ShardIndex { +//extends LongArrayDataBlock { + +// /** +// * Special value indicating an empty block entry in the index. +// * Used for both offset and length when a block doesn't exist. +// */ +// public static final long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; +// private static final int BYTES_PER_LONG = 8; +// private static final int LONGS_PER_BLOCK = 2; +// private static final long[] DUMMY_GRID_POSITION = null; +// +// private final IndexLocation location; +// private final ShardIndexAttributes indexAttributes; +// private final IndexCodecAdapter indexCodexAdapter; +// +// /** +// * Creates a ShardIndex with specified data. +// * +// * @param shardBlockGridSize the dimensions of the block grid within the shard +// * @param data the raw index data containing offsets and lengths +// * @param location where the index is stored (START or END of shard) +// * @param indexCodecAdapter data object for Shard Index codecs. +// */ +// public ShardIndex(int[] shardBlockGridSize, long[] data, IndexLocation location, final IndexCodecAdapter indexCodecAdapter) { +// +// // prepend the number of longs per block to the shard block grid size +// super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, data); +// this.indexCodexAdapter = indexCodecAdapter; +// this.location = location; +// this.indexAttributes = new ShardIndexAttributes(this); +// } +// +// /** +// * Creates an empty ShardIndex at the specified location. +// * +// * @param shardBlockGridSize the dimensions of the block grid within the shard +// * @param location where the index is stored (START or END of shard) +// * @param indexCodecAdapter data object for idnex codecs +// */ +// public ShardIndex(int[] shardBlockGridSize, IndexLocation location, final IndexCodecAdapter indexCodecAdapter) { +// +// this(shardBlockGridSize, emptyIndexData(shardBlockGridSize), location, indexCodecAdapter); +// } +// +// /** +// * Creates an empty ShardIndex at the default location (END). +// * +// * @param shardBlockGridSize the dimensions of the block grid within the shard +// * @param indexCodecAdapter data object for idnex codecs +// */ +// public ShardIndex(int[] shardBlockGridSize, final IndexCodecAdapter indexCodecAdapter) { +// +// this(shardBlockGridSize, IndexLocation.END, indexCodecAdapter); +// } +// +// /** +// * Creates an empty ShardIndex at the specified location. +// * +// * @param shardBlockGridSize the dimensions of the block grid within the shard +// * @param location where the index is stored (START or END of shard) +// * @param blockCodecInfo blockCodecInfo for the IndexCodecAdapter +// */ +// public ShardIndex(int[] shardBlockGridSize, IndexLocation location, final BlockCodecInfo blockCodecInfo) { +// +// this(shardBlockGridSize, location, new IndexCodecAdapter(blockCodecInfo)); +// } +// +// /** +// * Creates an empty ShardIndex at the default location (END). +// * +// * @param shardBlockGridSize the dimensions of the block grid within the shard +// * @param blockCodecInfo blockCodecInfo for the IndexCodecAdapter +// */ +// public ShardIndex(int[] shardBlockGridSize, final BlockCodecInfo blockCodecInfo) { +// +// this(shardBlockGridSize, IndexLocation.END, blockCodecInfo); +// } +// +// public IndexCodecAdapter getIndexCodexAdapter() { +// +// return indexCodexAdapter; +// } +// +// /** +// * Checks existence of the block at a given grid position. +// * +// * @param gridPosition the n-dimensional position of the block in the shard grid +// * @return true if the block exists, false otherwise +// */ +// public boolean exists(int[] gridPosition) { +// +// return getOffset(gridPosition) != EMPTY_INDEX_NBYTES || +// getNumBytes(gridPosition) != EMPTY_INDEX_NBYTES; +// } +// +// /** +// * Checks existence of the block at a given flat index. +// * +// * @param index the flattened index of the block +// * @return true if the block exists, false otherwise +// */ +// public boolean exists(int index) { +// +// return data[index * 2] != EMPTY_INDEX_NBYTES || +// data[index * 2 + 1] != EMPTY_INDEX_NBYTES; +// } +// +// /** +// * Gets the total number of blocks that can be stored in this index. +// * +// * @return the total number of blocks in the shard grid +// */ +// public int getNumBlocks() { +// +// /* getSize() is the number of data entries; each block takes 2 entries (offset and length) +// * so the product of the dimension sizes, divided by 2, is the number of blocks. */ +// return Arrays.stream(getSize()).reduce(1, (x, y) -> x * y) / 2; +// } +// +// /** +// * Checks if the index is completely empty (no blocks exist). +// * +// * @return true if no blocks exist in the index, false otherwise +// */ +// public boolean isEmpty() { +// +// return !IntStream.range(0, getNumBlocks()).anyMatch(this::exists); +// } +// +// /** +// * Gets the location of this index within the shard. +// * +// * @return the index location (START or END) +// */ +// public IndexLocation getLocation() { +// +// return location; +// } +// +// /** +// * Gets the offset in this shard in bytes for the block at a grid position. +// * +// * @param gridPosition the n-dimensional position of the block in the shard grid +// * @return the offset in bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist +// */ +// public long getOffset(int... gridPosition) { +// +// return data[getOffsetIndex(gridPosition)]; +// } +// +// /** +// * Gets the offset in this shard in bytes for the block at a given index. +// * +// * @param index the flattened index of the block +// * @return the offset in bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist +// */ +// public long getOffsetByBlockIndex(int index) { +// +// return data[index * 2]; +// } +// +// /** +// * Gets the number of bytes for the block at a grid position. +// * +// * @param gridPosition the n-dimensional position of the block in the shard grid +// * @return the number of bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist +// */ +// public long getNumBytes(int... gridPosition) { +// +// return data[getNumBytesIndex(gridPosition)]; +// } +// +// /** +// * Gets the number of bytes for the block at a given index. +// * +// * @param index the flattened index of the block +// * @return the number of bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist +// */ +// public long getNumBytesByBlockIndex(int index) { +// +// return data[index * 2 + 1]; +// } +// +// /** +// * Sets the offset and number of bytes for a block at the specified position. +// * +// * @param offset the byte offset of the block in the shard +// * @param nbytes the number of bytes the block occupies +// * @param gridPosition the n-dimensional position of the block in the shard grid +// */ +// public void set(long offset, long nbytes, int[] gridPosition) { +// +// final int i = getOffsetIndex(gridPosition); +// data[i] = offset; +// data[i + 1] = nbytes; +// } +// +// /** +// * Marks a block position as empty. +// * +// * @param gridPosition the n-dimensional position of the block to mark as empty +// */ +// public void setEmpty(int[] gridPosition) { +// +// set(EMPTY_INDEX_NBYTES, EMPTY_INDEX_NBYTES, gridPosition); +// } +// +// /** +// * Calculates the flattened array index for the offset value of a block. +// * +// * @param gridPosition the n-dimensional position of the block +// * @return the index in the data array where the offset is stored +// */ +// protected int getOffsetIndex(int... gridPosition) { +// +// int idx = (int) gridPosition[0]; +// int cumulativeSize = 1; +// for (int i = 1; i < gridPosition.length; i++) { +// cumulativeSize *= size[i]; +// idx += gridPosition[i] * cumulativeSize; +// } +// return idx * 2; +// } +// +// /** +// * Calculates the flattened array index for the number of bytes value of a block. +// * +// * @param gridPosition the n-dimensional position of the block +// * @return the index in the data array where the number of bytes is stored +// */ +// protected int getNumBytesIndex(int... gridPosition) { +// +// return getOffsetIndex(gridPosition) + 1; +// } +// +// /** +// * Calculates the total size of the index in bytes after compression. +// * +// * @return the total number of bytes the index occupies after applying all codecs +// */ +// public long numBytes() { +// +// final long numEntries = Arrays.stream(getSize()).reduce(1, (x, y) -> x * y); +// return getIndexCodexAdapter().encodedSize(numEntries * BYTES_PER_LONG); +// } +// +// +// /** +// * DatasetAttributes for the ShardIndex, used for codec operations. +// */ +// private static class ShardIndexAttributes extends DatasetAttributes { +// +// /** +// * Creates attributes for the given ShardIndex. +// * +// * @param index the ShardIndex +// */ +// public ShardIndexAttributes(ShardIndex index) { +// super( +// Arrays.stream(index.getSize()).mapToLong(it -> it).toArray(), +// index.getSize(), +// index.getSize(), +// DataType.UINT64, +// index.indexCodexAdapter.getBlockCodecInfo(), +// index.indexCodexAdapter.getDataCodecs() +// ); +// } +// } +// +// /** +// * Calculates the start byte of the index within a shard. +// * +// * @param index the ShardIndex +// * @param objectSize the total size of the shard in bytes +// * @return the start byte of the index +// */ +// public static long indexStartByte(final ShardIndex index, long objectSize) { +// +// return indexStartByte(index.numBytes(), index.location, objectSize); +// } +// +// /** +// * Calculates the start byte an index within a shard. +// * +// * @param indexSize the size of the index in bytes +// * @param indexLocation the location of the index (START or END) +// * @param objectSize the total size of the shard in bytes +// * @return the start byte of the index +// */ +// public static long indexStartByte(final long indexSize, final IndexLocation indexLocation, final long objectSize) { +// +// if (indexLocation == IndexLocation.START) { +// return 0L; +// } else { +// return objectSize - indexSize; +// } +// } +// +// /** +// * Creates an empty index data array filled with {@link #EMPTY_INDEX_NBYTES}. +// * +// * @param size the dimensions of the block grid +// * @return an array filled with empty values +// */ +// private static long[] emptyIndexData(final int[] size) { +// +// final int N = 2 * Arrays.stream(size).reduce(1, (x, y) -> x * y); +// final long[] data = new long[N]; +// Arrays.fill(data, EMPTY_INDEX_NBYTES); +// return data; +// } +// +// /** +// * Prepends a value to an array. +// * +// * @param value the value to prepend +// * @param array the original array +// * @return a new array with the value prepended +// */ +// private static int[] prepend(final int value, final int[] array) { +// +// final int[] indexBlockSize = new int[array.length + 1]; +// indexBlockSize[0] = value; +// System.arraycopy(array, 0, indexBlockSize, 1, array.length); +// return indexBlockSize; +// } +// +// @Override +// public boolean equals(Object other) { +// +// if (other instanceof ShardIndex) { +// +// final ShardIndex index = (ShardIndex)other; +// if (this.location != index.location) +// return false; +// +// if (!Arrays.equals(this.size, index.size)) +// return false; +// +// if (!Arrays.equals(this.data, index.data)) +// return false; +// +// } +// return true; +// } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java index 3289cd61e..8ac28b550 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java @@ -25,175 +25,176 @@ import java.util.Objects; @NameConfig.Name(ShardingCodec.TYPE) -public class ShardingCodec implements BlockCodecInfo { - - private static final long serialVersionUID = -5879797314954717810L; +public class ShardingCodec { +//implements BlockCodecInfo { +// private static final long serialVersionUID = -5879797314954717810L; +// public static final String TYPE = "sharding_indexed"; - - public static final String CHUNK_SHAPE_KEY = "chunk_shape"; - public static final String INDEX_LOCATION_KEY = "index_location"; - public static final String CODECS_KEY = "codecs"; - public static final String INDEX_CODECS_KEY = "index_codecs"; - private DatasetAttributes attributes = null; - - public enum IndexLocation { - START, END - } - - @N5Annotations.ReverseArray // TODO need to reverse for zarr, not for n5 - @NameConfig.Parameter(CHUNK_SHAPE_KEY) - private final int[] blockSize; - - @NameConfig.Parameter(CODECS_KEY) - private final CodecInfo[] codecs; - - @NameConfig.Parameter(INDEX_CODECS_KEY) - private final IndexCodecAdapter indexCodecs; - - @NameConfig.Parameter(value = INDEX_LOCATION_KEY, optional = true) - private final IndexLocation indexLocation; - - protected BlockCodec dataBlockSerializer = null; - - /** - * Used via reflections by the NameConfig serializer. - */ - @SuppressWarnings("unused") - private ShardingCodec() { - - blockSize = null; - codecs = null; - indexCodecs = null; - indexLocation = IndexLocation.END; - } - - public ShardingCodec( - final int[] blockSize, - final CodecInfo[] codecs, - final IndexCodecAdapter indexCodecs, - final IndexLocation indexLocation) { - - this.blockSize = blockSize; - this.codecs = codecs; - this.indexCodecs = indexCodecs; - this.indexLocation = indexLocation; - } - - public ShardingCodec( - final int[] blockSize, - final CodecInfo[] codecs, - final CodecInfo[] indexCodecs, - final IndexLocation indexLocation) { - - this(blockSize, codecs, IndexCodecAdapter.create(indexCodecs), indexLocation); - } - - public IndexLocation getIndexLocation() { - - return indexLocation; - } - - public BlockCodecInfo getBlockCodecInfo() { - - Objects.requireNonNull(codecs); - if (codecs.length == 0) - throw new IllegalArgumentException("Sharding CodecInfo requires a single BlockCodecInfo. None found."); - - return (BlockCodecInfo)codecs[0]; - } - public BlockCodec getBlockCodec() { - - return (BlockCodec)dataBlockSerializer; - } - - public DataCodec[] getCodecs() { - - Objects.requireNonNull(codecs); - final DataCodec[] dataCodecs = new DataCodec[codecs.length - 1]; - for (int i = 1; i < codecs.length; i++) - dataCodecs[i-1] = (DataCodec)codecs[i]; - return dataCodecs; - } - - public IndexCodecAdapter getIndexCodecAdapter() { - - return indexCodecs; - } - - @Override - public long[] getKeyPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { - - final long[] blockPosition = datablock.getGridPosition(); - return attributes.getShardPositionForBlock(blockPosition); - } - - @Override - public long[] getKeyPositionForBlock(DatasetAttributes attributes, final long... blockPosition) { - - return attributes.getShardPositionForBlock(blockPosition); - } - - public BlockCodec create(DatasetAttributes attributes, final DataCodec[] codecs) { - - // TODO - return null; - -// this.attributes = attributes; -// this.dataBlockSerializer = getBlockCodecInfo().create(attributes, getCodecs()); -// return ((BlockCodec)dataBlockSerializer); - } - - public ReadData encode(DataBlock dataBlock) { - - return this.getBlockCodec().encode(dataBlock); - } - - public DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException { - - final ReadData splitableReadData = readData.materialize(); - final long[] shardPosition = getKeyPositionForBlock(attributes, gridPosition); - final VirtualShard shard = new VirtualShard<>(attributes, shardPosition, splitableReadData); - final int[] relativeBlockPosition = shard.getRelativeBlockPosition(gridPosition); - return shard.getBlock(relativeBlockPosition); - } - - public ShardIndex createIndex(final DatasetAttributes attributes) { - - return new ShardIndex(attributes.getBlocksPerShard(), getIndexLocation(), getIndexCodecAdapter()); - } - - @Override - public String getType() { - - return TYPE; - } - - public static IndexLocationAdapter indexLocationAdapter = new IndexLocationAdapter(); - - public static class IndexLocationAdapter implements JsonSerializer, JsonDeserializer { - - @Override - public IndexLocation deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { - - if (!json.isJsonPrimitive()) - return null; - - return IndexLocation.valueOf(json.getAsString().toUpperCase()); - } - - @Override - public JsonElement serialize(IndexLocation src, Type typeOfSrc, JsonSerializationContext context) { - - return new JsonPrimitive(src.name().toLowerCase()); - } - } - - @Override - public BlockCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs) { - - // TODO - return null; - } +// +// public static final String CHUNK_SHAPE_KEY = "chunk_shape"; +// public static final String INDEX_LOCATION_KEY = "index_location"; +// public static final String CODECS_KEY = "codecs"; +// public static final String INDEX_CODECS_KEY = "index_codecs"; +// private DatasetAttributes attributes = null; +// +// public enum IndexLocation { +// START, END +// } +// +// @N5Annotations.ReverseArray // TODO need to reverse for zarr, not for n5 +// @NameConfig.Parameter(CHUNK_SHAPE_KEY) +// private final int[] blockSize; +// +// @NameConfig.Parameter(CODECS_KEY) +// private final CodecInfo[] codecs; +// +// @NameConfig.Parameter(INDEX_CODECS_KEY) +// private final IndexCodecAdapter indexCodecs; +// +// @NameConfig.Parameter(value = INDEX_LOCATION_KEY, optional = true) +// private final IndexLocation indexLocation; +// +// protected BlockCodec dataBlockSerializer = null; +// +// /** +// * Used via reflections by the NameConfig serializer. +// */ +// @SuppressWarnings("unused") +// private ShardingCodec() { +// +// blockSize = null; +// codecs = null; +// indexCodecs = null; +// indexLocation = IndexLocation.END; +// } +// +// public ShardingCodec( +// final int[] blockSize, +// final CodecInfo[] codecs, +// final IndexCodecAdapter indexCodecs, +// final IndexLocation indexLocation) { +// +// this.blockSize = blockSize; +// this.codecs = codecs; +// this.indexCodecs = indexCodecs; +// this.indexLocation = indexLocation; +// } +// +// public ShardingCodec( +// final int[] blockSize, +// final CodecInfo[] codecs, +// final CodecInfo[] indexCodecs, +// final IndexLocation indexLocation) { +// +// this(blockSize, codecs, IndexCodecAdapter.create(indexCodecs), indexLocation); +// } +// +// public IndexLocation getIndexLocation() { +// +// return indexLocation; +// } +// +// public BlockCodecInfo getBlockCodecInfo() { +// +// Objects.requireNonNull(codecs); +// if (codecs.length == 0) +// throw new IllegalArgumentException("Sharding CodecInfo requires a single BlockCodecInfo. None found."); +// +// return (BlockCodecInfo)codecs[0]; +// } +// public BlockCodec getBlockCodec() { +// +// return (BlockCodec)dataBlockSerializer; +// } +// +// public DataCodec[] getCodecs() { +// +// Objects.requireNonNull(codecs); +// final DataCodec[] dataCodecs = new DataCodec[codecs.length - 1]; +// for (int i = 1; i < codecs.length; i++) +// dataCodecs[i-1] = (DataCodec)codecs[i]; +// return dataCodecs; +// } +// +// public IndexCodecAdapter getIndexCodecAdapter() { +// +// return indexCodecs; +// } +// +// @Override +// public long[] getKeyPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { +// +// final long[] blockPosition = datablock.getGridPosition(); +// return attributes.getShardPositionForBlock(blockPosition); +// } +// +// @Override +// public long[] getKeyPositionForBlock(DatasetAttributes attributes, final long... blockPosition) { +// +// return attributes.getShardPositionForBlock(blockPosition); +// } +// +// public BlockCodec create(DatasetAttributes attributes, final DataCodec[] codecs) { +// +// // TODO +// return null; +// +//// this.attributes = attributes; +//// this.dataBlockSerializer = getBlockCodecInfo().create(attributes, getCodecs()); +//// return ((BlockCodec)dataBlockSerializer); +// } +// +// public ReadData encode(DataBlock dataBlock) { +// +// return this.getBlockCodec().encode(dataBlock); +// } +// +// public DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException { +// +// final ReadData splitableReadData = readData.materialize(); +// final long[] shardPosition = getKeyPositionForBlock(attributes, gridPosition); +// final VirtualShard shard = new VirtualShard<>(attributes, shardPosition, splitableReadData); +// final int[] relativeBlockPosition = shard.getRelativeBlockPosition(gridPosition); +// return shard.getBlock(relativeBlockPosition); +// } +// +// public ShardIndex createIndex(final DatasetAttributes attributes) { +// +// return new ShardIndex(attributes.getBlocksPerShard(), getIndexLocation(), getIndexCodecAdapter()); +// } +// +// @Override +// public String getType() { +// +// return TYPE; +// } +// +// public static IndexLocationAdapter indexLocationAdapter = new IndexLocationAdapter(); +// +// public static class IndexLocationAdapter implements JsonSerializer, JsonDeserializer { +// +// @Override +// public IndexLocation deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { +// +// if (!json.isJsonPrimitive()) +// return null; +// +// return IndexLocation.valueOf(json.getAsString().toUpperCase()); +// } +// +// @Override +// public JsonElement serialize(IndexLocation src, Type typeOfSrc, JsonSerializationContext context) { +// +// return new JsonPrimitive(src.name().toLowerCase()); +// } +// } +// +// @Override +// public BlockCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs) { +// +// // TODO +// return null; +// } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java index fb479ad98..65129cef6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java @@ -19,112 +19,112 @@ //TODO : consider different names? LazyShard? ShardView? PartialReadShard? OnDemand? public class VirtualShard extends AbstractShard { - private final ReadData shardData; - - public VirtualShard( - final DatasetAttributes datasetAttributes, - long[] gridPosition, - final ReadData shardData) { - - super(datasetAttributes, gridPosition, null); - this.shardData = shardData; - } - - @SuppressWarnings("unchecked") - public DataBlock getBlock(ReadData blockData, long... blockGridPosition) throws IOException { - - // TODO - return null; - } - - @Override - public List> getBlocks() { - - return getBlocks(IntStream.range(0, getNumBlocks()).toArray()); - } - - public List> getBlocks(final int[] blockIndexes) { - - // will not contain nulls - final ShardIndex index = getIndex(); - final ArrayList> blocks = new ArrayList<>(); - - if (index.isEmpty()) - return blocks; - - // sort index offsets - // and keep track of relevant positions - final long[] indexData = index.getData(); - List sortedOffsets = Arrays.stream(blockIndexes) - .mapToObj(i -> new long[]{indexData[i * 2], i}) - .filter(x -> x[0] != ShardIndex.EMPTY_INDEX_NBYTES) - .sorted(Comparator.comparingLong(a -> ((long[])a)[0])) - .collect(Collectors.toList()); - - final int nd = getDatasetAttributes().getNumDimensions(); - long[] position = new long[nd]; - - final int[] blocksPerShard = getDatasetAttributes().getBlocksPerShard(); - final long[] blockGridMin = IntStream.range(0, nd) - .mapToLong(i -> blocksPerShard[i] * getGridPosition()[i]) - .toArray(); - - for (long[] offsetIndex : sortedOffsets) { - final long offset = offsetIndex[0]; - if (offset < 0) - continue; - - final long idx = offsetIndex[1]; - GridIterator.indexToPosition(idx, blocksPerShard, blockGridMin, position); - - final long numBytes = index.getNumBytesByBlockIndex((int)idx); - //TODO Caleb: Do this with a single access (start at first offset, read through the last) - try { - final ReadData blockData = shardData.slice(offset, numBytes); - final DataBlock block = getBlock(blockData, position.clone()); - blocks.add(block); - } catch (final N5Exception.N5NoSuchKeyException e) { - return blocks; - } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to read block from " + Arrays.toString(position), e); - } - } - return blocks; - } - - @Override - public DataBlock getBlock(int... relativePosition) { - - final ShardIndex idx = getIndex(); - if (!idx.exists(relativePosition)) - return null; - - final long blockOffset = idx.getOffset(relativePosition); - final long blockSize = idx.getNumBytes(relativePosition); - - final long[] blockPosInDataset = getDatasetAttributes().getBlockPositionFromShardPosition(getGridPosition(), relativePosition); - try { - final ReadData blockData = shardData.slice(blockOffset, blockSize); - return getBlock(blockData, blockPosInDataset); - } catch (final N5Exception.N5NoSuchKeyException e) { - return null; - } catch (final IOException | UncheckedIOException e) { - throw new N5IOException("Failed to read block from " + Arrays.toString(blockPosInDataset), e); - } - } - - @Override - public ShardIndex getIndex() { - - // TODO - return null; - -// index = createIndex(); +// private final ReadData shardData; +// +// public VirtualShard( +// final DatasetAttributes datasetAttributes, +// long[] gridPosition, +// final ReadData shardData) { +// +// super(datasetAttributes, gridPosition, null); +// this.shardData = shardData; +// } +// +// @SuppressWarnings("unchecked") +// public DataBlock getBlock(ReadData blockData, long... blockGridPosition) throws IOException { +// +// // TODO +// return null; +// } +// +// @Override +// public List> getBlocks() { +// +// return getBlocks(IntStream.range(0, getNumBlocks()).toArray()); +// } +// +// public List> getBlocks(final int[] blockIndexes) { +// +// // will not contain nulls +// final ShardIndex index = getIndex(); +// final ArrayList> blocks = new ArrayList<>(); +// +// if (index.isEmpty()) +// return blocks; +// +// // sort index offsets +// // and keep track of relevant positions +// final long[] indexData = index.getData(); +// List sortedOffsets = Arrays.stream(blockIndexes) +// .mapToObj(i -> new long[]{indexData[i * 2], i}) +// .filter(x -> x[0] != ShardIndex.EMPTY_INDEX_NBYTES) +// .sorted(Comparator.comparingLong(a -> ((long[])a)[0])) +// .collect(Collectors.toList()); +// +// final int nd = getDatasetAttributes().getNumDimensions(); +// long[] position = new long[nd]; +// +// final int[] blocksPerShard = getDatasetAttributes().getBlocksPerShard(); +// final long[] blockGridMin = IntStream.range(0, nd) +// .mapToLong(i -> blocksPerShard[i] * getGridPosition()[i]) +// .toArray(); +// +// for (long[] offsetIndex : sortedOffsets) { +// final long offset = offsetIndex[0]; +// if (offset < 0) +// continue; +// +// final long idx = offsetIndex[1]; +// GridIterator.indexToPosition(idx, blocksPerShard, blockGridMin, position); +// +// final long numBytes = index.getNumBytesByBlockIndex((int)idx); +// //TODO Caleb: Do this with a single access (start at first offset, read through the last) +// try { +// final ReadData blockData = shardData.slice(offset, numBytes); +// final DataBlock block = getBlock(blockData, position.clone()); +// blocks.add(block); +// } catch (final N5Exception.N5NoSuchKeyException e) { +// return blocks; +// } catch (final IOException | UncheckedIOException e) { +// throw new N5IOException("Failed to read block from " + Arrays.toString(position), e); +// } +// } +// return blocks; +// } +// +// @Override +// public DataBlock getBlock(int... relativePosition) { +// +// final ShardIndex idx = getIndex(); +// if (!idx.exists(relativePosition)) +// return null; +// +// final long blockOffset = idx.getOffset(relativePosition); +// final long blockSize = idx.getNumBytes(relativePosition); +// +// final long[] blockPosInDataset = getDatasetAttributes().getBlockPositionFromShardPosition(getGridPosition(), relativePosition); // try { -// ShardIndex.readFromShard(shardData, index); -// } catch (N5Exception.N5NoSuchKeyException e) { +// final ReadData blockData = shardData.slice(blockOffset, blockSize); +// return getBlock(blockData, blockPosInDataset); +// } catch (final N5Exception.N5NoSuchKeyException e) { // return null; +// } catch (final IOException | UncheckedIOException e) { +// throw new N5IOException("Failed to read block from " + Arrays.toString(blockPosInDataset), e); // } -// return index; - } +// } +// +// @Override +// public ShardIndex getIndex() { +// +// // TODO +// return null; +// +//// index = createIndex(); +//// try { +//// ShardIndex.readFromShard(shardData, index); +//// } catch (N5Exception.N5NoSuchKeyException e) { +//// return null; +//// } +//// return index; +// } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java index e9e727dec..947473f76 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java @@ -13,7 +13,7 @@ import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData.SegmentsAndData; -class ShardIndex { +public class ShardIndex { private ShardIndex() { // utility class. should not be instantiated. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java index 75eb1d8e9..006eec51b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java @@ -139,7 +139,7 @@ private ReadData deleteBlockRecursive( } } - public static DatasetAccess create( + public static ShardedDatasetAccess create( final DataType dataType, int[] blockSize, BlockCodecInfo blockCodecInfo, diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index f9929c263..fa2004357 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1647,45 +1647,46 @@ public void testPathsWithIllegalUriCharacters() throws IOException, URISyntaxExc @Test public void testWriteReadShardOnUnshardedDataset() { - try (N5Writer writer = createTempN5Writer()) { - final String datasetName = "testWriteShardOnUnshardedDataset"; - final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64, new RawCompression()); - writer.createDataset(datasetName, datasetAttributes); - - - final DataBlock block0 = (DataBlock) DataType.UINT64.createDataBlock(blockSize, new long[]{0,0,0}); - final DataBlock block1 = (DataBlock) DataType.UINT64.createDataBlock(blockSize, new long[]{1,0,0}); - - final InMemoryShard writeShard = new InMemoryShard<>(datasetAttributes, new long[]{0, 0, 0}); - boolean added0 = writeShard.addBlock(block0); - boolean added1 = writeShard.addBlock(block1); - - assertTrue("Block 0 should be added to shard", added0); - assertFalse("Block 1 should not be added to shard because it's in a different shard position", added1); - - final List> writeBlocks = writeShard.getBlocks(); - assertEquals("block as shard should not have the second block which is outside this shard", 1, writeBlocks.size()); - assertEquals(block0, writeBlocks.get(0)); - - writer.writeShard(datasetName, datasetAttributes, writeShard); - final DataBlock readAsBlock0 = writer.readBlock(datasetName, datasetAttributes, block0.getGridPosition()); - assertBlockEquals(block0, readAsBlock0); - - assertFalse("Block 1 should not exist because it's not contained in the written shard", writer.blockExists(datasetName, datasetAttributes, block1.getGridPosition())); - - final Shard readShard = writer.readShard(datasetName, datasetAttributes, writeShard.getGridPosition()); - Assert.assertArrayEquals("shard read position should be same as write position", writeShard.getGridPosition(), readShard.getGridPosition()); - Assert.assertArrayEquals("shard position should be the same as block position when unsharded", block0.getGridPosition(), readShard.getGridPosition()); - Assert.assertArrayEquals("shard size should equal block size when unsharded", readShard.getBlockSize(), readShard.getSize()); - - - final List> readBlocks = readShard.getBlocks(); - assertEquals("read shard should contain one block", 1, readBlocks.size()); - final DataBlock readAsShardBlock0 = readBlocks.get(0); - - assertBlockEquals(block0, readAsShardBlock0); - - } + // TODO +// try (N5Writer writer = createTempN5Writer()) { +// final String datasetName = "testWriteShardOnUnshardedDataset"; +// final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64, new RawCompression()); +// writer.createDataset(datasetName, datasetAttributes); +// +// +// final DataBlock block0 = (DataBlock) DataType.UINT64.createDataBlock(blockSize, new long[]{0,0,0}); +// final DataBlock block1 = (DataBlock) DataType.UINT64.createDataBlock(blockSize, new long[]{1,0,0}); +// +// final InMemoryShard writeShard = new InMemoryShard<>(datasetAttributes, new long[]{0, 0, 0}); +// boolean added0 = writeShard.addBlock(block0); +// boolean added1 = writeShard.addBlock(block1); +// +// assertTrue("Block 0 should be added to shard", added0); +// assertFalse("Block 1 should not be added to shard because it's in a different shard position", added1); +// +// final List> writeBlocks = writeShard.getBlocks(); +// assertEquals("block as shard should not have the second block which is outside this shard", 1, writeBlocks.size()); +// assertEquals(block0, writeBlocks.get(0)); +// +// writer.writeShard(datasetName, datasetAttributes, writeShard); +// final DataBlock readAsBlock0 = writer.readBlock(datasetName, datasetAttributes, block0.getGridPosition()); +// assertBlockEquals(block0, readAsBlock0); +// +// assertFalse("Block 1 should not exist because it's not contained in the written shard", writer.blockExists(datasetName, datasetAttributes, block1.getGridPosition())); +// +// final Shard readShard = writer.readShard(datasetName, datasetAttributes, writeShard.getGridPosition()); +// Assert.assertArrayEquals("shard read position should be same as write position", writeShard.getGridPosition(), readShard.getGridPosition()); +// Assert.assertArrayEquals("shard position should be the same as block position when unsharded", block0.getGridPosition(), readShard.getGridPosition()); +// Assert.assertArrayEquals("shard size should equal block size when unsharded", readShard.getBlockSize(), readShard.getSize()); +// +// +// final List> readBlocks = readShard.getBlocks(); +// assertEquals("read shard should contain one block", 1, readBlocks.size()); +// final DataBlock readAsShardBlock0 = readBlocks.get(0); +// +// assertBlockEquals(block0, readAsShardBlock0); +// +// } } public static void assertBlockEquals(final DataBlock expected, final DataBlock actual) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java index c032d2285..a833d82b8 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java @@ -20,7 +20,6 @@ import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; -import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shardstuff.DatasetAccess; import org.janelia.saalfeldlab.n5.shardstuff.PositionValueAccess; import org.janelia.saalfeldlab.n5.shardstuff.TestPositionValueAccess; @@ -56,7 +55,6 @@ public void testN5BlockCodec() throws Exception { final DatasetAttributes attributes = new DatasetAttributes( new long[]{32, 32, 32}, blockSize, - blockSize, dataType, new N5BlockCodecInfo(), dataCodecInfo); @@ -77,7 +75,6 @@ public void testRawBytesBlockCodec() throws Exception { final RawBlockCodecInfo codec = new RawBlockCodecInfo(byteOrder); final DatasetAttributes attributes = new DatasetAttributes( new long[]{32, 32, 32}, - blockSize, //shardSize blockSize, dataType, codec, @@ -123,7 +120,6 @@ public void testEmptyBlock() throws Exception { final DatasetAttributes attributes = new DatasetAttributes( new long[]{64, 64}, new int[]{8, 8}, - new int[]{8, 8}, DataType.UINT8, blockCodecInfo); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java index c13a67ffe..170f0e558 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java @@ -16,7 +16,6 @@ import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo; import org.janelia.saalfeldlab.n5.shard.ShardingCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; public class BlockIterators { @@ -24,72 +23,72 @@ public class BlockIterators { public static void main(String[] args) { // blockIterator(); - shardBlockIterator(); +// shardBlockIterator(); } public static void shardBlockIterator() { - final DatasetAttributes attrs = new DatasetAttributes( - new long[] {12, 8}, // image size - new int[] {6, 4}, // shard size - new int[] {2, 2}, // block size - DataType.UINT8, - new ShardingCodec( - new int[] {2, 2}, - new CodecInfo[] { new N5BlockCodecInfo() }, - new DeterministicSizeCodecInfo[] { new RawBlockCodecInfo() }, - IndexLocation.END - )); - - shardPositions(attrs) - .forEach(x -> System.out.println(Arrays.toString(x))); - } - - public static void blockIterator() { - - final DatasetAttributes attrs = new DatasetAttributes( - new long[] {12, 8}, - new int[] {2, 2}, - DataType.UINT8, - new RawCompression()); - - blockPositions(attrs).forEach(x -> System.out.println(Arrays.toString(x))); - } - - public static long[] blockGridSize(final DatasetAttributes attrs ) { - // this could be a nice method for DatasetAttributes - - return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getBlockSize()[i])).toArray(); - - } - - public static long[] shardGridSize(final DatasetAttributes attrs ) { - // this could be a nice method for DatasetAttributes - - return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getShardSize()[i])).toArray(); - +// final DatasetAttributes attrs = new DatasetAttributes( +// new long[] {12, 8}, // image size +// new int[] {6, 4}, // shard size +// new int[] {2, 2}, // block size +// DataType.UINT8, +// new ShardingCodec( +// new int[] {2, 2}, +// new CodecInfo[] { new N5BlockCodecInfo() }, +// new DeterministicSizeCodecInfo[] { new RawBlockCodecInfo() }, +// IndexLocation.END +// )); +// +// shardPositions(attrs) +// .forEach(x -> System.out.println(Arrays.toString(x))); } - public static Stream blockPositions( DatasetAttributes attrs ) { - return toStream(new GridIterator(blockGridSize(attrs))); - } - - public static Stream shardPositions( DatasetAttributes attrs ) { - - final int[] blocksPerShard = attrs.getBlocksPerShard(); - return toStream( new GridIterator(shardGridSize(attrs))) - .flatMap( shardPosition -> { - - final int nd = attrs.getNumDimensions(); - final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]); - return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); - }); - } - - public static Stream toStream( final Iterator it ) { - return StreamSupport.stream( Spliterators.spliteratorUnknownSize( - it, Spliterator.ORDERED), - false); - } +// public static void blockIterator() { +// +// final DatasetAttributes attrs = new DatasetAttributes( +// new long[] {12, 8}, +// new int[] {2, 2}, +// DataType.UINT8, +// new RawCompression()); +// +// blockPositions(attrs).forEach(x -> System.out.println(Arrays.toString(x))); +// } +// +// public static long[] blockGridSize(final DatasetAttributes attrs ) { +// // this could be a nice method for DatasetAttributes +// +// return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getBlockSize()[i])).toArray(); +// +// } +// +// public static long[] shardGridSize(final DatasetAttributes attrs ) { +// // this could be a nice method for DatasetAttributes +// +// return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getShardSize()[i])).toArray(); +// +// } +// +// public static Stream blockPositions( DatasetAttributes attrs ) { +// return toStream(new GridIterator(blockGridSize(attrs))); +// } +// +// public static Stream shardPositions( DatasetAttributes attrs ) { +// +// final int[] blocksPerShard = attrs.getBlocksPerShard(); +// return toStream( new GridIterator(shardGridSize(attrs))) +// .flatMap( shardPosition -> { +// +// final int nd = attrs.getNumDimensions(); +// final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]); +// return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); +// }); +// } +// +// public static Stream toStream( final Iterator it ) { +// return StreamSupport.stream( Spliterators.spliteratorUnknownSize( +// it, Spliterator.ORDERED), +// false); +// } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java index f606a8672..ea1406d83 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -7,7 +7,6 @@ import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.junit.After; import org.junit.Ignore; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 495b91fc0..16bfa8c87 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -21,7 +21,6 @@ import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.junit.After; import org.junit.Assert; From 1db5b73211e25f1b596d58724cf1f7afb3cca52b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Sep 2025 21:53:55 -0400 Subject: [PATCH 349/423] fix: ShardedDatasetAccess create method with validation --- .../n5/shardstuff/ShardedDatasetAccess.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java index 006eec51b..3b0207402 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java @@ -151,25 +151,28 @@ public static ShardedDatasetAccess create( // The inner-most codec (the DataBlock codec) is at index 0. final int[][] blockSizes = new int[m][]; + // NestedGrid validates block sizes, so instantiate it before creating the blockCodecs + // blockCodecInfo.create below could fail unexpecedly with invalid blockSizes + // so validate first + blockSizes[m-1] = blockSize; + BlockCodecInfo tmpInfo = blockCodecInfo; + for (int l = m - 1; l > 0; --l) { + final ShardCodecInfo info = (ShardCodecInfo) tmpInfo; + blockSizes[l-1] = info.getInnerBlockSize(); + tmpInfo = info.getInnerBlockCodecInfo(); + } + final NestedGrid grid = new NestedGrid(blockSizes); + final BlockCodec[] blockCodecs = new BlockCodec[m]; for (int l = m - 1; l >= 0; --l) { - blockSizes[l] = blockSize; + blockCodecs[l] = blockCodecInfo.create(dataType, blockSizes[l], dataCodecInfos); if (l > 0) { final ShardCodecInfo info = (ShardCodecInfo) blockCodecInfo; blockCodecInfo = info.getInnerBlockCodecInfo(); dataCodecInfos = info.getInnerDataCodecInfos(); - blockSize = info.getInnerBlockSize(); } } - // NestedGrid validates block sizes, so instantiate it before creating the blockCodecs - final NestedGrid grid = new NestedGrid(blockSizes); - - final BlockCodec[] blockCodecs = new BlockCodec[m]; - for (int l = m - 1; l >= 0; --l) { - blockCodecs[l] = blockCodecInfo.create(dataType, blockSize, dataCodecInfos); - } - return new ShardedDatasetAccess<>(grid, blockCodecs); } From e61b513f0a0fa164663c75693ecac65c30c2e357 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 Sep 2025 21:54:57 -0400 Subject: [PATCH 350/423] fix: null check on existingData in writeBlockRecursive is not enough * when the existingData is a LazyReadData --- .../n5/shardstuff/ShardedDatasetAccess.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java index 3b0207402..7e44d7348 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java @@ -3,6 +3,7 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; @@ -49,12 +50,22 @@ private DataBlock readBlockRecursive( } @Override - public void writeBlock(final PositionValueAccess kva, final DataBlock dataBlock) throws N5IOException { + public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException { final NestedPosition position = new NestedPosition(grid, dataBlock.getGridPosition()); final long[] key = position.key(); - final ReadData existingData = kva.get(key); + + // need to read the shard anyway, and currently (Sept 24 2025) + // have no way to tell if they key exist from what is in this method except to attempt + // to materialize and catch the N5NoSuchKeyException + ReadData existingData = null; + try { + existingData = pva.get(key); + if (existingData != null) + existingData.materialize(); + } catch (N5NoSuchKeyException e) {} + final ReadData modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1); - kva.put(key, modifiedData); + pva.put(key, modifiedData); } private ReadData writeBlockRecursive( From 6232eda77537e3ba82276de3e64cd837de9be0c2 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 26 Sep 2025 08:24:33 -0400 Subject: [PATCH 351/423] fix: ShardedDatasetAccess set existingData to null if no key --- .../saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java index 7e44d7348..78a3198f3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.shardstuff; + import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; @@ -62,7 +63,9 @@ public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlo existingData = pva.get(key); if (existingData != null) existingData.materialize(); - } catch (N5NoSuchKeyException e) {} + } catch (N5NoSuchKeyException e) { + existingData = null; + } final ReadData modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1); pva.put(key, modifiedData); From 8f7f5da363ee52caa292c8cae982562d4547ee07 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 6 Oct 2025 16:09:30 -0400 Subject: [PATCH 352/423] feat: serialization for DefaultShardCodecInfo and RawCompression --- .../saalfeldlab/n5/RawCompression.java | 2 + .../n5/shardstuff/DefaultShardCodecInfo.java | 53 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index bf55bcfcc..6d3ebc86e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -31,8 +31,10 @@ import org.janelia.saalfeldlab.n5.Compression.CompressionType; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeDataCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("raw") +@NameConfig.Name("raw") public class RawCompression implements Compression, DeterministicSizeDataCodec { private static final long serialVersionUID = 7526445806847086477L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultShardCodecInfo.java index 37e80f49b..cbcd50570 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultShardCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultShardCodecInfo.java @@ -1,16 +1,27 @@ package org.janelia.saalfeldlab.n5.shardstuff; +import java.lang.reflect.Type; import java.util.Arrays; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + /** * Default (and probably only) implementation of {@link ShardCodecInfo}. */ // TODO rename? +@NameConfig.Name(value = "ShardingCodec") public class DefaultShardCodecInfo implements ShardCodecInfo { @Override @@ -25,6 +36,11 @@ public String getType() { private final DataCodecInfo[] indexDataCodecInfos; private final IndexLocation indexLocation; + DefaultShardCodecInfo() { + // for serialization + this(null, null, null, null, null, null); + } + public DefaultShardCodecInfo( final int[] innerBlockSize, final BlockCodecInfo innerBlockCodecInfo, @@ -86,4 +102,41 @@ public RawShardCodec create(final int[] blockSize, final DataCodecInfo... codecs return new RawShardCodec(size, indexLocation, indexCodec); } + + public static DefaultShardCodecInfoAdapter adapter = new DefaultShardCodecInfoAdapter(); + + public static class DefaultShardCodecInfoAdapter implements JsonDeserializer, JsonSerializer { + + @Override + public DefaultShardCodecInfo deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { + + if (!json.isJsonObject()) + return null; + + JsonObject obj = json.getAsJsonObject(); + + int[] innerBlockSize = context.deserialize(obj.get("innerBlockSize"), int[].class); + BlockCodecInfo innerBlockCodecInfo = context.deserialize(obj.get("innerBlockCodecInfo"), BlockCodecInfo[].class); + DataCodecInfo[] innerDataCodecInfos = context.deserialize(obj.get("innerDataCodecInfos"), DataCodecInfo[].class); + BlockCodecInfo indexBlockCodecInfo = context.deserialize(obj.get("indexBlockCodecInfo"), BlockCodecInfo[].class); + DataCodecInfo[] indexDataCodecInfos = context.deserialize(obj.get("indexDataCodecInfos"), DataCodecInfo[].class); + IndexLocation indexLocation = IndexLocation.valueOf(obj.get("indexLocation").getAsString()); + + return new DefaultShardCodecInfo(innerBlockSize, innerBlockCodecInfo, innerDataCodecInfos, indexBlockCodecInfo, indexDataCodecInfos, indexLocation); + } + + @Override + public JsonElement serialize(DefaultShardCodecInfo src, Type typeOfSrc, JsonSerializationContext context) { + + final JsonObject obj = new JsonObject(); + obj.add("innerBlockSize", context.serialize(src.innerBlockSize)); + obj.add("innerBlockCodecInfo", context.serialize(src.innerBlockCodecInfo)); + obj.add("innerDataCodecInfos", context.serialize(src.innerDataCodecInfos)); + obj.add("indexBlockCodecInfo", context.serialize(src.indexBlockCodecInfo)); + obj.add("indexDataCodecInfos", context.serialize(src.indexDataCodecInfos)); + obj.add("indexLocation", context.serialize(src.indexLocation)); + return obj; + } + } + } From e1725361db21fb66d019135492291ff9e7856fff Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 6 Oct 2025 16:09:41 -0400 Subject: [PATCH 353/423] partially working ShardTest --- .../saalfeldlab/n5/shard/ShardTest.java | 508 +++++++++--------- 1 file changed, 268 insertions(+), 240 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 16bfa8c87..ba14ae454 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -12,6 +12,7 @@ import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.NameConfigAdapter; +import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.DatasetAttributes.DatasetAttributesAdapter; import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; import org.janelia.saalfeldlab.n5.codec.CodecInfo; @@ -21,6 +22,8 @@ import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; +import org.janelia.saalfeldlab.n5.shardstuff.DefaultShardCodecInfo; +import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.junit.After; import org.junit.Assert; @@ -32,10 +35,12 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; import java.io.File; import java.lang.reflect.Type; @@ -63,201 +68,201 @@ public class ShardTest { // TODO -// private static final boolean LOCAL_DEBUG = false; -// -// private static final N5FSTest tempN5Factory = new N5FSTest() { -// -// @Override public N5Writer createTempN5Writer() { -// -// if (LOCAL_DEBUG) { -// final N5Writer writer = new ShardedN5Writer("src/test/resources/test.n5"); -// writer.remove(""); // Clear old when starting new test -// return writer; -// } -// -// final String basePath = new File(tempN5PathName()).toURI().normalize().getPath(); -// try { -// String uri = new URI("file", null, basePath, null).toString(); -// return new ShardedN5Writer(uri); -// } catch (URISyntaxException e) { -// e.printStackTrace(); -// } -// return null; -// } -// -// private String tempN5PathName() { -// -// try { -// final File tmpFile = Files.createTempDirectory("n5-shard-test-").toFile(); -// tmpFile.delete(); -// tmpFile.mkdir(); -// tmpFile.deleteOnExit(); -// return tmpFile.getCanonicalPath(); -// } catch (final Exception e) { -// throw new RuntimeException(e); -// } -// } -// }; -// -// public static GsonBuilder gsonBuilder() { -// return new GsonBuilder(); -// } -// -// @Parameterized.Parameters(name = "IndexLocation({0}), Index ByteOrder({1})") -// public static Collection data() { -// -// final ArrayList params = new ArrayList<>(); -// for (IndexLocation indexLoc : IndexLocation.values()) { -// for (ByteOrder indexByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { -// params.add(new Object[]{indexLoc, indexByteOrder}); -// } -// } -// final int numParams = params.size(); -// final Object[][] paramArray = new Object[numParams][]; -// Arrays.setAll(paramArray, params::get); -// return Arrays.asList(paramArray); -// } -// -// @Parameterized.Parameter() -// public IndexLocation indexLocation; -// -// @Parameterized.Parameter(1) -// public ByteOrder indexByteOrder; -// -// @After -// public void removeTempWriters() { -// -// tempN5Factory.removeTempWriters(); -// } -// -// private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] blockSize) { -// -// return new DatasetAttributes( -// dimensions, -// shardSize, -// blockSize, -// DataType.UINT8, -// new ShardingCodec( -// blockSize, -// new CodecInfo[]{new N5BlockCodecInfo()}, -// new CodecInfo[]{new RawBlockCodecInfo(), new Crc32cChecksumCodec()}, -// indexLocation -// ) -// ); -// } -// -// private DatasetAttributes getTestAttributes() { -// -// return getTestAttributes(new long[]{8, 8}, new int[]{4, 4}, new int[]{2, 2}); -// } -// -// private DatasetAttributes getTestAttributes3d() { -// -// final int[] blockSize = {33, 22, 11}; -// final int[] shardSize = {blockSize[0] * 2, blockSize[1] * 2, blockSize[2] * 2}; -// return getTestAttributes(new long[]{10, 20, 30}, shardSize, blockSize); -// } -// -// @Test -// public void writeReadBlocksTest() { -// -// final N5Writer writer = tempN5Factory.createTempN5Writer(); -// final DatasetAttributes datasetAttributes = getTestAttributes( -// new long[]{24, 24}, -// new int[]{8, 8}, -// new int[]{2, 2} -// ); -// -// final String dataset = "writeReadBlocks"; -// writer.remove(dataset); -// writer.createDataset(dataset, datasetAttributes); -// -// final int[] blockSize = datasetAttributes.getBlockSize(); -// final int numElements = blockSize[0] * blockSize[1]; -// -// final byte[] data = new byte[numElements]; -// for (int i = 0; i < data.length; i++) { -// data[i] = (byte)((100) + (10) + i); -// } -// -// writer.writeBlocks( -// dataset, -// datasetAttributes, -// /* shard (0, 0) */ -// new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), -// new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), -// new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), -// new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), -// -// /* shard (1, 0) */ -// new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), -// new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), -// -// /* shard (2, 2) */ -// new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) -// ); -// -// final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); -// -// final String[][] keys = new String[][]{ -// {dataset, "0", "0"}, -// {dataset, "1", "0"}, -// {dataset, "2", "2"} -// }; -// for (String[] key : keys) { -// final String shard = kva.compose(writer.getURI(), key); -// Assert.assertTrue("Shard at" + Arrays.toString(key) + "Does not exist", kva.exists(shard)); -// } -// -// final long[][] blockIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; -// for (long[] blockIndex : blockIndices) { -// final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); -// Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); -// } -// -// final byte[] data2 = new byte[numElements]; -// for (int i = 0; i < data2.length; i++) { -// data2[i] = (byte)(10 + i); -// } -// writer.writeBlocks( -// dataset, -// datasetAttributes, -// /* shard (0, 0) */ -// new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data2), -// new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data2), -// -// /* shard (0, 1) */ -// new ByteArrayDataBlock(blockSize, new long[]{0, 4}, data2), -// new ByteArrayDataBlock(blockSize, new long[]{0, 5}, data2), -// -// /* shard (2, 2) */ -// new ByteArrayDataBlock(blockSize, new long[]{10, 10}, data2) -// ); -// -// final String[][] keys2 = new String[][]{ -// {dataset, "0", "0"}, -// {dataset, "1", "0"}, -// {dataset, "0", "1"}, -// {dataset, "2", "2"} -// }; -// for (String[] key : keys2) { -// final String shard = kva.compose(writer.getURI(), key); -// Assert.assertTrue("Shard at" + Arrays.toString(key) + "Does not exist", kva.exists(shard)); -// } -// -// final long[][] oldBlockIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; -// for (long[] blockIndex : oldBlockIndices) { -// final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); -// Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); -// } -// -// final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; -// for (long[] blockIndex : newBlockIndices) { -// final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); -// Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])block.getData()); -// } -// } -// + private static final boolean LOCAL_DEBUG = false; + + private static final N5FSTest tempN5Factory = new N5FSTest() { + + @Override public N5Writer createTempN5Writer() { + + if (LOCAL_DEBUG) { + final N5Writer writer = new ShardedN5Writer("src/test/resources/test.n5"); + writer.remove(""); // Clear old when starting new test + return writer; + } + + final String basePath = new File(tempN5PathName()).toURI().normalize().getPath(); + try { + String uri = new URI("file", null, basePath, null).toString(); + return new ShardedN5Writer(uri); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + return null; + } + + private String tempN5PathName() { + + try { + final File tmpFile = Files.createTempDirectory("n5-shard-test-").toFile(); + tmpFile.delete(); + tmpFile.mkdir(); + tmpFile.deleteOnExit(); + return tmpFile.getCanonicalPath(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + }; + + public static GsonBuilder gsonBuilder() { + return new GsonBuilder(); + } + + @Parameterized.Parameters(name = "IndexLocation({0}), Index ByteOrder({1})") + public static Collection data() { + + final ArrayList params = new ArrayList<>(); + for (IndexLocation indexLoc : IndexLocation.values()) { + for (ByteOrder indexByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { + params.add(new Object[]{indexLoc, indexByteOrder}); + } + } + final int numParams = params.size(); + final Object[][] paramArray = new Object[numParams][]; + Arrays.setAll(paramArray, params::get); + return Arrays.asList(paramArray); + } + + @Parameterized.Parameter() + public IndexLocation indexLocation; + + @Parameterized.Parameter(1) + public ByteOrder indexByteOrder; + + @After + public void removeTempWriters() { + + tempN5Factory.removeTempWriters(); + } + + private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] blockSize) { + + return new DatasetAttributes( + dimensions, + shardSize, + blockSize, + DataType.UINT8); + } + + private DatasetAttributes getTestAttributes() { + + return getTestAttributes(new long[]{8, 8}, new int[]{4, 4}, new int[]{2, 2}); + } + + private DatasetAttributes getTestAttributes3d() { + + final int[] blockSize = {33, 22, 11}; + final int[] shardSize = {blockSize[0] * 2, blockSize[1] * 2, blockSize[2] * 2}; + return getTestAttributes(new long[]{10, 20, 30}, shardSize, blockSize); + } + + @Test + public void writeReadBlocksTest() { + + final N5Writer writer = tempN5Factory.createTempN5Writer(); + + final DatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + new int[]{2, 2} + ); + + final String dataset = "writeReadBlocks"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); + + final int[] blockSize = datasetAttributes.getBlockSize(); + final int numElements = blockSize[0] * blockSize[1]; + + final byte[] data = new byte[numElements]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)((100) + (10) + i); + } + + writer.writeBlocks( + dataset, + datasetAttributes, + /* shard (0, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + + /* shard (1, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), + + /* shard (2, 2) */ + new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) + ); + + final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); + + final String[][] keys = new String[][]{ + {dataset, "0", "0"}, + {dataset, "1", "0"}, + {dataset, "2", "2"} + }; + + // TODO check that these are the only keys + + for (String[] key : keys) { + final String shard = kva.compose(writer.getURI(), key); + Assert.assertTrue("Shard at" + Arrays.toString(key) + "Does not exist", kva.exists(shard)); + } + + final long[][] blockIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; + for (long[] blockIndex : blockIndices) { + final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + } + + final byte[] data2 = new byte[numElements]; + for (int i = 0; i < data2.length; i++) { + data2[i] = (byte)(10 + i); + } + writer.writeBlocks( + dataset, + datasetAttributes, + /* shard (0, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data2), + new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data2), + + /* shard (0, 1) */ + new ByteArrayDataBlock(blockSize, new long[]{0, 4}, data2), + new ByteArrayDataBlock(blockSize, new long[]{0, 5}, data2), + + /* shard (2, 2) */ + new ByteArrayDataBlock(blockSize, new long[]{10, 10}, data2) + ); + + final String[][] keys2 = new String[][]{ + {dataset, "0", "0"}, + {dataset, "1", "0"}, + {dataset, "0", "1"}, + {dataset, "2", "2"} + }; + + // TODO check that other keys do not exist + + for (String[] key : keys2) { + final String shard = kva.compose(writer.getURI(), key); + Assert.assertTrue("Shard at" + Arrays.toString(key) + "Does not exist", kva.exists(shard)); + } + + final long[][] oldBlockIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; + for (long[] blockIndex : oldBlockIndices) { + final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + } + + final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; + for (long[] blockIndex : newBlockIndices) { + final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])block.getData()); + } + } + // @Test // public void writeReadBlockTest() { // @@ -558,42 +563,42 @@ public class ShardTest { // IndexLocation.END) // ); // } -// -// /** -// * An N5Writer that serializing the sharding codecs, enabling testing of -// * shard functionality, despite the fact that the N5 format does not support -// * sharding. -// */ -// public static class ShardedN5Writer extends N5FSWriter { -// -// Gson gson; -// + + /** + * An N5Writer that serializing the sharding codecs, enabling testing of + * shard functionality, despite the fact that the N5 format does not support + * sharding. + */ + public static class ShardedN5Writer extends N5FSWriter { + + Gson gson; + // TestDatasetAttributesAdapter adapter = new TestDatasetAttributesAdapter(); -// -// public ShardedN5Writer(String basePath) { -// -// this(basePath, new GsonBuilder()); -// } -// -// public ShardedN5Writer(String basePath, GsonBuilder gsonBuilder) { -// -// super(basePath); -// gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); -// gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); + + public ShardedN5Writer(String basePath) { + + this(basePath, new GsonBuilder()); + } + + public ShardedN5Writer(String basePath, GsonBuilder gsonBuilder) { + + super(basePath); + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); // gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, new TestDatasetAttributesAdapter()); -// gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBlockCodecInfo.byteOrderAdapter); -// gsonBuilder.registerTypeHierarchyAdapter(ShardingCodec.IndexLocation.class, ShardingCodec.indexLocationAdapter); -// gsonBuilder.disableHtmlEscaping(); -// gson = gsonBuilder.create(); -// } -// -// @Override -// public Gson getGson() { -// -// // the super constructor needs the gson instance, unfortunately -// return gson == null ? super.gson : gson; -// } -// + gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBlockCodecInfo.byteOrderAdapter); + gsonBuilder.registerTypeHierarchyAdapter(DefaultShardCodecInfo.class, DefaultShardCodecInfo.adapter); + gsonBuilder.disableHtmlEscaping(); + gson = gsonBuilder.create(); + } + + @Override + public Gson getGson() { + + // the super constructor needs the gson instance, unfortunately + return gson == null ? super.gson : gson; + } + // @Override // public DatasetAttributes createDatasetAttributes(final JsonElement attributes) { // @@ -608,8 +613,8 @@ public class ShardTest { // // return adapter.deserialize(attributes, DatasetAttributes.class, context); // } -// } -// + } + // public static class TestDatasetAttributesAdapter extends DatasetAttributesAdapter { // // @Override @@ -628,12 +633,16 @@ public class ShardTest { // // final long[] dimensions = context.deserialize(obj.get(DatasetAttributes.DIMENSIONS_KEY), long[].class); // final int[] blockSize = context.deserialize(obj.get(DatasetAttributes.BLOCK_SIZE_KEY), int[].class); -// final int[] shardSize = context.deserialize(obj.get(DatasetAttributes.SHARD_SIZE_KEY), int[].class); -// // final DataType dataType = context.deserialize(obj.get(DatasetAttributes.DATA_TYPE_KEY), DataType.class); // // final CodecInfo[] codecs; -// if (obj.has(DatasetAttributes.CODEC_KEY)) { +// if (obj.has(DatasetAttributes.CODEC_KEY)) { DefaultShardCodecInfo shardCodecInfo = new DefaultShardCodecInfo( +// blockSize, +// new N5BlockCodecInfo(), +// new DataCodecInfo[]{new RawCompression()}, +// new RawBlockCodecInfo(), +// new DataCodecInfo[]{new RawCompression()}, +// IndexLocation.END); // codecs = context.deserialize(obj.get(DatasetAttributes.CODEC_KEY), CodecInfo[].class); // } else if (obj.has(DatasetAttributes.COMPRESSION_KEY)) { // final Compression compression @@ -652,7 +661,7 @@ public class ShardTest { // blockCodecInfo = null; // dataCodecInfos = (DataCodecInfo[])codecs; // } -// return new DatasetAttributes(dimensions, shardSize, blockSize, dataType, blockCodecInfo, dataCodecInfos); +// return new DatasetAttributes(dimensions, blockSize, dataType, blockCodecInfo, dataCodecInfos); // } // // @Override @@ -662,16 +671,35 @@ public class ShardTest { // obj.add(DatasetAttributes.DIMENSIONS_KEY, context.serialize(src.getDimensions())); // obj.add(DatasetAttributes.BLOCK_SIZE_KEY, context.serialize(src.getBlockSize())); // -// final int[] shardSize = src.getShardSize(); -// if (shardSize != null) { -// obj.add(DatasetAttributes.SHARD_SIZE_KEY, context.serialize(shardSize)); -// } +// obj.add(DatasetAttributes.DATA_TYPE_K// public static class TestDatasetAttributesAdapter extends DatasetAttributesAdapter {EY, context.serialize(src.getDataType())); +// +// +// final DataCodecInfo[] byteCodecs = src.getDataCodecInfos(); +// final BlockCodecInfo blockCodecInfo = src.getBlockCodecInfo(); +// final CodecInfo[] allCodecs = new CodecInfo[byteCodecs.length + 1]; +// allCodecs[0] = blockCodecInfo; +// System.arraycopy(byteCodecs, 0, allCodecs, 1, byteCodecs.length); // -// obj.add(DatasetAttributes.DATA_TYPE_KEY, context.serialize(src.getDataType())); +// // obj.add(DatasetAttributes.CODEC_KEY, context.serialize(concatenateCodecs(src))); // return obj; +// +//// final JsonElement out = context.serialize(src); +//// return out; // } // } +// +// +// private static CodecInfo[] concatenateCodecs(final DatasetAttributes attributes) { +// +// final DataCodecInfo[] byteCodecs = attributes.getDataCodecInfos(); +// final BlockCodecInfo blockCodecInfo = attributes.getBlockCodecInfo(); +// final CodecInfo[] allCodecs = new CodecInfo[byteCodecs.length + 1]; +// allCodecs[0] = blockCodecInfo; +// System.arraycopy(byteCodecs, 0, allCodecs, 1, byteCodecs.length); +// +// return allCodecs; +// } } From 2091dad4ed8682579d3a37c2dd71176262d9d2ee Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 6 Oct 2025 16:13:38 -0400 Subject: [PATCH 354/423] make DatasetAccess transient --- .../saalfeldlab/n5/DatasetAttributes.java | 34 +++---------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 1337e8604..86c176546 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -101,7 +101,8 @@ public class DatasetAttributes implements Serializable { private final BlockCodecInfo blockCodecInfo; private final DataCodecInfo[] dataCodecInfos; - private final DatasetAccess access; + + private transient final DatasetAccess access; public DatasetAttributes( final long[] dimensions, @@ -172,6 +173,8 @@ public DatasetAttributes( final int[] blockSize, final DataType dataType) { + // TODO add compression arg + this(dimensions, shardSize, dataType, defaultShardCodecInfo(blockSize)); } @@ -180,40 +183,13 @@ protected static BlockCodecInfo defaultShardCodecInfo(int[] innerBlockSize) { return new DefaultShardCodecInfo( innerBlockSize, - new N5BlockCodecInfo(), + new N5BlockCodecInfo(), // TODO call default method new DataCodecInfo[]{new RawCompression()}, new RawBlockCodecInfo(), new DataCodecInfo[]{new RawCompression()}, IndexLocation.END); } - private void validateBlockShardSizes(long[] dimensions, int[] shardSize, int[] blockSize) { - - final int nd = dimensions.length; - - if (blockSize.length != nd) - throw new IllegalArgumentException(String.format("Number of block dimensions (%d) must equal number of dimensions (%d).", - blockSize.length, nd)); - - if (shardSize.length != nd) - throw new IllegalArgumentException(String.format("Number of shard dimensions (%d) must equal number of dimensions (%d).", - shardSize.length, nd)); - - for (int i = 0; i < blockSize.length; i++) { - - if (blockSize[i] <= 0) - throw new IllegalArgumentException(String.format("Block size in dimension %d (%d) is <= 0", - i, blockSize[i])); - - if (shardSize[i] < blockSize[i]) - throw new IllegalArgumentException(String.format("Shard size in dimension %d (%d) is larger than the block size (%d)", - i, shardSize[i], blockSize[i])); - else if (shardSize[i] % blockSize[i] != 0) - throw new IllegalArgumentException(String.format("Shard size in dimension %d (%d) not a multiple of the block size (%d)", - i, shardSize[i], blockSize[i])); - } - } - protected BlockCodecInfo defaultArrayCodec() { return new N5BlockCodecInfo(); From ff994d4c4bf7b222d856fcfd6aea7669806d0a61 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 6 Oct 2025 16:11:02 -0400 Subject: [PATCH 355/423] feat: nested implementation of readBlocks/writeBlocks --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 30 +-- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 34 +-- .../org/janelia/saalfeldlab/n5/N5Writer.java | 1 + .../n5/shardstuff/DatasetAccess.java | 9 +- .../saalfeldlab/n5/shardstuff/Nesting.java | 30 ++- .../n5/shardstuff/ShardedDatasetAccess.java | 244 +++++++++++++++++- 6 files changed, 284 insertions(+), 64 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index d0aca8588..8ba870fc0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -32,9 +32,13 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.shardstuff.Nesting; +import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; import org.janelia.saalfeldlab.n5.shardstuff.PositionValueAccess; import com.google.gson.Gson; @@ -132,25 +136,13 @@ default List> readBlocks( final DatasetAttributes datasetAttributes, final List blockPositions) throws N5Exception { - // TODO -// if (datasetAttributes.isSharded()) { -// /* Group by shard position */ -// //TODO: make static? -// final Map> shardBlockMap = datasetAttributes.groupBlockPositions(blockPositions); -// final ArrayList> blocks = new ArrayList<>(); -// for( Map.Entry> e : shardBlockMap.entrySet()) { -// -// Shard currentShard = readShard(pathName, datasetAttributes, e.getKey().get()); -// if (currentShard == null) -// continue; -// -// for (final long[] blkPosition : e.getValue()) { -// blocks.add(currentShard.getBlock(currentShard.getRelativeBlockPosition(blkPosition))); -// } -// } -// return blocks; -// } - return GsonN5Reader.super.readBlocks(pathName, datasetAttributes, blockPositions); + try { + final PositionValueAccess posKva = PositionValueAccess.fromKva( + getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName)); + return datasetAttributes.getDatasetAccess().readBlocks(posKva, blockPositions); + } catch (N5Exception.N5NoSuchKeyException e) { + return null; + } } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 2d06d5b7d..2f7900b0d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -223,32 +223,14 @@ default void writeBlocks( final DatasetAttributes datasetAttributes, final DataBlock... dataBlocks) throws N5Exception { -// if (datasetAttributes.isSharded()) { -// /* Group blocks by shard index */ -// final Map>> shardBlockMap = datasetAttributes.groupBlocks( -// Arrays.stream(dataBlocks).collect(Collectors.toList())); -// -// //TODO: extract static method? -// for (final Entry>> e : shardBlockMap.entrySet()) { -// -// final long[] shardPosition = e.getKey().get(); -// final Shard currentShard = readShard(datasetPath, datasetAttributes, shardPosition); -// final InMemoryShard newShard; -// if (currentShard != null) { -// newShard = InMemoryShard.fromShard(currentShard); -// } else { -// newShard = new InMemoryShard<>(datasetAttributes, shardPosition); -// } -// -// for (DataBlock blk : e.getValue()) -// newShard.addBlock(blk); -// -// writeShard(datasetPath, datasetAttributes, newShard); -// } -// -// } else { - GsonN5Writer.super.writeBlocks(datasetPath, datasetAttributes, dataBlocks); -// } + try { + final PositionValueAccess posKva = PositionValueAccess.fromKva( + getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath)); + datasetAttributes.getDatasetAccess().writeBlocks(posKva, Arrays.asList(dataBlocks)); + } catch (final UncheckedIOException e) { + throw new N5IOException( + "Failed to write blocks into dataset " + datasetPath, e); + } } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index cf85bce56..4aea4ccd0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -39,6 +39,7 @@ import java.io.ObjectOutputStream; import java.io.Serializable; import java.io.UncheckedIOException; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java index 768f9c5b2..c8ffd455a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java @@ -2,6 +2,9 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; + +import java.util.List; /** * Wrap an instantiated DataBlock/shard codec hierarchy to implement (single and @@ -18,7 +21,7 @@ public interface DatasetAccess { void deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; - // TODO: batch read/write/delete methods -// List> readBlocks(PositionValueAccess kva, List positions); -// void writeBlocks(PositionValueAccess kva, List> blocks); + List> readBlocks(PositionValueAccess kva, List positions); + + void writeBlocks(PositionValueAccess kva, List> blocks); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java index 332107ed5..3367879f0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java @@ -18,7 +18,7 @@ // ==> postpone until necessary // [ ] equals / hashcode // [ ] should we have prefix()? suffix()? head()? tail()? -// [ ] Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...). +// [+] Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...). // For nested = {X,Y,Z} compare by Z, then Y, then X. // For X = {x,y,z} compare by z, then y, then x. (flattening order) // @@ -39,7 +39,7 @@ public static void main(String[] args) { - public static class NestedPosition { + public static class NestedPosition implements Comparable { private final NestedGrid grid; private final long[] position; @@ -94,7 +94,7 @@ public long[] relative(final int level) { * @return absolute grid position */ public long[] absolute(final int level) { - return grid.relativePosition(position, level); + return grid.absolutePosition(position, level); } public long[] key() { @@ -115,13 +115,35 @@ public String toString() { return sb.toString(); } + @Override public int compareTo(NestedPosition o) { + + final int dimensionInequality = Integer.compare(numDimensions(), o.numDimensions()); + if (dimensionInequality != 0) + return dimensionInequality; + + final int levelInequality = Integer.compare(level, o.level); + if (levelInequality != 0) + return levelInequality; + + final long[] otherAbsPos = o.absolute(level); + final long[] absPos = absolute(level); + + for (int i = absPos.length - 1; i >= 0; --i) { + final long diff = absPos[i] - otherAbsPos[i]; + if (diff != 0) + return (int)diff; + } + + return 0; + } + // TODO: equals() and hashCode() // TODO: should we have prefix()? suffix()? head()? tail()? } /** - * TODO + * TODO better property names */ public static class NestedGrid { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java index 78a3198f3..b39bef069 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java @@ -12,6 +12,13 @@ import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.TreeMap; +import java.util.stream.Collectors; + public class ShardedDatasetAccess implements DatasetAccess { private final NestedGrid grid; @@ -50,23 +57,93 @@ private DataBlock readBlockRecursive( } } + @Override + public List> readBlocks(PositionValueAccess pva, List positions) { + + if (grid.numLevels() == 1) { + return positions.stream().map(it -> readBlock(pva, it)).collect(Collectors.toList()); + } + + final List blockPositions = positions.stream().map(it -> new NestedPosition(grid, it)).collect(Collectors.toList()); + + final int outermostLevel = grid.numLevels() - 1; + final Collection> blocksPerOutermostShard = groupInnerPositions(grid, blockPositions, outermostLevel); + + final ArrayList> blocks = new ArrayList<>(blockPositions.size()); + for (List blocksInSingleShard : blocksPerOutermostShard) { + if (blocksInSingleShard.isEmpty()) + continue; + + final NestedPosition firstBlock = blocksInSingleShard.get(0); + final ReadData readData = pva.get(firstBlock.key()); + final List> shardBlocks = readShardRecursive(readData, blocksInSingleShard, outermostLevel); + + blocks.addAll(shardBlocks); + } + + //TODO Caleb: No guarantee of order; If we want that, we need to sort the result. + //TODO Caleb: Also no guarantee of uniqueness. May want to use a Set instead of a list if we care. + return blocks; + } + + /** + * Bulk Read operation on a shard. `positions` MUST all be in the same shard. + * That is, for each `position` in `positions`, `position.absolute(level)` must be the same. + * + * @param readData for the corresponding shard + * @param positions of blocks within the shard to be read + * @param level of the shard + * @return list of blocks read from a single shard + */ + private List> readShardRecursive( + final ReadData readData, + final List positions, + final int level + ) { + // Cannot have a shard at level 0 + if (readData == null || level == 0) { + return null; + } + + if (positions.isEmpty()) { + return Collections.emptyList(); + } + + + final NestedPosition firstBlock = positions.get(0); + final long[] shardPosition = firstBlock.absolute(level); + + final BlockCodec codec = (BlockCodec) codecs[level]; + final RawShard shard = codec.decode(readData, shardPosition).getData(); + + final ArrayList> blocks = new ArrayList<>(positions.size()); + if (level == 1) { + final int innerMostLevel = 0; + //Base case; read the blocks + for (NestedPosition blockPosition : positions) { + final long[] elementPos = blockPosition.relative(innerMostLevel); + final ReadData elementData = shard.getElementData(elementPos); + final DataBlock block = readBlockRecursive(elementData, blockPosition, innerMostLevel); + blocks.add(block); + } + } else { + // group the blocks by shard for next level, and call again for each nested shard + final Collection> nextLevelShards = groupInnerPositions(grid, positions, level - 1); + for (List innerPositions : nextLevelShards) { + final List> innerBlocks = readShardRecursive(readData, innerPositions, level - 1); + blocks.addAll(innerBlocks); + } + } + + return blocks; + } + @Override public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException { final NestedPosition position = new NestedPosition(grid, dataBlock.getGridPosition()); final long[] key = position.key(); - // need to read the shard anyway, and currently (Sept 24 2025) - // have no way to tell if they key exist from what is in this method except to attempt - // to materialize and catch the N5NoSuchKeyException - ReadData existingData = null; - try { - existingData = pva.get(key); - if (existingData != null) - existingData.materialize(); - } catch (N5NoSuchKeyException e) { - existingData = null; - } - + final ReadData existingData = getExistingReadData(pva, key); final ReadData modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1); pva.put(key, modifiedData); } @@ -97,6 +174,81 @@ private ReadData writeBlockRecursive( } } + @Override public void writeBlocks(PositionValueAccess pva, List> dataBlocks) { + + if (grid.numLevels() == 1) { + dataBlocks.forEach(it -> writeBlock(pva, it)); + return; + } + + final List> nestedPositionDataBlocks = dataBlocks.stream() + .map(it -> new NestedPositionDataBlock<>(grid, it)) + .collect(Collectors.toList()); + + final int outermostLevel = grid.numLevels() - 1; + final Collection>> dataBlocksPerShard = groupInnerPositions(grid, nestedPositionDataBlocks, outermostLevel); + + for (List> dataBlocksForShard : dataBlocksPerShard) { + if (dataBlocksForShard.isEmpty()) + continue; + + final NestedPositionDataBlock firstDataBlock = dataBlocksForShard.get(0); + final long[] shardKey = firstDataBlock.key(); + + final ReadData existingReadData = getExistingReadData(pva, shardKey); + final ReadData writeShardReadData = writeShardRecursive(existingReadData, dataBlocksForShard, outermostLevel); + + pva.put(shardKey, writeShardReadData); + } + } + + private ReadData writeShardRecursive( + final ReadData existingShard, + final List> dataBlocks, + final int level) { + + // cannot have shard level 0, or nothing to write + if (level == 0 || dataBlocks.isEmpty()) { + return null; + } + + final BlockCodec codec = (BlockCodec)codecs[level]; + final NestedPositionDataBlock firstBlock = dataBlocks.get(0); + final long[] shardPosition = firstBlock.absolute(level); + + final RawShard shard; + if (existingShard == null) + shard = new RawShard(grid.relativeBlockSize(level)); + else + shard = codec.decode(existingShard, shardPosition).getData(); + + if (level == 1) { + final int innerMostLevel = 0; + // Base case, write the blocks + for (NestedPositionDataBlock nestedPosDataBlock : dataBlocks) { + final DataBlock dataBlock = nestedPosDataBlock.getDataBlock(); + + final long[] blockRelativePos = nestedPosDataBlock.relative(innerMostLevel); + final ReadData blockReadData = shard.getElementData(blockRelativePos); + final ReadData modifiedShardBlock = writeBlockRecursive(blockReadData, dataBlock, nestedPosDataBlock, innerMostLevel); + shard.setElementData(modifiedShardBlock, blockRelativePos); + } + } else { + final Collection>> dataBlocksForInnerShards = groupInnerPositions(grid, dataBlocks, level - 1); + for (List> innerShardDataBlocks : dataBlocksForInnerShards) { + if (innerShardDataBlocks.isEmpty()) + continue; + + final ReadData innerShardReadData = writeShardRecursive(existingShard, innerShardDataBlocks, level - 1); + + final NestedPositionDataBlock firstInnerNestedPosBlock = innerShardDataBlocks.get(0); + final long[] relPosInShard = firstInnerNestedPosBlock.relative(level - 1); + shard.setElementData(innerShardReadData, relPosInShard); + } + } + return codec.encode(new RawShardDataBlock(shardPosition, shard)); + } + @Override public void deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { final NestedPosition position = new NestedPosition(grid, gridPosition); @@ -197,4 +349,72 @@ private static int nestingDepth(BlockCodecInfo info) { return 1; } } + + private static ReadData getExistingReadData(final PositionValueAccess pva, final long[] key) { + // need to read the shard anyway, and currently (Sept 24 2025) + // have no way to tell if they key exist from what is in this method except to attempt + // to materialize and catch the N5NoSuchKeyException + try { + ReadData existingData = pva.get(key); + if (existingData != null) + existingData.materialize(); + return existingData; + } catch (N5NoSuchKeyException e) { + return null; + } + } + + /** + * Sort a list of {@link NestedPosition}s by their parent level {@link NestedPosition}. + * nestedPositions are grouped at `outerLevel`. + * If {@code NestedPosition.level()} level is already equivalent to {@code NestedGrid.numLevels() - 1}, nestedPositions is returned. + * + * @param grid to sort the blocky by + * @param nestedPositions to group per shard. + * @param outerLevel of the outerLevel shard position to group by. must be in range {@code [1, NestedGrid.numLevels() - 1]} + * @return map of outerLevel shard positions to inner level block positions + */ + private static Collection> groupInnerPositions(final NestedGrid grid, final List nestedPositions, final int outerLevel) { + + if (outerLevel < 1 || outerLevel >= grid.numLevels()) + throw new IllegalArgumentException("outerLevel must be in range [1, grid.numLevels() - 1]"); + + if (nestedPositions.isEmpty()) + return Collections.emptyList(); + + final TreeMap> blocksPerShard = new TreeMap<>(); + for (T nestedPosition : nestedPositions) { + final NestedPosition outerNestedPosition; + if (nestedPosition.level() == outerLevel) + outerNestedPosition = nestedPosition; + else + outerNestedPosition = new NestedPosition(grid, nestedPosition.absolute(0), outerLevel); + + + final List blocks = blocksPerShard.computeIfAbsent(outerNestedPosition, it -> new ArrayList<>()); + blocks.add(nestedPosition); + } + return blocksPerShard.values(); + } + + /** + * NestedPosition wrapper for a DataBlock. Useful for grouping DataBlock by nested shard position. + * + * @param type of the datablock + */ + private static class NestedPositionDataBlock extends NestedPosition { + + private final DataBlock dataBlock; + + private NestedPositionDataBlock(NestedGrid grid, DataBlock dataBlock) { + + super(grid, dataBlock.getGridPosition(), 0); + this.dataBlock = dataBlock; + } + + private DataBlock getDataBlock() { + + return dataBlock; + } + } } From 3dcb8f0c28271c54e845d74641aae63e6ca81a63 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 8 Oct 2025 18:06:28 -0400 Subject: [PATCH 356/423] test: start backward compatibility tests * check that old data can be read * check that written data matches some specfied version * adds legacy data from previous major versions * adds scripts to create data for future versions --- .../n5/backward/CompatibilityTest.java | 141 ++++++++++++++++++ .../n5/backward/CreateSampleData.java | 69 +++++++++ .../backward/data-1.5.0.n5/attributes.json | 1 + .../resources/backward/data-1.5.0.n5/raw/0/0 | Bin 0 -> 32 bytes .../resources/backward/data-1.5.0.n5/raw/0/1 | Bin 0 -> 17 bytes .../resources/backward/data-1.5.0.n5/raw/1/0 | Bin 0 -> 20 bytes .../resources/backward/data-1.5.0.n5/raw/1/1 | Bin 0 -> 14 bytes .../data-1.5.0.n5/raw/attributes.json | 1 + .../backward/data-2.5.1.n5/attributes.json | 1 + .../resources/backward/data-2.5.1.n5/raw/0/0 | Bin 0 -> 32 bytes .../resources/backward/data-2.5.1.n5/raw/0/1 | Bin 0 -> 17 bytes .../resources/backward/data-2.5.1.n5/raw/1/0 | Bin 0 -> 20 bytes .../resources/backward/data-2.5.1.n5/raw/1/1 | Bin 0 -> 14 bytes .../data-2.5.1.n5/raw/attributes.json | 1 + .../backward/data-3.1.3.n5/attributes.json | 1 + .../resources/backward/data-3.1.3.n5/raw/0/0 | Bin 0 -> 32 bytes .../resources/backward/data-3.1.3.n5/raw/0/1 | Bin 0 -> 17 bytes .../resources/backward/data-3.1.3.n5/raw/1/0 | Bin 0 -> 20 bytes .../resources/backward/data-3.1.3.n5/raw/1/1 | Bin 0 -> 14 bytes .../data-3.1.3.n5/raw/attributes.json | 1 + 20 files changed, 216 insertions(+) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java create mode 100644 src/test/resources/backward/data-1.5.0.n5/attributes.json create mode 100644 src/test/resources/backward/data-1.5.0.n5/raw/0/0 create mode 100644 src/test/resources/backward/data-1.5.0.n5/raw/0/1 create mode 100644 src/test/resources/backward/data-1.5.0.n5/raw/1/0 create mode 100644 src/test/resources/backward/data-1.5.0.n5/raw/1/1 create mode 100644 src/test/resources/backward/data-1.5.0.n5/raw/attributes.json create mode 100644 src/test/resources/backward/data-2.5.1.n5/attributes.json create mode 100644 src/test/resources/backward/data-2.5.1.n5/raw/0/0 create mode 100644 src/test/resources/backward/data-2.5.1.n5/raw/0/1 create mode 100644 src/test/resources/backward/data-2.5.1.n5/raw/1/0 create mode 100644 src/test/resources/backward/data-2.5.1.n5/raw/1/1 create mode 100644 src/test/resources/backward/data-2.5.1.n5/raw/attributes.json create mode 100644 src/test/resources/backward/data-3.1.3.n5/attributes.json create mode 100644 src/test/resources/backward/data-3.1.3.n5/raw/0/0 create mode 100644 src/test/resources/backward/data-3.1.3.n5/raw/0/1 create mode 100644 src/test/resources/backward/data-3.1.3.n5/raw/1/0 create mode 100644 src/test/resources/backward/data-3.1.3.n5/raw/1/1 create mode 100644 src/test/resources/backward/data-3.1.3.n5/raw/attributes.json diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java new file mode 100644 index 000000000..4cdc1a272 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java @@ -0,0 +1,141 @@ +package org.janelia.saalfeldlab.n5.backward; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Files; +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.N5FSReader; +import org.janelia.saalfeldlab.n5.N5FSWriter; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.junit.Test; + +import com.google.gson.JsonElement; + +public class CompatibilityTest { + + String[][] readVersionsDataset = { + {"data-1.5.0.n5", "raw"}, + {"data-2.5.1.n5", "raw"}, + {"data-3.1.3.n5", "raw"} }; + + String writeVersion = "data-3.1.3.n5"; + String writeDataset = "raw"; + String[] writePathsToTest = {"0/0", "0/1", "1/0", "1/1"}; + + @Test + public void testBackwardReads() throws NumberFormatException, IOException { + + for (String[] versionDset : readVersionsDataset) + backwardReadHelper(versionDset[0], versionDset[1]); + } + + public void backwardReadHelper(final String base, final String dsetPath) throws NumberFormatException, IOException { + + final N5FSReader n5 = new N5FSReader("src/test/resources/backward/" + base); + assertTrue(n5.datasetExists(dsetPath)); + final DatasetAttributes attrs = n5.getDatasetAttributes(dsetPath); + + // equivalent to the assertTrue above, but be extra sure + assertNotNull(attrs); + + byte value = 0; + long[] p = new long[2]; + + DataBlock b00 = n5.readBlock(dsetPath, attrs, p); + assertNotNull(b00); + assertArrayEquals(new int[]{5,4}, b00.getSize()); + assertArrayEquals(expectedData(20, value), b00.getData()); + + p[0] = 1; + p[1] = 0; + value++; + DataBlock b10 = n5.readBlock(dsetPath, attrs, p); + assertNotNull(b10); + assertArrayEquals(new int[]{2,4}, b10.getSize()); + assertArrayEquals(expectedData(8, value), b10.getData()); + + p[0] = 0; + p[1] = 1; + value++; + DataBlock b01 = n5.readBlock(dsetPath, attrs, p); + assertNotNull(b01); + assertArrayEquals(new int[]{5,1}, b01.getSize()); + assertArrayEquals(expectedData(5, value), b01.getData()); + + p[0] = 1; + p[1] = 1; + value++; + DataBlock b11 = n5.readBlock(dsetPath, attrs, p); + assertNotNull(b11); + assertArrayEquals(new int[]{2,1}, b11.getSize()); + assertArrayEquals(expectedData(2, value), b11.getData()); + + n5.close(); + } + + @Test + public void testBlockData() throws IOException { + + final N5FSReader n5Legacy = new N5FSReader("src/test/resources/backward/" + writeVersion); + final URI uriLegacy = n5Legacy.getURI(); + + final File basePath = Files.createTempDirectory("n5-blockDataTest-").toFile(); + basePath.delete(); + basePath.mkdir(); + basePath.deleteOnExit(); + + N5FSWriter n5My = CreateSampleData.createSampleData( + basePath.getCanonicalPath(), writeDataset, new RawCompression()); + URI uriMy = n5My.getURI(); + + // check attributes + final JsonElement attrsLegacy = ((GsonKeyValueN5Reader)n5Legacy).getAttributes(writeDataset); + final JsonElement attrsMy = ((GsonKeyValueN5Reader)n5My).getAttributes(writeDataset); + assertEquals(attrsLegacy, attrsMy); + + final KeyValueAccess kva = n5My.getKeyValueAccess(); + for (final String path : writePathsToTest) { + final byte[] dataMy = read(kva, kva.compose(uriMy, writeDataset, path)); + final byte[] dataLegacy = read(kva, kva.compose(uriLegacy, writeDataset, path)); + assertArrayEquals(dataLegacy, dataMy); + } + + n5My.remove(); + n5My.close(); + n5Legacy.close(); + } + + private byte[] read(KeyValueAccess kva, String path) { + + int N = (int)kva.size(path); + byte[] data = new byte[N]; + try (LockedChannel ch = kva.lockForReading(path); + InputStream is = ch.newInputStream();) { + + is.read(data); + } catch (IOException e) { + return null; + } + return data; + } + + private static byte[] expectedData(int size, byte value ) { + byte[] data = new byte[size]; + Arrays.fill(data, value); + return data; + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java new file mode 100644 index 000000000..73334cdcd --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java @@ -0,0 +1,69 @@ +package org.janelia.saalfeldlab.n5.backward; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.Compression; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5FSWriter; +import org.janelia.saalfeldlab.n5.RawCompression; + +public class CreateSampleData { + + public static void main(String[] args) throws IOException { + + File f = new File("src/test/resources/data-4.0.0-alpha-X.n5"); + System.out.println(f.getCanonicalPath()); + createSampleData(f.getCanonicalPath(), "raw", new RawCompression()); + } + + public static N5FSWriter createSampleData(String baseDir, String dataset, Compression compression) throws IOException { + + N5FSWriter n5 = new N5FSWriter(baseDir); + final String dsetPath = compression.getType(); + + long[] dimensions = new long[]{7, 5}; + int[] blkSizeDset = new int[]{5, 4}; + int[] blkSize = new int[]{5, 4}; + + final DatasetAttributes attrs = new DatasetAttributes(dimensions, blkSizeDset, DataType.UINT8, compression); + n5.createDataset(dsetPath, attrs); + + byte val = 0; + long[] pos = new long[]{0, 0}; + n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + + pos[0] = 1; + pos[1] = 0; + blkSize[0] = 2; + blkSize[1] = 4; + val++; + n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + + pos[0] = 0; + pos[1] = 1; + blkSize[0] = 5; + blkSize[1] = 1; + val++; + n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + + pos[0] = 1; + pos[1] = 1; + blkSize[0] = 2; + blkSize[1] = 1; + val++; + n5.writeBlock(dsetPath, attrs, createDataBlock( blkSize, pos, val )); + + return n5; + } + + public static ByteArrayDataBlock createDataBlock(int[] size, long[] gridPosition, byte value) throws IOException { + int N = Arrays.stream(size).reduce(1, (x,y) -> x*y); + final byte[] data = new byte[N]; + Arrays.fill(data, value); + return new ByteArrayDataBlock(size, gridPosition, data); + } +} diff --git a/src/test/resources/backward/data-1.5.0.n5/attributes.json b/src/test/resources/backward/data-1.5.0.n5/attributes.json new file mode 100644 index 000000000..c050c3d21 --- /dev/null +++ b/src/test/resources/backward/data-1.5.0.n5/attributes.json @@ -0,0 +1 @@ +{"n5":"1.5.0"} \ No newline at end of file diff --git a/src/test/resources/backward/data-1.5.0.n5/raw/0/0 b/src/test/resources/backward/data-1.5.0.n5/raw/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..a129c42df4f256d1b075f9af34ea7c8440562ebb GIT binary patch literal 32 UcmZQzU|?ckU| Date: Tue, 14 Oct 2025 16:17:22 -0400 Subject: [PATCH 357/423] style: rm unused dependencies --- .../janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java | 4 ---- .../java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java | 1 - 2 files changed, 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index 697d2dbcd..3801e671b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -28,10 +28,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Arrays; - import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import com.google.gson.Gson; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java index 24e34fa2f..44d84662d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java @@ -1,6 +1,5 @@ package org.janelia.saalfeldlab.n5.codec; -import java.util.Arrays; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; From 493db08e279ebe577cdf90f001c8b717495835aa Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 14 Oct 2025 16:23:03 -0400 Subject: [PATCH 358/423] remove read/write/delete Shard interface methods --- .../org/janelia/saalfeldlab/n5/N5Reader.java | 11 -------- .../org/janelia/saalfeldlab/n5/N5Writer.java | 27 ------------------- 2 files changed, 38 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index d6bae4ad7..76fc2fbb2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -301,17 +301,6 @@ DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception; - /** - * Reads the {@link Shard} at the corresponding grid position. - * - * @param the data access type for the blocks in the shard - * @param datasetPath to read the shard from - * @param datasetAttributes for the shard - * @param shardGridPosition of the shard we are reading - * @return the shard - */ - Shard readShard(final String datasetPath, final DatasetAttributes datasetAttributes, long... shardGridPosition); - /** * Reads multiple {@link DataBlock}s. *

      diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 4aea4ccd0..e1e9cf455 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -285,20 +285,6 @@ default void writeBlocks( writeBlock(datasetPath, datasetAttributes, block); } - /** - * Writes a {@link Shard}. - * - * @param datasetPath dataset path - * @param datasetAttributes the dataset attributes - * @param shard the shard - * @param the data block data type - * @throws N5Exception the exception - */ - void writeShard( - final String datasetPath, - final DatasetAttributes datasetAttributes, - final Shard shard) throws N5Exception; - /** * Deletes the block at {@code gridPosition}. * @@ -312,19 +298,6 @@ boolean deleteBlock( final String datasetPath, final long... gridPosition) throws N5Exception; - /** - * Deletes the shard at {@code gridPosition}. - * - * @param datasetPath dataset path - * @param gridPosition position of shard to be deleted - * @throws N5Exception if the block exists but could not be deleted - * - * @return {@code true} if the shard at {@code gridPosition} existed and was deleted. - */ - boolean deleteShard( - final String datasetPath, - final long... gridPosition) throws N5Exception; - /** * Save a {@link Serializable} as an N5 {@link DataBlock} at a given offset. * The From 901642c0d78b41e5d7084fa898ad2cb3e503c9c0 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 14 Oct 2025 16:24:14 -0400 Subject: [PATCH 359/423] test: ignore testWriteInvalidBlock for the moment --- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index fa2004357..19d71ba74 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -56,12 +56,11 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5ClassCastException; import org.janelia.saalfeldlab.n5.N5Reader.Version; -import org.janelia.saalfeldlab.n5.shard.InMemoryShard; -import org.janelia.saalfeldlab.n5.shard.Shard; import org.janelia.saalfeldlab.n5.url.UriAttributeTest; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import com.google.gson.GsonBuilder; @@ -440,6 +439,7 @@ public void testWriteReadSerializableBlock() throws ClassNotFoundException { } @Test + @Ignore // TODO public void testWriteInvalidBlock() { final Compression compression = getCompressions()[0]; From a379ed400bc17351b54d85019b787aa7e08de5ab Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 14 Oct 2025 16:29:44 -0400 Subject: [PATCH 360/423] remove read/write Shard implementations --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 18 ----------------- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 20 ------------------- 2 files changed, 38 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 8ba870fc0..d0deab06a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -96,24 +96,6 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { } } - default Shard readShard( - final String keyPath, - final DatasetAttributes datasetAttributes, - long... shardGridPosition) { - - // TODO - -// final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(keyPath), shardGridPosition); -// try { -// final ReadData readData = getKeyValueAccess().createReadData(path).materialize(); -// return new VirtualShard<>( datasetAttributes, shardGridPosition, readData); -// } catch (N5Exception.N5NoSuchKeyException e) { -// return null; -// } - - return null; - } - @Override default DataBlock readBlock( final String pathName, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 2f7900b0d..441225767 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -251,26 +251,6 @@ default void writeBlock( } - @Override - default void writeShard( - final String path, - final DatasetAttributes datasetAttributes, - final Shard shard) throws N5Exception { - - // TODO - -// final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shard.getGridPosition()); -// try ( -// final LockedChannel channel = getKeyValueAccess().lockForWriting(shardPath); -// final OutputStream shardOut = channel.newOutputStream() -// ) { -// shard.createReadData().writeTo(shardOut); -// } catch (final IOException | UncheckedIOException e) { -// throw new N5IOException( -// "Failed to write shard " + Arrays.toString(shard.getGridPosition()) + " into dataset " + path, e); -// } - } - @Override default boolean remove(final String path) throws N5Exception { From feb70afebd049746b95c954952bc5b11a47b9341 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 14 Oct 2025 16:32:12 -0400 Subject: [PATCH 361/423] createDataset takes Block/DataCodecInfos * and has a default BlockCodecInfo * guard against null dataCodecInfos --- .../saalfeldlab/n5/DatasetAttributes.java | 10 ++-- .../org/janelia/saalfeldlab/n5/N5Writer.java | 47 ++++++++++++------- .../n5/http/HttpReaderFsWriter.java | 12 ++--- 3 files changed, 41 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 86c176546..2c674efec 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -115,9 +115,13 @@ public DatasetAttributes( this.dataType = dataType; this.blockCodecInfo = blockCodecInfo == null ? defaultBlockCodecInfo() : blockCodecInfo; - this.dataCodecInfos = Arrays.stream(dataCodecInfos) - .filter(it -> it != null && !(it instanceof RawCompression)) - .toArray(DataCodecInfo[]::new); + + if (dataCodecInfos == null) + this.dataCodecInfos = new DataCodecInfo[0]; + else + this.dataCodecInfos = Arrays.stream(dataCodecInfos) + .filter(it -> it != null && !(it instanceof RawCompression)) + .toArray(DataCodecInfo[]::new); final ShardedDatasetAccess shardAccess = ShardedDatasetAccess.create( getDataType(), outerBlockSize, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index e1e9cf455..6d5e155b9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -224,32 +224,43 @@ default void createDataset( * @param dimensions the dataset dimensions * @param blockSize the block size * @param dataType the data type - * @param codecs codecs to encode/decode with - * @throws N5Exception the exception + * @param blockCodecInfo the block codec + * @param dataCodecs data codecs */ default void createDataset( final String datasetPath, final long[] dimensions, final int[] blockSize, final DataType dataType, - final CodecInfo... codecs) throws N5Exception { + final DataCodecInfo... dataCodecInfos) throws N5Exception { - final BlockCodecInfo blockCodecInfo; - final DataCodecInfo[] dataCodecs; - if (codecs == null || codecs.length == 0) { - blockCodecInfo = null; - dataCodecs = new DataCodecInfo[0]; - } else if (codecs[0] instanceof BlockCodecInfo) { - blockCodecInfo = (BlockCodecInfo) codecs[0]; - dataCodecs = new DataCodecInfo[codecs.length - 1]; - System.arraycopy(codecs, 1, dataCodecs, 0, dataCodecs.length); - } else { - blockCodecInfo = null; - dataCodecs = new DataCodecInfo[codecs.length]; - System.arraycopy(codecs, 0, dataCodecs, 0, dataCodecs.length); - } + // TODO default block codec? + // TODO better doc + createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, dataCodecInfos)); + } + + /** + * Creates a dataset. This does not create any data but the path and + * mandatory attributes only. + * + * @param datasetPath dataset path + * @param dimensions the dataset dimensions + * @param blockSize the block size + * @param dataType the data type + * @param blockCodecInfo the block codec + * @param dataCodecs data codecs + */ + default void createDataset( + final String datasetPath, + final long[] dimensions, + final int[] blockSize, + final DataType dataType, + final BlockCodecInfo blockCodecInfo, + final DataCodecInfo[] dataCodecInfos) throws N5Exception { - createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, blockCodecInfo, dataCodecs)); + // TODO default block codec? + // TODO better doc + createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, blockCodecInfo, dataCodecInfos)); } /** diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index 38ceee229..b05593e85 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -39,7 +39,9 @@ import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.shard.Shard; import java.io.Serializable; @@ -303,13 +305,9 @@ public HttpRead writer.writeBlocks(datasetPath, datasetAttributes, dataBlocks); } - @Override public void writeShard(String path, DatasetAttributes datasetAttributes, Shard shard) throws N5Exception { + @Override public void createDataset(String datasetPath, long[] dimensions, int[] blockSize, DataType dataType, + BlockCodecInfo blockCodecInfo, DataCodecInfo[] dataCodecInfos) throws N5Exception { - writer.writeShard(path, datasetAttributes, shard); - } - - @Override public void createDataset(String datasetPath, long[] dimensions, int[] blockSize, DataType dataType, CodecInfo... codecs) throws N5Exception { - - writer.createDataset(datasetPath, dimensions, blockSize, dataType, codecs); + writer.createDataset(datasetPath, dimensions, blockSize, dataType, blockCodecInfo, dataCodecInfos); } } From 98d7618080fca32357f382822bb99a38ee297a11 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 14 Oct 2025 16:32:32 -0400 Subject: [PATCH 362/423] n5.list returns a sorted list --- src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 770cece13..2a60743a0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -28,6 +28,7 @@ */ package org.janelia.saalfeldlab.n5.cache; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashSet; @@ -183,6 +184,7 @@ public String[] list(final String normalPathKey) { for (final String child : cacheInfo.children) { children[i++] = child; } + Arrays.sort(children); return children; } From e73ecd3ee41b88e6e02fbf2ff07ba20bd339696b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 14 Oct 2025 16:34:06 -0400 Subject: [PATCH 363/423] DatasetAttributes clean up --- .../saalfeldlab/n5/DatasetAttributes.java | 36 +++---------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 2c674efec..9c7764406 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -36,13 +36,9 @@ import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.shard.BlockAsShardCodec; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec; -import org.janelia.saalfeldlab.n5.util.Position; import org.janelia.saalfeldlab.n5.shardstuff.DatasetAccess; import org.janelia.saalfeldlab.n5.shardstuff.DefaultShardCodecInfo; import org.janelia.saalfeldlab.n5.shardstuff.ShardCodecInfo; @@ -51,13 +47,9 @@ import java.io.Serializable; import java.lang.reflect.Type; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.stream.IntStream; +import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; @@ -138,15 +130,15 @@ public DatasetAttributes( * @param dimensions the dimensions of the dataset * @param blockSize the size of the blocks in the dataset * @param dataType the data type of the dataset - * @param compression the codecs used encode/decode the data + * @param dataCodecInfos the codecs used encode/decode the data */ public DatasetAttributes( final long[] dimensions, final int[] blockSize, final DataType dataType, - final DataCodecInfo compression) { + final DataCodecInfo... dataCodecInfos) { - this(dimensions, blockSize, dataType, null, compression); + this(dimensions, blockSize, dataType, null, dataCodecInfos); } /** @@ -161,7 +153,7 @@ public DatasetAttributes( final int[] blockSize, final DataType dataType) { - this(dimensions, blockSize, dataType, null); + this(dimensions, blockSize, dataType, new DataCodecInfo[0]); } /** @@ -194,12 +186,6 @@ protected static BlockCodecInfo defaultShardCodecInfo(int[] innerBlockSize) { IndexLocation.END); } - protected BlockCodecInfo defaultArrayCodec() { - - return new N5BlockCodecInfo(); - } - - protected BlockCodecInfo defaultBlockCodecInfo() { return new N5BlockCodecInfo(); @@ -242,18 +228,6 @@ public int[] getBlockSize() { // return blocksPerShard; // } - /** - * Returns the number of blocks per dimension for this dataset. - * - * @return blocks per dataset - */ - public long[] blocksPerDataset() { - - return IntStream.range(0, getNumDimensions()) - .mapToLong(i -> (long)Math.ceil((double)getDimensions()[i] / getBlockSize()[i])) - .toArray(); - } - // /** // * Returns the number of shards per dimension for this dataset. // * From f5385ba686be1123dde7871f791dc04c6a45b739 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 14 Oct 2025 16:34:38 -0400 Subject: [PATCH 364/423] feat: add block / data codecinfo getters * useful for zarr --- .../org/janelia/saalfeldlab/n5/DatasetAttributes.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 9c7764406..4989414c0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -394,6 +394,16 @@ public DatasetAccess getDatasetAccess() { return (DatasetAccess)access; } + public BlockCodecInfo getBlockCodecInfo() { + + return blockCodecInfo; + } + + public DataCodecInfo[] getDataCodecInfos() { + + return dataCodecInfos; + } + public HashMap asMap() { final HashMap map = new HashMap<>(); From 6e2dfd364acb68c859e281d3c4f7b514f05e33db Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 14 Oct 2025 16:41:38 -0400 Subject: [PATCH 365/423] feat: DatasetAttributes can map grid positions to relative path * necessary for zarr's dimensionSeparator * use this in PositionValueAccess --- .../saalfeldlab/n5/DatasetAttributes.java | 6 ++++- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 16 +++++-------- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 6 +++-- .../n5/shardstuff/PositionValueAccess.java | 24 +++++++++---------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 4989414c0..199120c7d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -170,7 +170,6 @@ public DatasetAttributes( final DataType dataType) { // TODO add compression arg - this(dimensions, shardSize, dataType, defaultShardCodecInfo(blockSize)); } @@ -404,6 +403,11 @@ public DataCodecInfo[] getDataCodecInfos() { return dataCodecInfos; } + public String relativeBlockPath(long... position) { + + return Arrays.stream(position).mapToObj(Long::toString).collect(Collectors.joining("/")); + } + public HashMap asMap() { final HashMap map = new HashMap<>(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index d0deab06a..6db0d0746 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -104,14 +104,14 @@ default DataBlock readBlock( try { final PositionValueAccess posKva = PositionValueAccess.fromKva( - getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName)); + getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), + p -> datasetAttributes.relativeBlockPath(p)); return datasetAttributes.getDatasetAccess().readBlock(posKva, gridPosition); } catch (N5Exception.N5NoSuchKeyException e) { return null; } } - @Override default List> readBlocks( final String pathName, @@ -120,7 +120,8 @@ default List> readBlocks( try { final PositionValueAccess posKva = PositionValueAccess.fromKva( - getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName)); + getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), + p -> datasetAttributes.relativeBlockPath(p)); return datasetAttributes.getDatasetAccess().readBlocks(posKva, blockPositions); } catch (N5Exception.N5NoSuchKeyException e) { return null; @@ -160,13 +161,8 @@ default String absoluteDataBlockPath( final String normalPath, final long... gridPosition) { - final String[] components = new String[gridPosition.length + 1]; - components[0] = normalPath; - int i = 0; - for (final long p : gridPosition) - components[++i] = Long.toString(p); - - return getKeyValueAccess().compose(getURI(), components); + final String relativeBlockPath = getDatasetAttributes(normalPath).relativeBlockPath(gridPosition); + return getKeyValueAccess().compose(getURI(), normalPath, relativeBlockPath); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 441225767..4d3299bc7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -225,7 +225,8 @@ default void writeBlocks( try { final PositionValueAccess posKva = PositionValueAccess.fromKva( - getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath)); + getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), + p -> datasetAttributes.relativeBlockPath(p)); datasetAttributes.getDatasetAccess().writeBlocks(posKva, Arrays.asList(dataBlocks)); } catch (final UncheckedIOException e) { throw new N5IOException( @@ -241,7 +242,8 @@ default void writeBlock( try { final PositionValueAccess posKva = PositionValueAccess.fromKva( - getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path)); + getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), + p -> datasetAttributes.relativeBlockPath(p)); datasetAttributes.getDatasetAccess().writeBlock(posKva, dataBlock); } catch (final UncheckedIOException e) { throw new N5IOException( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java index ae5f91c69..68343da1e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.OutputStream; import java.net.URI; +import java.util.function.Function; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; @@ -27,9 +28,10 @@ public interface PositionValueAccess { public static PositionValueAccess fromKva( final KeyValueAccess kva, final URI uri, - final String normalPath) { + final String normalPath, + final Function blockPositionToPath) { - return new KvaPositionValueAccess(kva, uri, normalPath); + return new KvaPositionValueAccess(kva, uri, normalPath, blockPositionToPath); } class KvaPositionValueAccess implements PositionValueAccess { @@ -37,27 +39,23 @@ class KvaPositionValueAccess implements PositionValueAccess { private final KeyValueAccess kva; private final URI uri; private final String normalPath; + private final Function blockPositionToPath; KvaPositionValueAccess(final KeyValueAccess kva, final URI uri, - final String normalPath) { + final String normalPath, + final Function blockPositionToPath) { + this.kva = kva; this.uri = uri; this.normalPath = normalPath; + this.blockPositionToPath = blockPositionToPath; } // TODO this duplicates GsonKeyValueReader.absoluteDataBlockPath // is this where we want the logic? - private String absolutePath( - final long... gridPosition) { - - final String[] components = new String[gridPosition.length + 1]; - components[0] = normalPath; - int i = 0; - for (final long p : gridPosition) - components[++i] = Long.toString(p); - - return kva.compose(uri, components); + private String absolutePath( final long... gridPosition) { + return kva.compose(uri, normalPath, blockPositionToPath.apply(gridPosition)); } @Override From 61e6075ab9bb112bf0aad86f2b0cf09d10f933cc Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 16 Oct 2025 10:52:25 -0400 Subject: [PATCH 366/423] fix: NameConfig.Names should not collid with CompressionType name * because Compression is also a DataCodecInfo * causing serialization issues for zarr --- .../java/org/janelia/saalfeldlab/n5/Bzip2Compression.java | 2 +- src/main/java/org/janelia/saalfeldlab/n5/Compression.java | 7 ++++--- .../java/org/janelia/saalfeldlab/n5/GzipCompression.java | 2 +- .../java/org/janelia/saalfeldlab/n5/Lz4Compression.java | 2 +- .../java/org/janelia/saalfeldlab/n5/RawCompression.java | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 5fa871088..8e61c543f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -38,7 +38,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("bzip2") -@NameConfig.Name("bzip2") +@NameConfig.Name("bzip2-compression") public class Bzip2Compression implements Compression { private static final long serialVersionUID = -4873117458390529118L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index 143d614cb..8c059c8a0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -49,11 +49,12 @@ * serialization. *

      * See also: an alternative method for serializing general {@link CodecInfo}s is - * with the {@link NameConfigAdapter}. + * with the {@link NameConfigAdapter}. This interface remains for legacy + * (de)serialization. * * @author Stephan Saalfeld */ -public interface Compression extends Serializable, DataCodecInfo, DataCodec { +public interface Compression extends Serializable, DataCodec, DataCodecInfo { /** * Annotation for runtime discovery of compression schemes. @@ -77,7 +78,6 @@ public interface Compression extends Serializable, DataCodecInfo, DataCodec { @Target(ElementType.FIELD) @interface CompressionParameter {} - @Override default String getType() { final CompressionType compressionType = getClass().getAnnotation(CompressionType.class); @@ -91,4 +91,5 @@ default String getType() { default DataCodec create() { return this; } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index 9c81fef76..a870c954b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -42,7 +42,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("gzip") -@NameConfig.Name("gzip") +@NameConfig.Name("gzip-compression") public class GzipCompression implements Compression { private static final long serialVersionUID = 8630847239813334263L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index ea01879df..eb1d52f1a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -35,7 +35,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("lz4") -@NameConfig.Name("lz4") +@NameConfig.Name("lz4-compression") public class Lz4Compression implements Compression { private static final long serialVersionUID = -9071316415067427256L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index 6d3ebc86e..2e48a1e32 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -34,7 +34,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("raw") -@NameConfig.Name("raw") +@NameConfig.Name("raw-compression") public class RawCompression implements Compression, DeterministicSizeDataCodec { private static final long serialVersionUID = 7526445806847086477L; From ace0b16a439ac9387cb89ad5b6ef1a4f905d4476 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 16 Oct 2025 10:52:42 -0400 Subject: [PATCH 367/423] test: remove hard-coded compressor for testOverwriteBlock --- .../java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 19d71ba74..8664c4cbd 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -261,7 +261,6 @@ public void testWriteReadByteBlock() { final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); - } } } @@ -492,8 +491,9 @@ public void testWriteInvalidBlock() { @Test public void testOverwriteBlock() { - try (final N5Writer n5 = createTempN5Writer("test.n5")) { - n5.createDataset(datasetName, dimensions, blockSize, DataType.INT32, new GzipCompression()); + final Compression compression = getCompressions()[0]; + try (final N5Writer n5 = createTempN5Writer("test-overwrite")) { + n5.createDataset(datasetName, dimensions, blockSize, DataType.INT32, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final IntArrayDataBlock randomDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); From 16cd811e3d41944b73a6bdbcc3bb82948a5413b2 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 16 Oct 2025 13:04:24 -0400 Subject: [PATCH 368/423] feat: StringDataBlock uses UTF_8 encoding --- .../saalfeldlab/n5/StringDataBlock.java | 71 +++++++++++++++---- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 0e628161b..fe846ebd6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -1,22 +1,20 @@ -/*- - * #%L - * Not HDF5 - * %% - * Copyright (C) 2017 - 2025 Stephan Saalfeld - * %% +/** + * Copyright (c) 2017, Stephan Saalfeld + * All rights reserved. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS @@ -24,14 +22,61 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. - * #L% */ package org.janelia.saalfeldlab.n5; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + public class StringDataBlock extends AbstractDataBlock { - public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { + protected static final Charset ENCODING = StandardCharsets.UTF_8; + protected static final String NULLCHAR = "\0"; + protected byte[] serializedData = null; + protected String[] actualData = null; + + public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { + super(size, gridPosition, new String[0], a -> a.length); + actualData = data; + } + + public StringDataBlock(final int[] size, final long[] gridPosition, final byte[] data) { + super(size, gridPosition, new String[0], a -> a.length); + serializedData = data; + } + + public void readData(final ByteBuffer buffer) { + + if (buffer.hasArray()) { + if (buffer.array() != serializedData) + buffer.get(serializedData); + actualData = deserialize(buffer.array()); + } else + actualData = ENCODING.decode(buffer).toString().split(NULLCHAR); + } + + protected byte[] serialize(String[] strings) { + final String flattenedArray = String.join(NULLCHAR, strings) + NULLCHAR; + return flattenedArray.getBytes(ENCODING); + } + + protected String[] deserialize(byte[] rawBytes) { + final String rawChars = new String(rawBytes, ENCODING); + return rawChars.split(NULLCHAR); + } + + @Override + public int getNumElements() { + if (serializedData == null) + serializedData = serialize(actualData); + return serializedData.length; + } - super(size, gridPosition, data, a -> a.length); - } + @Override + public String[] getData() { + if (actualData == null) + actualData = deserialize(serializedData); + return actualData; + } } From 1f13153f0301ccc7cb23c97f04b05eace6b261d3 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 17 Oct 2025 14:20:27 -0400 Subject: [PATCH 369/423] refactor: rename ShardedDatasetAccess to DefaultDatasetAccess * move creation into DatasetAttributes --- .../saalfeldlab/n5/DatasetAttributes.java | 63 ++++++++++++++++--- .../n5/shardstuff/DatasetAccess.java | 4 +- ...tAccess.java => DefaultDatasetAccess.java} | 52 ++------------- .../n5/shardstuff/RawShardTest.java | 26 ++++---- 4 files changed, 78 insertions(+), 67 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/shardstuff/{ShardedDatasetAccess.java => DefaultDatasetAccess.java} (89%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 199120c7d..82d4188ed 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -35,15 +35,18 @@ import com.google.gson.JsonParseException; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; + +import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.shardstuff.DatasetAccess; +import org.janelia.saalfeldlab.n5.shardstuff.DefaultDatasetAccess; import org.janelia.saalfeldlab.n5.shardstuff.DefaultShardCodecInfo; +import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; import org.janelia.saalfeldlab.n5.shardstuff.ShardCodecInfo; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; -import org.janelia.saalfeldlab.n5.shardstuff.ShardedDatasetAccess; import java.io.Serializable; import java.lang.reflect.Type; @@ -89,6 +92,10 @@ public class DatasetAttributes implements Serializable { // number of samples per block per dimension private final int[] blockSize; + // TODO add a getter? + // the shard size + private final int[] outerBlockSize; + private final DataType dataType; private final BlockCodecInfo blockCodecInfo; @@ -105,6 +112,7 @@ public DatasetAttributes( this.dimensions = dimensions; this.dataType = dataType; + this.outerBlockSize = outerBlockSize; this.blockCodecInfo = blockCodecInfo == null ? defaultBlockCodecInfo() : blockCodecInfo; @@ -115,12 +123,8 @@ public DatasetAttributes( .filter(it -> it != null && !(it instanceof RawCompression)) .toArray(DataCodecInfo[]::new); - final ShardedDatasetAccess shardAccess = ShardedDatasetAccess.create( - getDataType(), outerBlockSize, - this.blockCodecInfo, this.dataCodecInfos); - access = shardAccess; - - this.blockSize = shardAccess.getGrid().getBlockSize(0); + access = createDatasetAccess(); + blockSize = access.getGrid().getBlockSize(0); } /** @@ -174,6 +178,51 @@ public DatasetAttributes( defaultShardCodecInfo(blockSize)); } + private DatasetAccess createDatasetAccess() { + + final int m = nestingDepth(blockCodecInfo); + + // There are m codecs: 1 DataBlock codecs, and m-1 shard codecs. + // The inner-most codec (the DataBlock codec) is at index 0. + final int[][] blockSizes = new int[m][]; + + // NestedGrid validates block sizes, so instantiate it before creating the blockCodecs + // blockCodecInfo.create below could fail unexpecedly with invalid + // blockSizes so validate first + blockSizes[m - 1] = outerBlockSize; + BlockCodecInfo tmpInfo = blockCodecInfo; + for (int l = m - 1; l > 0; --l) { + final ShardCodecInfo info = (ShardCodecInfo)tmpInfo; + blockSizes[l - 1] = info.getInnerBlockSize(); + tmpInfo = info.getInnerBlockCodecInfo(); + } + + BlockCodecInfo currentBlockCodecInfo = blockCodecInfo; + DataCodecInfo[] currentDataCodecInfos = dataCodecInfos; + + final NestedGrid grid = new NestedGrid(blockSizes); + final BlockCodec[] blockCodecs = new BlockCodec[m]; + for (int l = m - 1; l >= 0; --l) { + blockCodecs[l] = currentBlockCodecInfo.create(dataType, blockSizes[l], currentDataCodecInfos); + if (l > 0) { + final ShardCodecInfo info = (ShardCodecInfo)currentBlockCodecInfo; + currentBlockCodecInfo = info.getInnerBlockCodecInfo(); + currentDataCodecInfos = info.getInnerDataCodecInfos(); + } + } + + return new DefaultDatasetAccess<>(grid, blockCodecs); + } + + private static int nestingDepth(BlockCodecInfo info) { + + if (info instanceof ShardCodecInfo) { + return 1 + nestingDepth(((ShardCodecInfo)info).getInnerBlockCodecInfo()); + } else { + return 1; + } + } + protected static BlockCodecInfo defaultShardCodecInfo(int[] innerBlockSize) { return new DefaultShardCodecInfo( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java index c8ffd455a..d2c710064 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java @@ -2,7 +2,7 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; +import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; import java.util.List; @@ -24,4 +24,6 @@ public interface DatasetAccess { List> readBlocks(PositionValueAccess kva, List positions); void writeBlocks(PositionValueAccess kva, List> blocks); + + NestedGrid getGrid(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultDatasetAccess.java similarity index 89% rename from src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java rename to src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultDatasetAccess.java index b39bef069..20872aa53 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardedDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultDatasetAccess.java @@ -3,6 +3,7 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; import org.janelia.saalfeldlab.n5.codec.BlockCodec; @@ -13,18 +14,19 @@ import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.TreeMap; import java.util.stream.Collectors; -public class ShardedDatasetAccess implements DatasetAccess { +public class DefaultDatasetAccess implements DatasetAccess { private final NestedGrid grid; private final BlockCodec[] codecs; - public ShardedDatasetAccess(final NestedGrid grid, final BlockCodec[] codecs) { + public DefaultDatasetAccess(final NestedGrid grid, final BlockCodec[] codecs) { this.grid = grid; this.codecs = codecs; } @@ -109,7 +111,6 @@ private List> readShardRecursive( return Collections.emptyList(); } - final NestedPosition firstBlock = positions.get(0); final long[] shardPosition = firstBlock.absolute(level); @@ -305,51 +306,6 @@ private ReadData deleteBlockRecursive( } } - public static ShardedDatasetAccess create( - final DataType dataType, - int[] blockSize, - BlockCodecInfo blockCodecInfo, - DataCodecInfo[] dataCodecInfos - ) { - final int m = nestingDepth(blockCodecInfo); - - // There are m codecs: 1 DataBlock codecs, and m-1 shard codecs. - // The inner-most codec (the DataBlock codec) is at index 0. - final int[][] blockSizes = new int[m][]; - - // NestedGrid validates block sizes, so instantiate it before creating the blockCodecs - // blockCodecInfo.create below could fail unexpecedly with invalid blockSizes - // so validate first - blockSizes[m-1] = blockSize; - BlockCodecInfo tmpInfo = blockCodecInfo; - for (int l = m - 1; l > 0; --l) { - final ShardCodecInfo info = (ShardCodecInfo) tmpInfo; - blockSizes[l-1] = info.getInnerBlockSize(); - tmpInfo = info.getInnerBlockCodecInfo(); - } - - final NestedGrid grid = new NestedGrid(blockSizes); - final BlockCodec[] blockCodecs = new BlockCodec[m]; - for (int l = m - 1; l >= 0; --l) { - blockCodecs[l] = blockCodecInfo.create(dataType, blockSizes[l], dataCodecInfos); - if (l > 0) { - final ShardCodecInfo info = (ShardCodecInfo) blockCodecInfo; - blockCodecInfo = info.getInnerBlockCodecInfo(); - dataCodecInfos = info.getInnerDataCodecInfos(); - } - } - - return new ShardedDatasetAccess<>(grid, blockCodecs); - } - - private static int nestingDepth(BlockCodecInfo info) { - if (info instanceof ShardCodecInfo) { - return 1 + nestingDepth(((ShardCodecInfo) info).getInnerBlockCodecInfo()); - } else { - return 1; - } - } - private static ReadData getExistingReadData(final PositionValueAccess pva, final long[] key) { // need to read the shard anyway, and currently (Sept 24 2025) // have no way to tell if they key exist from what is in this method except to attempt diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java index 1cecacc43..adc399508 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java @@ -4,25 +4,29 @@ import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; public class RawShardTest { public static void main(String[] args) { + + int[] datablockSize = {3, 3, 3}; + int[] level1ShardSize = {6, 6, 6}; + int[] level2ShardSize = {24, 24, 24}; + // DataBlocks are 3x3x3 // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - new int[] {3, 3, 3}, + datablockSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -30,7 +34,7 @@ public static void main(String[] args) { IndexLocation.END ); final ShardCodecInfo c2 = new DefaultShardCodecInfo( - new int[] {6, 6, 6}, + level1ShardSize, c1, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -38,14 +42,14 @@ public static void main(String[] args) { IndexLocation.START ); - // TODO: This should go into DatasetAttributes. - // DatasetAccess replaces DatasetAttributes.blockCodec - // DatasetAttributes.getDataAccess() replaces DatasetAttributes.getBlockCodec() - // All information required for ShardedDatasetAccess.create(...) is known to DatasetAttributes already. - final DatasetAccess datasetAccess = ShardedDatasetAccess.create(DataType.INT8, - new int[] {24, 24, 24}, + DatasetAttributes attributes = new DatasetAttributes( + new long[] {}, + level2ShardSize, + DataType.INT8, c2, - new DataCodecInfo[] {new RawCompression()}); + new RawCompression()); + + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); // TODO: N5Reader/Writer needs to provide a PositionValueAccess implementation on top of its KVA. // The read/write/deleteBlock methods would getDataAccess() from the DatasetAttributes and call it with that PositionValueAccess. From 24a75d6006b4341862336bcfc753774b5f7fe887 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 17 Oct 2025 14:22:57 -0400 Subject: [PATCH 370/423] test: AbstractN5Test fix random seed --- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 8664c4cbd..2cf394c42 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -187,7 +187,7 @@ protected Compression[] getCompressions() { @Before public void setUpOnce() { - final Random rnd = new Random(); + final Random rnd = new Random(111); byteBlock = new byte[blockNumElements]; shortBlock = new short[blockNumElements]; intBlock = new int[blockNumElements]; From 10fc3df0da66da5ac3a29ebec4c957d9bddbda4d Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 17 Oct 2025 15:01:07 -0400 Subject: [PATCH 371/423] refactor: rm old shard classes, shardstuff -> shard --- .../saalfeldlab/n5/DatasetAttributes.java | 12 +- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 7 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 9 +- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 3 - .../org/janelia/saalfeldlab/n5/N5Reader.java | 2 - .../org/janelia/saalfeldlab/n5/N5Writer.java | 4 - .../saalfeldlab/n5/shard/AbstractShard.java | 47 -- .../n5/shard/BlockAsShardCodec.java | 111 ---- .../{shardstuff => shard}/DatasetAccess.java | 4 +- .../DefaultDatasetAccess.java | 6 +- .../DefaultShardCodecInfo.java | 4 +- .../saalfeldlab/n5/shard/InMemoryShard.java | 89 --- .../n5/{shardstuff => shard}/Nesting.java | 2 +- .../PositionValueAccess.java | 2 +- .../n5/{shardstuff => shard}/RawShard.java | 4 +- .../{shardstuff => shard}/RawShardCodec.java | 10 +- .../RawShardDataBlock.java | 2 +- .../janelia/saalfeldlab/n5/shard/Shard.java | 229 ------- .../{shardstuff => shard}/ShardCodecInfo.java | 4 +- .../saalfeldlab/n5/shard/ShardIndex.java | 602 +++++++----------- .../saalfeldlab/n5/shard/ShardingCodec.java | 200 ------ .../saalfeldlab/n5/shard/VirtualShard.java | 130 ---- .../saalfeldlab/n5/shardstuff/ShardIndex.java | 235 ------- .../saalfeldlab/n5/DatasetAttributesTest.java | 2 - .../saalfeldlab/n5/N5BlockAsShardTest.java | 57 -- .../saalfeldlab/n5/codec/BlockCodecTests.java | 4 +- .../saalfeldlab/n5/demo/BlockIterators.java | 1 - .../n5/http/HttpReaderFsWriter.java | 2 - .../saalfeldlab/n5/shard/ShardTest.java | 4 +- .../n5/shardstuff/NestedGridTest.java | 2 +- .../n5/shardstuff/RawShardTest.java | 6 +- .../shardstuff/TestPositionValueAccess.java | 1 + 32 files changed, 268 insertions(+), 1529 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java rename src/main/java/org/janelia/saalfeldlab/n5/{shardstuff => shard}/DatasetAccess.java (87%) rename src/main/java/org/janelia/saalfeldlab/n5/{shardstuff => shard}/DefaultDatasetAccess.java (98%) rename src/main/java/org/janelia/saalfeldlab/n5/{shardstuff => shard}/DefaultShardCodecInfo.java (97%) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java rename src/main/java/org/janelia/saalfeldlab/n5/{shardstuff => shard}/Nesting.java (99%) rename src/main/java/org/janelia/saalfeldlab/n5/{shardstuff => shard}/PositionValueAccess.java (98%) rename src/main/java/org/janelia/saalfeldlab/n5/{shardstuff => shard}/RawShard.java (92%) rename src/main/java/org/janelia/saalfeldlab/n5/{shardstuff => shard}/RawShardCodec.java (92%) rename src/main/java/org/janelia/saalfeldlab/n5/{shardstuff => shard}/RawShardDataBlock.java (94%) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java rename src/main/java/org/janelia/saalfeldlab/n5/{shardstuff => shard}/ShardCodecInfo.java (89%) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 82d4188ed..1140bd090 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -41,12 +41,12 @@ import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.shardstuff.DatasetAccess; -import org.janelia.saalfeldlab.n5.shardstuff.DefaultDatasetAccess; -import org.janelia.saalfeldlab.n5.shardstuff.DefaultShardCodecInfo; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; -import org.janelia.saalfeldlab.n5.shardstuff.ShardCodecInfo; -import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.DatasetAccess; +import org.janelia.saalfeldlab.n5.shard.DefaultDatasetAccess; +import org.janelia.saalfeldlab.n5.shard.DefaultShardCodecInfo; +import org.janelia.saalfeldlab.n5.shard.ShardCodecInfo; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; import java.io.Serializable; import java.lang.reflect.Type; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 6db0d0746..ff5c5256e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -32,18 +32,13 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.UncheckedIOException; -import java.util.ArrayList; import java.util.List; -import java.util.Map; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; -import org.janelia.saalfeldlab.n5.shardstuff.PositionValueAccess; +import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; import com.google.gson.Gson; import com.google.gson.JsonElement; -import org.janelia.saalfeldlab.n5.shard.Shard; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 4d3299bc7..4098b9ddb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -29,26 +29,19 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; -import java.io.OutputStream; import java.io.UncheckedIOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.stream.Collectors; import com.google.gson.JsonSyntaxException; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.shardstuff.PositionValueAccess; +import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonNull; import com.google.gson.JsonObject; -import org.janelia.saalfeldlab.n5.shard.InMemoryShard; -import org.janelia.saalfeldlab.n5.shard.Shard; -import org.janelia.saalfeldlab.n5.util.Position; /** * Default implementation of {@link N5Writer} with JSON attributes parsed with diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 125002ff6..45c2c1780 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -50,9 +50,7 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.N5Exception.N5JsonParseException; -import org.janelia.saalfeldlab.n5.codec.CodecInfo; /** * Utility class for working with JSON. @@ -68,7 +66,6 @@ static Gson registerGson(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, DatasetAttributes.getJsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBlockCodecInfo.byteOrderAdapter); -// gsonBuilder.registerTypeHierarchyAdapter(ShardingCodec.IndexLocation.class, ShardingCodec.indexLocationAdapter); gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.disableHtmlEscaping(); return gsonBuilder.create(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 76fc2fbb2..2eb932b38 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -28,8 +28,6 @@ */ package org.janelia.saalfeldlab.n5; -import org.janelia.saalfeldlab.n5.shard.Shard; - import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.ObjectInputStream; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 6d5e155b9..5bd85e6d9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -29,17 +29,13 @@ package org.janelia.saalfeldlab.n5; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.DataCodec; -import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.shard.Shard; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; import java.io.Serializable; import java.io.UncheckedIOException; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java deleted file mode 100644 index 7d7d4511c..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/AbstractShard.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import org.janelia.saalfeldlab.n5.DatasetAttributes; - - -public abstract class AbstractShard implements Shard { - -// protected final DatasetAttributes datasetAttributes; -// -// protected ShardIndex index; -// -// private final long[] gridPosition; -// -// public AbstractShard( -// final DatasetAttributes datasetAttributes, -// final long[] gridPosition, -// final ShardIndex index) { -// -// this.datasetAttributes = datasetAttributes; -// this.gridPosition = gridPosition; -// this.index = index; -// } -// -// @Override -// public DatasetAttributes getDatasetAttributes() { -// -// return datasetAttributes; -// } -// -// @Override -// public int[] getSize() { -// -// return getDatasetAttributes().getShardSize(); -// } -// -// @Override -// public int[] getBlockSize() { -// -// return datasetAttributes.getBlockSize(); -// } -// -// @Override -// public long[] getGridPosition() { -// -// return gridPosition; -// } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java deleted file mode 100644 index c8e2d136d..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/BlockAsShardCodec.java +++ /dev/null @@ -1,111 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.LongArrayDataBlock; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.BlockCodec; -import org.janelia.saalfeldlab.n5.codec.DataCodec; -import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo; -import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter; -import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.serialization.NameConfig; - -@NameConfig.Serialize(false) -public class BlockAsShardCodec extends ShardingCodec { - -// private static final LongArrayDataBlock BLOCK_AS_SHARD_INDEX = new LongArrayDataBlock(new int[0], new long[0], new long[]{0, -1}); -// private static final BlockCodec BLOCK_AS_SHARD_BLOCK_SERIALIZER = new BlockCodec() { -// -// @Override public ReadData encode(DataBlock dataBlock) throws N5Exception.N5IOException { -// -// return ReadData.from(new byte[0]); -// } -// -// @Override public DataBlock decode(ReadData readData, long[] gridPosition) throws N5Exception.N5IOException { -// -// return BLOCK_AS_SHARD_INDEX; -// } -// }; -// -// private static final RawBlockCodecInfo VIRTUAL_SHARD_INDEX_CODEC = new RawBlockCodecInfo() { -// -// public BlockCodec create(DatasetAttributes attributes, DataCodec... dataCodec) { -// -// return BLOCK_AS_SHARD_BLOCK_SERIALIZER; -// } -// }; -// private static final DataCodecInfo[] EMPTY_SHARD_CODECS = new DataCodecInfo[0]; -// -// private static final DeterministicSizeCodecInfo[] NO_OP_INDEX_CODECS = new DeterministicSizeCodecInfo[0]; -// private static final IndexCodecAdapter BLOCK_AS_SHARD_INDEX_CODEC_ADAPTER = new IndexCodecAdapter(VIRTUAL_SHARD_INDEX_CODEC, NO_OP_INDEX_CODECS) { -// -// @Override public long encodedSize(long initialSize) { -// -// return 0; -// } -// }; -// -// final BlockCodecInfo datasetArrayCodec; -// -// public BlockAsShardCodec(BlockCodecInfo datasetArrayCodec) { -// -// super(null, EMPTY_SHARD_CODECS, BLOCK_AS_SHARD_INDEX_CODEC_ADAPTER, IndexLocation.START); -// this.datasetArrayCodec = datasetArrayCodec; -// } -// -// @Override -// public ShardIndex createIndex(DatasetAttributes attributes) { -// -// return new ShardIndex(attributes.getBlocksPerShard(), getIndexLocation(), BLOCK_AS_SHARD_INDEX_CODEC_ADAPTER); -// } -// -// @Override -// public BlockCodecInfo getBlockCodecInfo() { -// -// return datasetArrayCodec; -// } -// -// @Override -// public long[] getKeyPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { -// -// return datasetArrayCodec.getKeyPositionForBlock(attributes, datablock); -// } -// -// @Override -// public long[] getKeyPositionForBlock(DatasetAttributes attributes, long... blockPosition) { -// -// return datasetArrayCodec.getKeyPositionForBlock(attributes, blockPosition); -// } -// -// @Override -// public BlockCodec create(DatasetAttributes attributes, DataCodec... codecs) { -// -// // TODO -// return null; -// -//// dataBlockSerializer = datasetArrayCodec.create(attributes, codecs); -//// return (BlockCodec)dataBlockSerializer; -// } -// -// @Override -// public long encodedSize(long size) { -// -// return datasetArrayCodec.encodedSize(size); -// } -// -// @Override -// public long decodedSize(long size) { -// -// return datasetArrayCodec.decodedSize(size); -// } -// -// @Override -// public String getType() { -// //TODO Caleb: can we ensure this is never called? -// return datasetArrayCodec.getType(); -// } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java similarity index 87% rename from src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java rename to src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index d2c710064..82ab75634 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -1,8 +1,8 @@ -package org.janelia.saalfeldlab.n5.shardstuff; +package org.janelia.saalfeldlab.n5.shard; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; import java.util.List; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java similarity index 98% rename from src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultDatasetAccess.java rename to src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 20872aa53..58992f87a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -1,4 +1,4 @@ -package org.janelia.saalfeldlab.n5.shardstuff; +package org.janelia.saalfeldlab.n5.shard; import org.janelia.saalfeldlab.n5.DataBlock; @@ -10,8 +10,8 @@ import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedPosition; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; import java.util.ArrayList; import java.util.Arrays; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java similarity index 97% rename from src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultShardCodecInfo.java rename to src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java index cbcd50570..25d7e5cda 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/DefaultShardCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java @@ -1,4 +1,4 @@ -package org.janelia.saalfeldlab.n5.shardstuff; +package org.janelia.saalfeldlab.n5.shard; import java.lang.reflect.Type; import java.util.Arrays; @@ -7,7 +7,7 @@ import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java deleted file mode 100644 index 0e97124cf..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/InMemoryShard.java +++ /dev/null @@ -1,89 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.util.Position; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -public class InMemoryShard extends AbstractShard { - -// /** Map {@link DataBlock#getGridPosition} as hashable {@link Position} to the block */ -// private final Map> blocks; -// -// public InMemoryShard(final DatasetAttributes datasetAttributes, final long[] shardPosition) { -// -// this(datasetAttributes, shardPosition, null); -// } -// -// public InMemoryShard( -// final DatasetAttributes datasetAttributes, -// final long[] gridPosition, -// ShardIndex index) { -// -// super(datasetAttributes, gridPosition, index); -// blocks = new TreeMap<>(); -// } -// -// private void storeBlock(DataBlock block) { -// -// blocks.put(Position.wrap(block.getGridPosition()), block); -// } -// -// @Override -// public DataBlock getBlock(int... blockGridPosition) { -// -// return blocks.get(Position.wrap(blockGridPosition)); -// } -// -// /** -// * Add the {@code block} to this shard. If the block is not contained in this shard, do not add it. -// * -// * @param block to add the shard -// * @return whether the block was added -// */ -// public boolean addBlock(DataBlock block) { -// -// final long[] shardPositionForBlock = datasetAttributes.getShardPositionForBlock(block.getGridPosition()); -// if (!Arrays.equals(shardPositionForBlock, getGridPosition())) -// return false; -// -// storeBlock(block); -// return true; -// } -// -// @Override -// public List> getBlocks() { -// -// return new ArrayList<>(blocks.values()); -// } -// -// @Override -// public ShardIndex getIndex() { -// -//// return index = index != null ? index : createIndex(); -// -// // TODO -// return index; -// } -// -// public static InMemoryShard fromShard(Shard shard) { -// -// if (shard == null) -// return null; -// -// if (shard instanceof InMemoryShard) -// return (InMemoryShard)shard; -// -// final InMemoryShard inMemoryShard = new InMemoryShard( -// shard.getDatasetAttributes(), -// shard.getGridPosition()); -// -// shard.forEach(inMemoryShard::addBlock); -// return inMemoryShard; -// } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java similarity index 99% rename from src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java rename to src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java index 3367879f0..16b454bb9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java @@ -1,4 +1,4 @@ -package org.janelia.saalfeldlab.n5.shardstuff; +package org.janelia.saalfeldlab.n5.shard; // TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java similarity index 98% rename from src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java rename to src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java index 68343da1e..7576ad8fc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java @@ -1,4 +1,4 @@ -package org.janelia.saalfeldlab.n5.shardstuff; +package org.janelia.saalfeldlab.n5.shard; import java.io.IOException; import java.io.OutputStream; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java similarity index 92% rename from src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShard.java rename to src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java index 98c8ea6ae..690b3d180 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java @@ -1,9 +1,9 @@ -package org.janelia.saalfeldlab.n5.shardstuff; +package org.janelia.saalfeldlab.n5.shard; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.segment.Segment; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; -import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.NDArray; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.NDArray; public class RawShard { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java similarity index 92% rename from src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardCodec.java rename to src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java index dfc9b1689..adaeb864e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java @@ -1,4 +1,6 @@ -package org.janelia.saalfeldlab.n5.shardstuff; +package org.janelia.saalfeldlab.n5.shard; + +import static org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation.START; import java.util.ArrayList; import java.util.List; @@ -9,10 +11,8 @@ import org.janelia.saalfeldlab.n5.readdata.segment.Segment; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; -import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; -import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.NDArray; - -import static org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation.START; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.NDArray; public class RawShardCodec implements BlockCodec { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java similarity index 94% rename from src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardDataBlock.java rename to src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java index 7f794e7c1..e36017ffd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java @@ -1,4 +1,4 @@ -package org.janelia.saalfeldlab.n5.shardstuff; +package org.janelia.saalfeldlab.n5.shard; import org.janelia.saalfeldlab.n5.DataBlock; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java deleted file mode 100644 index e9972c590..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Shard.java +++ /dev/null @@ -1,229 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.commons.io.output.CountingOutputStream; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.codec.BlockCodec; -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.shardstuff.DatasetAccess; -import org.janelia.saalfeldlab.n5.util.GridIterator; - -import static org.janelia.saalfeldlab.n5.N5Exception.*; - -public interface Shard { -//extends Iterable> { - -// /** -// * Returns the number of blocks this shard contains along all dimensions. -// * -// * The size of a shard expected to be smaller than or equal to the spacing of the shard grid. The dimensionality of size is expected to be equal to the dimensionality of the -// * dataset. Consistency is not enforced. -// * -// * @return size of the shard in units of blocks -// */ -// default int[] getBlockGridSize() { -// -// return getDatasetAttributes().getBlocksPerShard(); -// } -// -// DatasetAttributes getDatasetAttributes(); -// -// /** -// * Returns the size of shards in pixel units. -// * -// * @return shard size -// */ -// default int[] getSize() { -// return getDatasetAttributes().getShardSize(); -// } -// -// /** -// * Returns the size of blocks in pixel units. -// * -// * @return block size -// */ -// default int[] getBlockSize() { -// return getDatasetAttributes().getBlockSize(); -// } -// -// /** -// * Returns the position of this shard on the shard grid. -// * -// * The dimensionality of the grid position is expected to be equal to the dimensionality of the dataset. Consistency is not enforced. -// * -// * @return position on the shard grid -// */ -// long[] getGridPosition(); -// -// /** -// * Given a {@code blockPosition} relative to the dataset, return its position relative to this -// * shard. -// * -// * @param blockPositionInDataset dataset-relative block position -// * @return the shard-relative block positiorelativeBlockPositionn -// * @see {@link DatasetAttributes#getBlockPositionInShard(long[], long[])} -// * @see {@link #getBlockPositionFromShardPosition(long[], long[])} -// */ -// default int[] getRelativeBlockPosition(long... datasetBlockPosition) { -// -// return getDatasetAttributes().getShardRelativeBlockPosition( -// getGridPosition(), -// datasetBlockPosition); -// } -// -// /** -// * Tests whether the block at the {@code relativeBlockPosition} (relative to -// * this shard) exists. -// *

      -// * Avoids reading the block data, if possible. -// * -// * @return true of the block exists in this shard -// */ -// default boolean blockExists(int... relativeBlockPosition) { -// -// return getIndex().exists(relativeBlockPosition); -// } -// -// /** -// * Retrieve the DataBlock at {@code blockGridPosition} relative to this shard if it exists and is -// * a member of this shard. -// *

      -// * If needed, use {@code getRelativeBlockPosition} to convert block positions relative to the dataset into -// * positions relative to this shard. -// * -// * @param blockGridPosition position of the desired block relative to this shard -// * @return the block if it exists and is part of this shard, otherwise null -// */ -// DataBlock getBlock(int... blockGridPosition); -// -// default Iterator> iterator() { -// -// return new DataBlockIterator<>(this); -// } -// -// default int getNumBlocks() { -// -// return Arrays.stream(getBlockGridSize()).reduce(1, (x, y) -> x * y); -// } -// -// default List> getBlocks() { -// -// final List> blocks = new ArrayList<>(); -// for (DataBlock block : this) { -// blocks.add(block); -// } -// return blocks; -// } -// -// -// /** -// * @return the ShardIndex for this shard, or a new ShardIndex if the Shard is non-existent -// */ -// ShardIndex getIndex(); -// -// default ReadData createReadData() throws N5IOException { -// -// final DatasetAttributes datasetAttributes = getDatasetAttributes(); -// DatasetAccess access = datasetAttributes.getDatasetAccess(); -// -// // TODO make a PositionValueAccess that just stores the ReadData -// // and returns it? -// for( DataBlock b : this ) { -// access.writeBlock(null, b); -// } -// -// return null; -// -//// ShardingCodec shardingCodec = datasetAttributes.getShardingCodec(); -//// final BlockCodec blockCodecInfo = shardingCodec.getBlockCodec(); -// -//// final ShardIndex index = createIndex(); -//// final long indexSize = index.numBytes(); -//// long blocksStartBytes = index.getLocation() == ShardingCodec.IndexLocation.START ? indexSize : 0; -//// final AtomicLong blockOffset = new AtomicLong(blocksStartBytes); -//// -//// /* isIndexEmpty is true when writing to a non-sharded dataset through the Shard API. */ -//// final boolean isIndexEmpty = indexSize == 0; -//// if (index.getLocation() == ShardingCodec.IndexLocation.END || isIndexEmpty) { -//// return ReadData.from(out -> { -//// try (final CountingOutputStream countOut = new CountingOutputStream(out)) { -//// long prevCount = 0; -//// for (DataBlock block : getBlocks()) { -//// blockCodecInfo.encode(block).writeTo(countOut); -//// final int[] blockPosition = getRelativeBlockPosition(block.getGridPosition()); -//// final long curCount = countOut.getByteCount(); -//// final long blockWrittenSize = curCount - prevCount; -//// prevCount = curCount; -//// if (!isIndexEmpty) -//// synchronized (index) { -//// index.set(blockOffset.getAndAdd(blockWrittenSize), blockWrittenSize, blockPosition); -//// } -//// } -//// if (!isIndexEmpty) -//// synchronized (index) { -//// ShardIndex.write(out, index); -//// } -//// } -//// }); -//// } else { -//// final ArrayList blocksData = new ArrayList<>(); -//// for (DataBlock dataBlock : getBlocks()) { -//// ReadData readDataBlock = ReadData.from(out -> blockCodecInfo.encode(dataBlock).writeTo(out)); -//// blocksData.add(readDataBlock); -//// final long length = readDataBlock.length(); -//// synchronized (index) { -//// index.set(blockOffset.getAndAdd(length), length, getRelativeBlockPosition(dataBlock.getGridPosition())); -//// } -//// } -//// return ReadData.from(out -> { -//// ShardIndex.write(out, index); -//// for (ReadData blockData : blocksData) { -//// blockData.writeTo(out); -//// } -//// }); -//// } -// } -// -// class DataBlockIterator implements Iterator> { -// -// private final GridIterator it; -// private final Shard shard; -// private final ShardIndex index; -// private final DatasetAttributes attributes; -// private int blockIndex = 0; -// -// public DataBlockIterator(final Shard shard) { -// -// this.shard = shard; -// this.index = shard.getIndex(); -// this.attributes = shard.getDatasetAttributes(); -// this.blockIndex = 0; -// it = new GridIterator(shard.getBlockGridSize()); -// } -// -// @Override -// public boolean hasNext() { -// -// for (int i = blockIndex; i < attributes.getNumBlocksPerShard(); i++) { -// if (index.exists(i)) -// return true; -// } -// return false; -// } -// -// @Override -// public DataBlock next() { -// while (!index.exists(blockIndex++)) -// it.fwd(); -// -// return shard.getBlock(it.nextInt()); -// } -// } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java similarity index 89% rename from src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardCodecInfo.java rename to src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java index b819967b4..0f2d2150f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java @@ -1,9 +1,9 @@ -package org.janelia.saalfeldlab.n5.shardstuff; +package org.janelia.saalfeldlab.n5.shard; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; public interface ShardCodecInfo extends BlockCodecInfo { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index 09eae5498..c0534675b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -1,377 +1,235 @@ package org.janelia.saalfeldlab.n5.shard; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.function.IntFunction; +import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; -import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.Segment; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData.SegmentsAndData; -import java.util.Arrays; -import java.util.stream.IntStream; - -/** - * The ShardIndex tracks the offset and length of blocks contained within a - * shard. - *

      - * Blocks in a shard are arrayed in an n-dimensional grid, referred to as the - * {@code shardBlockGrid}. The ShardIndex is implemented as an (n+1)-dimensional - * {@link LongArrayDataBlock}, where the 0th dimensions is length 2 and contains - * the block offsets and lengths. The grid position of the index iteself is meaningless, - * and as a result, {@link #getGridPosition()} will return {@code null}. - *

      - * The index stores two values for each block: offset and number of bytes. Blocks - * that don't exist are marked with the special value {@link #EMPTY_INDEX_NBYTES}. - *

      - * Block grid positions in this class are relative to the shard. - * - * @see The - * Zarr V3 specification for the binary shard format - */ public class ShardIndex { -//extends LongArrayDataBlock { - -// /** -// * Special value indicating an empty block entry in the index. -// * Used for both offset and length when a block doesn't exist. -// */ -// public static final long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; -// private static final int BYTES_PER_LONG = 8; -// private static final int LONGS_PER_BLOCK = 2; -// private static final long[] DUMMY_GRID_POSITION = null; -// -// private final IndexLocation location; -// private final ShardIndexAttributes indexAttributes; -// private final IndexCodecAdapter indexCodexAdapter; -// -// /** -// * Creates a ShardIndex with specified data. -// * -// * @param shardBlockGridSize the dimensions of the block grid within the shard -// * @param data the raw index data containing offsets and lengths -// * @param location where the index is stored (START or END of shard) -// * @param indexCodecAdapter data object for Shard Index codecs. -// */ -// public ShardIndex(int[] shardBlockGridSize, long[] data, IndexLocation location, final IndexCodecAdapter indexCodecAdapter) { -// -// // prepend the number of longs per block to the shard block grid size -// super(prepend(LONGS_PER_BLOCK, shardBlockGridSize), DUMMY_GRID_POSITION, data); -// this.indexCodexAdapter = indexCodecAdapter; -// this.location = location; -// this.indexAttributes = new ShardIndexAttributes(this); -// } -// -// /** -// * Creates an empty ShardIndex at the specified location. -// * -// * @param shardBlockGridSize the dimensions of the block grid within the shard -// * @param location where the index is stored (START or END of shard) -// * @param indexCodecAdapter data object for idnex codecs -// */ -// public ShardIndex(int[] shardBlockGridSize, IndexLocation location, final IndexCodecAdapter indexCodecAdapter) { -// -// this(shardBlockGridSize, emptyIndexData(shardBlockGridSize), location, indexCodecAdapter); -// } -// -// /** -// * Creates an empty ShardIndex at the default location (END). -// * -// * @param shardBlockGridSize the dimensions of the block grid within the shard -// * @param indexCodecAdapter data object for idnex codecs -// */ -// public ShardIndex(int[] shardBlockGridSize, final IndexCodecAdapter indexCodecAdapter) { -// -// this(shardBlockGridSize, IndexLocation.END, indexCodecAdapter); -// } -// -// /** -// * Creates an empty ShardIndex at the specified location. -// * -// * @param shardBlockGridSize the dimensions of the block grid within the shard -// * @param location where the index is stored (START or END of shard) -// * @param blockCodecInfo blockCodecInfo for the IndexCodecAdapter -// */ -// public ShardIndex(int[] shardBlockGridSize, IndexLocation location, final BlockCodecInfo blockCodecInfo) { -// -// this(shardBlockGridSize, location, new IndexCodecAdapter(blockCodecInfo)); -// } -// -// /** -// * Creates an empty ShardIndex at the default location (END). -// * -// * @param shardBlockGridSize the dimensions of the block grid within the shard -// * @param blockCodecInfo blockCodecInfo for the IndexCodecAdapter -// */ -// public ShardIndex(int[] shardBlockGridSize, final BlockCodecInfo blockCodecInfo) { -// -// this(shardBlockGridSize, IndexLocation.END, blockCodecInfo); -// } -// -// public IndexCodecAdapter getIndexCodexAdapter() { -// -// return indexCodexAdapter; -// } -// -// /** -// * Checks existence of the block at a given grid position. -// * -// * @param gridPosition the n-dimensional position of the block in the shard grid -// * @return true if the block exists, false otherwise -// */ -// public boolean exists(int[] gridPosition) { -// -// return getOffset(gridPosition) != EMPTY_INDEX_NBYTES || -// getNumBytes(gridPosition) != EMPTY_INDEX_NBYTES; -// } -// -// /** -// * Checks existence of the block at a given flat index. -// * -// * @param index the flattened index of the block -// * @return true if the block exists, false otherwise -// */ -// public boolean exists(int index) { -// -// return data[index * 2] != EMPTY_INDEX_NBYTES || -// data[index * 2 + 1] != EMPTY_INDEX_NBYTES; -// } -// -// /** -// * Gets the total number of blocks that can be stored in this index. -// * -// * @return the total number of blocks in the shard grid -// */ -// public int getNumBlocks() { -// -// /* getSize() is the number of data entries; each block takes 2 entries (offset and length) -// * so the product of the dimension sizes, divided by 2, is the number of blocks. */ -// return Arrays.stream(getSize()).reduce(1, (x, y) -> x * y) / 2; -// } -// -// /** -// * Checks if the index is completely empty (no blocks exist). -// * -// * @return true if no blocks exist in the index, false otherwise -// */ -// public boolean isEmpty() { -// -// return !IntStream.range(0, getNumBlocks()).anyMatch(this::exists); -// } -// -// /** -// * Gets the location of this index within the shard. -// * -// * @return the index location (START or END) -// */ -// public IndexLocation getLocation() { -// -// return location; -// } -// -// /** -// * Gets the offset in this shard in bytes for the block at a grid position. -// * -// * @param gridPosition the n-dimensional position of the block in the shard grid -// * @return the offset in bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist -// */ -// public long getOffset(int... gridPosition) { -// -// return data[getOffsetIndex(gridPosition)]; -// } -// -// /** -// * Gets the offset in this shard in bytes for the block at a given index. -// * -// * @param index the flattened index of the block -// * @return the offset in bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist -// */ -// public long getOffsetByBlockIndex(int index) { -// -// return data[index * 2]; -// } -// -// /** -// * Gets the number of bytes for the block at a grid position. -// * -// * @param gridPosition the n-dimensional position of the block in the shard grid -// * @return the number of bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist -// */ -// public long getNumBytes(int... gridPosition) { -// -// return data[getNumBytesIndex(gridPosition)]; -// } -// -// /** -// * Gets the number of bytes for the block at a given index. -// * -// * @param index the flattened index of the block -// * @return the number of bytes, or {@link #EMPTY_INDEX_NBYTES} if the block doesn't exist -// */ -// public long getNumBytesByBlockIndex(int index) { -// -// return data[index * 2 + 1]; -// } -// -// /** -// * Sets the offset and number of bytes for a block at the specified position. -// * -// * @param offset the byte offset of the block in the shard -// * @param nbytes the number of bytes the block occupies -// * @param gridPosition the n-dimensional position of the block in the shard grid -// */ -// public void set(long offset, long nbytes, int[] gridPosition) { -// -// final int i = getOffsetIndex(gridPosition); -// data[i] = offset; -// data[i + 1] = nbytes; -// } -// -// /** -// * Marks a block position as empty. -// * -// * @param gridPosition the n-dimensional position of the block to mark as empty -// */ -// public void setEmpty(int[] gridPosition) { -// -// set(EMPTY_INDEX_NBYTES, EMPTY_INDEX_NBYTES, gridPosition); -// } -// -// /** -// * Calculates the flattened array index for the offset value of a block. -// * -// * @param gridPosition the n-dimensional position of the block -// * @return the index in the data array where the offset is stored -// */ -// protected int getOffsetIndex(int... gridPosition) { -// -// int idx = (int) gridPosition[0]; -// int cumulativeSize = 1; -// for (int i = 1; i < gridPosition.length; i++) { -// cumulativeSize *= size[i]; -// idx += gridPosition[i] * cumulativeSize; -// } -// return idx * 2; -// } -// -// /** -// * Calculates the flattened array index for the number of bytes value of a block. -// * -// * @param gridPosition the n-dimensional position of the block -// * @return the index in the data array where the number of bytes is stored -// */ -// protected int getNumBytesIndex(int... gridPosition) { -// -// return getOffsetIndex(gridPosition) + 1; -// } -// -// /** -// * Calculates the total size of the index in bytes after compression. -// * -// * @return the total number of bytes the index occupies after applying all codecs -// */ -// public long numBytes() { -// -// final long numEntries = Arrays.stream(getSize()).reduce(1, (x, y) -> x * y); -// return getIndexCodexAdapter().encodedSize(numEntries * BYTES_PER_LONG); -// } -// -// -// /** -// * DatasetAttributes for the ShardIndex, used for codec operations. -// */ -// private static class ShardIndexAttributes extends DatasetAttributes { -// -// /** -// * Creates attributes for the given ShardIndex. -// * -// * @param index the ShardIndex -// */ -// public ShardIndexAttributes(ShardIndex index) { -// super( -// Arrays.stream(index.getSize()).mapToLong(it -> it).toArray(), -// index.getSize(), -// index.getSize(), -// DataType.UINT64, -// index.indexCodexAdapter.getBlockCodecInfo(), -// index.indexCodexAdapter.getDataCodecs() -// ); -// } -// } -// -// /** -// * Calculates the start byte of the index within a shard. -// * -// * @param index the ShardIndex -// * @param objectSize the total size of the shard in bytes -// * @return the start byte of the index -// */ -// public static long indexStartByte(final ShardIndex index, long objectSize) { -// -// return indexStartByte(index.numBytes(), index.location, objectSize); -// } -// -// /** -// * Calculates the start byte an index within a shard. -// * -// * @param indexSize the size of the index in bytes -// * @param indexLocation the location of the index (START or END) -// * @param objectSize the total size of the shard in bytes -// * @return the start byte of the index -// */ -// public static long indexStartByte(final long indexSize, final IndexLocation indexLocation, final long objectSize) { -// -// if (indexLocation == IndexLocation.START) { -// return 0L; -// } else { -// return objectSize - indexSize; -// } -// } -// -// /** -// * Creates an empty index data array filled with {@link #EMPTY_INDEX_NBYTES}. -// * -// * @param size the dimensions of the block grid -// * @return an array filled with empty values -// */ -// private static long[] emptyIndexData(final int[] size) { -// -// final int N = 2 * Arrays.stream(size).reduce(1, (x, y) -> x * y); -// final long[] data = new long[N]; -// Arrays.fill(data, EMPTY_INDEX_NBYTES); -// return data; -// } -// -// /** -// * Prepends a value to an array. -// * -// * @param value the value to prepend -// * @param array the original array -// * @return a new array with the value prepended -// */ -// private static int[] prepend(final int value, final int[] array) { -// -// final int[] indexBlockSize = new int[array.length + 1]; -// indexBlockSize[0] = value; -// System.arraycopy(array, 0, indexBlockSize, 1, array.length); -// return indexBlockSize; -// } -// -// @Override -// public boolean equals(Object other) { -// -// if (other instanceof ShardIndex) { -// -// final ShardIndex index = (ShardIndex)other; -// if (this.location != index.location) -// return false; -// -// if (!Arrays.equals(this.size, index.size)) -// return false; -// -// if (!Arrays.equals(this.data, index.data)) -// return false; -// -// } -// return true; -// } -} \ No newline at end of file + + private ShardIndex() { + // utility class. should not be instantiated. + } + + public enum IndexLocation { + START, END + } + + /** + * Access flat {@code T[]} array as n-dimensional array. + * + * @param + * element type + */ + public static class NDArray { + + final int[] size; + private final int[] stride; + final T[] data; + + NDArray(final int[] size, final IntFunction createArray) { + this.size = size; + stride = getStrides(size); + data = createArray.apply(getNumElements(size)); + } + + NDArray(final int[] size, final T[] data) { + this.size = size; + stride = getStrides(size); + this.data = data; + } + + T get(long... position) { + return data[index(position)]; + } + + void set(T value, long... position) { + data[index(position)] = value; + } + + private int index(long... position) { + int index = 0; + for (int i = 0; i < stride.length; i++) { + index += stride[i] * position[i]; + } + return index; + } + + public int[] size() { + return size; + } + + public int numElements() { + return data.length; + } + + public boolean allElementsNull() { + for (T t : data) { + if (t != null) { + return false; + } + } + return true; + } + } + + static int getNumElements(final int[] size) { + int numElements = 1; + for (int s : size) { + numElements *= s; + } + return numElements; + } + + static int[] getStrides(final int[] size) { + final int n = size.length; + final int[] stride = new int[n]; + stride[0] = 1; + for (int i = 1; i < n; i++) { + stride[i] = stride[i - 1] * size[i - 1]; + } + return stride; + } + + /** + * Special value indicating an empty block entry in the index. + * Used for both offset and length when a block doesn't exist. + */ + static final long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; + + /** + * Size of first dimension of the {@code DataBlock} representation of the shard index. + */ + private static final int LONGS_PER_BLOCK = 2; + + static NDArray fromDataBlock( final DataBlock block ) { + + final long[] blockData = block.getData(); + final int[] size = indexSizeFromBlockSize(block.getSize()); + final int n = getNumElements(size); + final SegmentLocation[] locations = new SegmentLocation[n]; + + for (int i = 0; i < n; i++) { + long offset = blockData[i * LONGS_PER_BLOCK]; + long length = blockData[i * LONGS_PER_BLOCK + 1]; + if (offset != EMPTY_INDEX_NBYTES && length != EMPTY_INDEX_NBYTES) { + locations[i] = SegmentLocation.at(offset, length); + } + } + return new NDArray<>(size, locations); + } + + static DataBlock toDataBlock( final NDArray locations, final long offset ) { + + final SegmentLocation[] data = locations.data; + + final int[] blockSize = blockSizeFromIndexSize(locations.size); + final long[] blockData = new long[data.length * 2]; + + for (int i = 0; i < data.length; ++i) { + if (data[i] != null) { + blockData[i * LONGS_PER_BLOCK] = data[i].offset() + offset; + blockData[i * LONGS_PER_BLOCK + 1] = data[i].length(); + } else { + blockData[i * LONGS_PER_BLOCK] = EMPTY_INDEX_NBYTES; + blockData[i * LONGS_PER_BLOCK + 1] = EMPTY_INDEX_NBYTES; + } + } + return new LongArrayDataBlock(blockSize, new long[blockSize.length], blockData); + } + + /** + * Prepends a value to an array. + * + * @param value the value to prepend + * @param array the original array + * @return a new array with the value prepended + */ + private static int[] prepend(final int value, final int[] array) { + + final int[] indexBlockSize = new int[array.length + 1]; + indexBlockSize[0] = value; + System.arraycopy(array, 0, indexBlockSize, 1, array.length); + return indexBlockSize; + } + + /** + * Prepends {@code LONGS_PER_BLOCK} to the {@code indexSize} array. + */ + static int[] blockSizeFromIndexSize(final int[] indexSize) { + return prepend(LONGS_PER_BLOCK, indexSize); + } + + /** + * Strips first element (should be {@code LONGS_PER_BLOCK} from the {@code blockSize} array. + */ + static int[] indexSizeFromBlockSize(final int[] blockSize) { + assert blockSize[ 0 ] == LONGS_PER_BLOCK; + return Arrays.copyOfRange(blockSize, 1, blockSize.length); + } + + /** + * Retrieves the {@code SegmentLocation} of each non-null {@code Segment} in + * {@code segments}. Returns a {@code NDArray} with entries + * corresponding tho the {@code segments} entries. + */ + static NDArray locations(final NDArray segments, final SegmentedReadData readData) { + + final Segment[] data = segments.data; + final SegmentLocation[] locations = new SegmentLocation[data.length]; + for (int i = 0; i < data.length; ++i) { + final Segment segment = data[i]; + if ( segment != null ) { + locations[i] = readData.location(segment); + } + } + return new NDArray<>(segments.size, locations); + } + + interface SegmentIndexAndData { + NDArray index(); + SegmentedReadData data(); + } + + /** + * Puts a {@code Segment} at each non-null {@code SegmentLocation} in {@code + * locations} on the given {@code readData}. Returns both the {@code + * SegmentedReadData} with these segments and a {@code NDArray} + * with segment entries corresponding to the {@code locations} entries. + */ + static SegmentIndexAndData segments(final NDArray locations, final ReadData readData) { + + final SegmentLocation[] locationsData = locations.data; + final Segment[] segmentsData = new Segment[locationsData.length]; + + final List presentLocations = new ArrayList<>(); + for (int i = 0; i < locationsData.length; i++) { + if (locationsData[i] != null) { + presentLocations.add(locationsData[i]); + } + } + + final SegmentsAndData segmentsAndData = SegmentedReadData.wrap(readData, presentLocations); + final Iterator presentSegments = segmentsAndData.segments().iterator(); + for (int i = 0; i < locationsData.length; i++) { + if (locationsData[i] != null) { + segmentsData[i] = presentSegments.next(); + } + } + + final NDArray index = new NDArray<>(locations.size, segmentsData); + final SegmentedReadData data = segmentsAndData.data(); + return new SegmentIndexAndData() { + @Override public NDArray index() {return index;} + @Override public SegmentedReadData data() {return data;} + }; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java deleted file mode 100644 index 8ac28b550..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardingCodec.java +++ /dev/null @@ -1,200 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.BlockCodec; -import org.janelia.saalfeldlab.n5.codec.DataCodec; -import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.codec.CodecInfo; -import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter; -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.serialization.N5Annotations; -import org.janelia.saalfeldlab.n5.serialization.NameConfig; - -import java.lang.reflect.Type; -import java.util.Objects; - -@NameConfig.Name(ShardingCodec.TYPE) -public class ShardingCodec { -//implements BlockCodecInfo { - -// private static final long serialVersionUID = -5879797314954717810L; -// - public static final String TYPE = "sharding_indexed"; -// -// public static final String CHUNK_SHAPE_KEY = "chunk_shape"; -// public static final String INDEX_LOCATION_KEY = "index_location"; -// public static final String CODECS_KEY = "codecs"; -// public static final String INDEX_CODECS_KEY = "index_codecs"; -// private DatasetAttributes attributes = null; -// -// public enum IndexLocation { -// START, END -// } -// -// @N5Annotations.ReverseArray // TODO need to reverse for zarr, not for n5 -// @NameConfig.Parameter(CHUNK_SHAPE_KEY) -// private final int[] blockSize; -// -// @NameConfig.Parameter(CODECS_KEY) -// private final CodecInfo[] codecs; -// -// @NameConfig.Parameter(INDEX_CODECS_KEY) -// private final IndexCodecAdapter indexCodecs; -// -// @NameConfig.Parameter(value = INDEX_LOCATION_KEY, optional = true) -// private final IndexLocation indexLocation; -// -// protected BlockCodec dataBlockSerializer = null; -// -// /** -// * Used via reflections by the NameConfig serializer. -// */ -// @SuppressWarnings("unused") -// private ShardingCodec() { -// -// blockSize = null; -// codecs = null; -// indexCodecs = null; -// indexLocation = IndexLocation.END; -// } -// -// public ShardingCodec( -// final int[] blockSize, -// final CodecInfo[] codecs, -// final IndexCodecAdapter indexCodecs, -// final IndexLocation indexLocation) { -// -// this.blockSize = blockSize; -// this.codecs = codecs; -// this.indexCodecs = indexCodecs; -// this.indexLocation = indexLocation; -// } -// -// public ShardingCodec( -// final int[] blockSize, -// final CodecInfo[] codecs, -// final CodecInfo[] indexCodecs, -// final IndexLocation indexLocation) { -// -// this(blockSize, codecs, IndexCodecAdapter.create(indexCodecs), indexLocation); -// } -// -// public IndexLocation getIndexLocation() { -// -// return indexLocation; -// } -// -// public BlockCodecInfo getBlockCodecInfo() { -// -// Objects.requireNonNull(codecs); -// if (codecs.length == 0) -// throw new IllegalArgumentException("Sharding CodecInfo requires a single BlockCodecInfo. None found."); -// -// return (BlockCodecInfo)codecs[0]; -// } -// public BlockCodec getBlockCodec() { -// -// return (BlockCodec)dataBlockSerializer; -// } -// -// public DataCodec[] getCodecs() { -// -// Objects.requireNonNull(codecs); -// final DataCodec[] dataCodecs = new DataCodec[codecs.length - 1]; -// for (int i = 1; i < codecs.length; i++) -// dataCodecs[i-1] = (DataCodec)codecs[i]; -// return dataCodecs; -// } -// -// public IndexCodecAdapter getIndexCodecAdapter() { -// -// return indexCodecs; -// } -// -// @Override -// public long[] getKeyPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { -// -// final long[] blockPosition = datablock.getGridPosition(); -// return attributes.getShardPositionForBlock(blockPosition); -// } -// -// @Override -// public long[] getKeyPositionForBlock(DatasetAttributes attributes, final long... blockPosition) { -// -// return attributes.getShardPositionForBlock(blockPosition); -// } -// -// public BlockCodec create(DatasetAttributes attributes, final DataCodec[] codecs) { -// -// // TODO -// return null; -// -//// this.attributes = attributes; -//// this.dataBlockSerializer = getBlockCodecInfo().create(attributes, getCodecs()); -//// return ((BlockCodec)dataBlockSerializer); -// } -// -// public ReadData encode(DataBlock dataBlock) { -// -// return this.getBlockCodec().encode(dataBlock); -// } -// -// public DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException { -// -// final ReadData splitableReadData = readData.materialize(); -// final long[] shardPosition = getKeyPositionForBlock(attributes, gridPosition); -// final VirtualShard shard = new VirtualShard<>(attributes, shardPosition, splitableReadData); -// final int[] relativeBlockPosition = shard.getRelativeBlockPosition(gridPosition); -// return shard.getBlock(relativeBlockPosition); -// } -// -// public ShardIndex createIndex(final DatasetAttributes attributes) { -// -// return new ShardIndex(attributes.getBlocksPerShard(), getIndexLocation(), getIndexCodecAdapter()); -// } -// -// @Override -// public String getType() { -// -// return TYPE; -// } -// -// public static IndexLocationAdapter indexLocationAdapter = new IndexLocationAdapter(); -// -// public static class IndexLocationAdapter implements JsonSerializer, JsonDeserializer { -// -// @Override -// public IndexLocation deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { -// -// if (!json.isJsonPrimitive()) -// return null; -// -// return IndexLocation.valueOf(json.getAsString().toUpperCase()); -// } -// -// @Override -// public JsonElement serialize(IndexLocation src, Type typeOfSrc, JsonSerializationContext context) { -// -// return new JsonPrimitive(src.name().toLowerCase()); -// } -// } -// -// @Override -// public BlockCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs) { -// -// // TODO -// return null; -// } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java deleted file mode 100644 index 65129cef6..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/VirtualShard.java +++ /dev/null @@ -1,130 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.util.GridIterator; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -//TODO : consider different names? LazyShard? ShardView? PartialReadShard? OnDemand? -public class VirtualShard extends AbstractShard { - -// private final ReadData shardData; -// -// public VirtualShard( -// final DatasetAttributes datasetAttributes, -// long[] gridPosition, -// final ReadData shardData) { -// -// super(datasetAttributes, gridPosition, null); -// this.shardData = shardData; -// } -// -// @SuppressWarnings("unchecked") -// public DataBlock getBlock(ReadData blockData, long... blockGridPosition) throws IOException { -// -// // TODO -// return null; -// } -// -// @Override -// public List> getBlocks() { -// -// return getBlocks(IntStream.range(0, getNumBlocks()).toArray()); -// } -// -// public List> getBlocks(final int[] blockIndexes) { -// -// // will not contain nulls -// final ShardIndex index = getIndex(); -// final ArrayList> blocks = new ArrayList<>(); -// -// if (index.isEmpty()) -// return blocks; -// -// // sort index offsets -// // and keep track of relevant positions -// final long[] indexData = index.getData(); -// List sortedOffsets = Arrays.stream(blockIndexes) -// .mapToObj(i -> new long[]{indexData[i * 2], i}) -// .filter(x -> x[0] != ShardIndex.EMPTY_INDEX_NBYTES) -// .sorted(Comparator.comparingLong(a -> ((long[])a)[0])) -// .collect(Collectors.toList()); -// -// final int nd = getDatasetAttributes().getNumDimensions(); -// long[] position = new long[nd]; -// -// final int[] blocksPerShard = getDatasetAttributes().getBlocksPerShard(); -// final long[] blockGridMin = IntStream.range(0, nd) -// .mapToLong(i -> blocksPerShard[i] * getGridPosition()[i]) -// .toArray(); -// -// for (long[] offsetIndex : sortedOffsets) { -// final long offset = offsetIndex[0]; -// if (offset < 0) -// continue; -// -// final long idx = offsetIndex[1]; -// GridIterator.indexToPosition(idx, blocksPerShard, blockGridMin, position); -// -// final long numBytes = index.getNumBytesByBlockIndex((int)idx); -// //TODO Caleb: Do this with a single access (start at first offset, read through the last) -// try { -// final ReadData blockData = shardData.slice(offset, numBytes); -// final DataBlock block = getBlock(blockData, position.clone()); -// blocks.add(block); -// } catch (final N5Exception.N5NoSuchKeyException e) { -// return blocks; -// } catch (final IOException | UncheckedIOException e) { -// throw new N5IOException("Failed to read block from " + Arrays.toString(position), e); -// } -// } -// return blocks; -// } -// -// @Override -// public DataBlock getBlock(int... relativePosition) { -// -// final ShardIndex idx = getIndex(); -// if (!idx.exists(relativePosition)) -// return null; -// -// final long blockOffset = idx.getOffset(relativePosition); -// final long blockSize = idx.getNumBytes(relativePosition); -// -// final long[] blockPosInDataset = getDatasetAttributes().getBlockPositionFromShardPosition(getGridPosition(), relativePosition); -// try { -// final ReadData blockData = shardData.slice(blockOffset, blockSize); -// return getBlock(blockData, blockPosInDataset); -// } catch (final N5Exception.N5NoSuchKeyException e) { -// return null; -// } catch (final IOException | UncheckedIOException e) { -// throw new N5IOException("Failed to read block from " + Arrays.toString(blockPosInDataset), e); -// } -// } -// -// @Override -// public ShardIndex getIndex() { -// -// // TODO -// return null; -// -//// index = createIndex(); -//// try { -//// ShardIndex.readFromShard(shardData, index); -//// } catch (N5Exception.N5NoSuchKeyException e) { -//// return null; -//// } -//// return index; -// } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java deleted file mode 100644 index 947473f76..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/shardstuff/ShardIndex.java +++ /dev/null @@ -1,235 +0,0 @@ -package org.janelia.saalfeldlab.n5.shardstuff; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.function.IntFunction; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.LongArrayDataBlock; -import org.janelia.saalfeldlab.n5.readdata.ReadData; -import org.janelia.saalfeldlab.n5.readdata.segment.Segment; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData.SegmentsAndData; - -public class ShardIndex { - - private ShardIndex() { - // utility class. should not be instantiated. - } - - public enum IndexLocation { - START, END - } - - /** - * Access flat {@code T[]} array as n-dimensional array. - * - * @param - * element type - */ - public static class NDArray { - - final int[] size; - private final int[] stride; - final T[] data; - - NDArray(final int[] size, final IntFunction createArray) { - this.size = size; - stride = getStrides(size); - data = createArray.apply(getNumElements(size)); - } - - NDArray(final int[] size, final T[] data) { - this.size = size; - stride = getStrides(size); - this.data = data; - } - - T get(long... position) { - return data[index(position)]; - } - - void set(T value, long... position) { - data[index(position)] = value; - } - - private int index(long... position) { - int index = 0; - for (int i = 0; i < stride.length; i++) { - index += stride[i] * position[i]; - } - return index; - } - - public int[] size() { - return size; - } - - public int numElements() { - return data.length; - } - - public boolean allElementsNull() { - for (T t : data) { - if (t != null) { - return false; - } - } - return true; - } - } - - static int getNumElements(final int[] size) { - int numElements = 1; - for (int s : size) { - numElements *= s; - } - return numElements; - } - - static int[] getStrides(final int[] size) { - final int n = size.length; - final int[] stride = new int[n]; - stride[0] = 1; - for (int i = 1; i < n; i++) { - stride[i] = stride[i - 1] * size[i - 1]; - } - return stride; - } - - /** - * Special value indicating an empty block entry in the index. - * Used for both offset and length when a block doesn't exist. - */ - static final long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; - - /** - * Size of first dimension of the {@code DataBlock} representation of the shard index. - */ - private static final int LONGS_PER_BLOCK = 2; - - static NDArray fromDataBlock( final DataBlock block ) { - - final long[] blockData = block.getData(); - final int[] size = indexSizeFromBlockSize(block.getSize()); - final int n = getNumElements(size); - final SegmentLocation[] locations = new SegmentLocation[n]; - - for (int i = 0; i < n; i++) { - long offset = blockData[i * LONGS_PER_BLOCK]; - long length = blockData[i * LONGS_PER_BLOCK + 1]; - if (offset != EMPTY_INDEX_NBYTES && length != EMPTY_INDEX_NBYTES) { - locations[i] = SegmentLocation.at(offset, length); - } - } - return new NDArray<>(size, locations); - } - - static DataBlock toDataBlock( final NDArray locations, final long offset ) { - - final SegmentLocation[] data = locations.data; - - final int[] blockSize = blockSizeFromIndexSize(locations.size); - final long[] blockData = new long[data.length * 2]; - - for (int i = 0; i < data.length; ++i) { - if (data[i] != null) { - blockData[i * LONGS_PER_BLOCK] = data[i].offset() + offset; - blockData[i * LONGS_PER_BLOCK + 1] = data[i].length(); - } else { - blockData[i * LONGS_PER_BLOCK] = EMPTY_INDEX_NBYTES; - blockData[i * LONGS_PER_BLOCK + 1] = EMPTY_INDEX_NBYTES; - } - } - return new LongArrayDataBlock(blockSize, new long[blockSize.length], blockData); - } - - /** - * Prepends a value to an array. - * - * @param value the value to prepend - * @param array the original array - * @return a new array with the value prepended - */ - private static int[] prepend(final int value, final int[] array) { - - final int[] indexBlockSize = new int[array.length + 1]; - indexBlockSize[0] = value; - System.arraycopy(array, 0, indexBlockSize, 1, array.length); - return indexBlockSize; - } - - /** - * Prepends {@code LONGS_PER_BLOCK} to the {@code indexSize} array. - */ - static int[] blockSizeFromIndexSize(final int[] indexSize) { - return prepend(LONGS_PER_BLOCK, indexSize); - } - - /** - * Strips first element (should be {@code LONGS_PER_BLOCK} from the {@code blockSize} array. - */ - static int[] indexSizeFromBlockSize(final int[] blockSize) { - assert blockSize[ 0 ] == LONGS_PER_BLOCK; - return Arrays.copyOfRange(blockSize, 1, blockSize.length); - } - - /** - * Retrieves the {@code SegmentLocation} of each non-null {@code Segment} in - * {@code segments}. Returns a {@code NDArray} with entries - * corresponding tho the {@code segments} entries. - */ - static NDArray locations(final NDArray segments, final SegmentedReadData readData) { - - final Segment[] data = segments.data; - final SegmentLocation[] locations = new SegmentLocation[data.length]; - for (int i = 0; i < data.length; ++i) { - final Segment segment = data[i]; - if ( segment != null ) { - locations[i] = readData.location(segment); - } - } - return new NDArray<>(segments.size, locations); - } - - interface SegmentIndexAndData { - NDArray index(); - SegmentedReadData data(); - } - - /** - * Puts a {@code Segment} at each non-null {@code SegmentLocation} in {@code - * locations} on the given {@code readData}. Returns both the {@code - * SegmentedReadData} with these segments and a {@code NDArray} - * with segment entries corresponding to the {@code locations} entries. - */ - static SegmentIndexAndData segments(final NDArray locations, final ReadData readData) { - - final SegmentLocation[] locationsData = locations.data; - final Segment[] segmentsData = new Segment[locationsData.length]; - - final List presentLocations = new ArrayList<>(); - for (int i = 0; i < locationsData.length; i++) { - if (locationsData[i] != null) { - presentLocations.add(locationsData[i]); - } - } - - final SegmentsAndData segmentsAndData = SegmentedReadData.wrap(readData, presentLocations); - final Iterator presentSegments = segmentsAndData.segments().iterator(); - for (int i = 0; i < locationsData.length; i++) { - if (locationsData[i] != null) { - segmentsData[i] = presentSegments.next(); - } - } - - final NDArray index = new NDArray<>(locations.size, segmentsData); - final SegmentedReadData data = segmentsAndData.data(); - return new SegmentIndexAndData() { - @Override public NDArray index() {return index;} - @Override public SegmentedReadData data() {return data;} - }; - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java index 327b7c7c9..e84f43e9a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java @@ -46,8 +46,6 @@ import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.shard.InMemoryShard; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.janelia.saalfeldlab.n5.util.Position; import org.junit.Test; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java deleted file mode 100644 index e41770ee7..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5BlockAsShardTest.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.janelia.saalfeldlab.n5; - -import com.google.gson.GsonBuilder; -import org.janelia.saalfeldlab.n5.shard.InMemoryShard; -import org.janelia.saalfeldlab.n5.shard.Shard; -import org.janelia.saalfeldlab.n5.shard.ShardIndex; -import org.junit.Ignore; - - -// TODO -@Ignore -public class N5BlockAsShardTest extends N5FSTest { - -// @Override -// protected N5Writer createN5Writer(String location, GsonBuilder gson) { -// -// return new N5FSWriter(location, gson) { -// -// @Override -// public void writeBlock(String path, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { -// -// final ShardIndex index = datasetAttributes.getShardingCodec().createIndex(datasetAttributes); -// final InMemoryShard shard = new InMemoryShard(datasetAttributes, dataBlock.getGridPosition(), index); -// shard.addBlock(dataBlock); -// writeShard(path, datasetAttributes, shard); -// } -// -// @Override -// public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { -// -// final Shard shard = readShard(pathName, datasetAttributes, gridPosition); -// if (shard == null) -// return null; -// -// return shard.getBlock(shard.getRelativeBlockPosition(gridPosition)); -// } -// }; -// } -// -// @Override -// protected N5Reader createN5Reader(String location, GsonBuilder gson) { -// -// return new N5FSReader(location, gson) { -// -// @Override -// public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { -// -// final Shard shard = readShard(pathName, datasetAttributes, gridPosition); -// if (shard == null) -// return null; -// -// return shard.getBlock(shard.getRelativeBlockPosition(gridPosition)); -// } -// }; -// } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java index a833d82b8..f83d2b0e3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java @@ -20,8 +20,8 @@ import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; -import org.janelia.saalfeldlab.n5.shardstuff.DatasetAccess; -import org.janelia.saalfeldlab.n5.shardstuff.PositionValueAccess; +import org.janelia.saalfeldlab.n5.shard.DatasetAccess; +import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; import org.janelia.saalfeldlab.n5.shardstuff.TestPositionValueAccess; import org.junit.Test; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java index 170f0e558..20b69082a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java @@ -15,7 +15,6 @@ import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo; -import org.janelia.saalfeldlab.n5.shard.ShardingCodec; import org.janelia.saalfeldlab.n5.util.GridIterator; public class BlockIterators { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index b05593e85..fd5a66170 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -40,9 +40,7 @@ import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.shard.Shard; import java.io.Serializable; import java.lang.reflect.Field; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index ba14ae454..840d04bf2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -22,8 +22,8 @@ import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; -import org.janelia.saalfeldlab.n5.shardstuff.DefaultShardCodecInfo; -import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.DefaultShardCodecInfo; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.junit.After; import org.junit.Assert; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java index 76d06ebb9..b3d286741 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java @@ -1,6 +1,6 @@ package org.janelia.saalfeldlab.n5.shardstuff; -import org.janelia.saalfeldlab.n5.shardstuff.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java index adc399508..5cfebf5d3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java @@ -10,7 +10,11 @@ import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.shardstuff.ShardIndex.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.DatasetAccess; +import org.janelia.saalfeldlab.n5.shard.DefaultShardCodecInfo; +import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; +import org.janelia.saalfeldlab.n5.shard.ShardCodecInfo; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; public class RawShardTest { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/TestPositionValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/TestPositionValueAccess.java index b97f36a84..5af490285 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/TestPositionValueAccess.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/TestPositionValueAccess.java @@ -6,6 +6,7 @@ import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; public class TestPositionValueAccess implements PositionValueAccess { From 7c5b609ef74a23975a06475f51c4d41073e205ed Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 21 Oct 2025 12:29:36 +0200 Subject: [PATCH 372/423] clean up --- .../saalfeldlab/n5/compression/CompressionTypesTest.java | 3 --- .../janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java | 3 --- .../saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java | 3 --- .../org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java | 3 --- 4 files changed, 12 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/compression/CompressionTypesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/compression/CompressionTypesTest.java index 25ecea36f..af467c856 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/compression/CompressionTypesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/compression/CompressionTypesTest.java @@ -26,9 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * - */ package org.janelia.saalfeldlab.n5.compression; import java.lang.reflect.Field; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java index 31f3e01af..ca65fd766 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java @@ -26,9 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * - */ package org.janelia.saalfeldlab.n5.kva; import org.janelia.saalfeldlab.n5.KeyValueAccess; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java index 7e0c7a912..ff1bf517f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java @@ -26,9 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * - */ package org.janelia.saalfeldlab.n5.kva; import static org.junit.Assert.assertArrayEquals; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java index 19e439799..486ae34f3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java @@ -26,9 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * - */ package org.janelia.saalfeldlab.n5.kva; import org.janelia.saalfeldlab.n5.AbstractN5Test; From efc6b5770d7e5a007e48d76dabc740591455a1e7 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 23 Sep 2025 21:21:25 +0200 Subject: [PATCH 373/423] Remove redundant modifier --- src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 61a5f6fa5..56c142403 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -335,7 +335,7 @@ static ReadData from(OutputStreamWriter generator) { * * @return an empty ReadData */ - public static ReadData empty() { + static ReadData empty() { return ByteArrayReadData.EMPTY; } From eeca2a4f02b8aab864c1d7823655ff2dcf3e2f4f Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 23 Sep 2025 21:28:00 +0200 Subject: [PATCH 374/423] Add prefetch() to ReadData interface default implementation does nothing --- .../saalfeldlab/n5/readdata/ReadData.java | 16 ++++++++++++++ .../segment/SliceTrackingReadData.java | 22 +++++++++++-------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 56c142403..e3e79352b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -32,7 +32,9 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Collection; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; /** * An abstraction over {@code byte[]} data. @@ -208,6 +210,20 @@ default void writeTo(OutputStream outputStream) throws N5IOException, IllegalSta } } + /** + * Indicates that the given slices will be subsequently read. + * {@code ReadData} implementations (optionally) may take steps to prepare + * for these subsequent slices. + * + * @param ranges + * slice ranges to prefetch + * + * @throws N5IOException + * if any I/O error occurs + */ + default void prefetch(final Collection ranges) throws N5IOException { + } + // ------------- Encoding / Decoding ---------------- // diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java index 2410353c1..57fc06e8c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java @@ -111,20 +111,23 @@ public ReadData encode(final OutputStreamOperator encoder) { return delegate.encode(encoder); } - - // --- -- - prefetching - -- --- - /** * Indicates that the given slices will be subsequently read. * {@code ReadData} implementations (optionally) may take steps to prepare * for these subsequent slices. + *

      + * Minimal implementation: Find offset and length covering all ranges that + * are not yet fully covered by existing slices. Then materialize the slice + * covering that range. + * + * @param ranges + * slice ranges to prefetch + * + * @throws N5IOException + * if any I/O error occurs */ - // TODO: where to put this? Could be in ReadData interface with empty default implementation? - public void prefetch(final Collection ranges) { - - // Minimal implementation: Find offset and length covering all ranges - // that are not yet fully covered by existing slices. Then materialize - // the slice covering that range. + @Override + public void prefetch(final Collection ranges) throws N5IOException { long fromIndex = Long.MAX_VALUE; long toIndex = Long.MIN_VALUE; @@ -141,6 +144,7 @@ public void prefetch(final Collection ranges) { } private boolean isCovered(final SegmentLocation slice) { + return Slices.findContainingSlice(slices, slice) != null; } } From 5a1d57c5ce0803d59033c478310091495d96a239 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 23 Sep 2025 21:31:47 +0200 Subject: [PATCH 375/423] DefaultSegmentedReadData forwards prefetch() to delegate --- .../n5/readdata/segment/DefaultSegmentedReadData.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java index 210fa8e9a..31319799a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java @@ -4,6 +4,7 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; @@ -244,6 +245,11 @@ public void writeTo(final OutputStream outputStream) throws N5IOException, Illeg delegate.writeTo(outputStream); } + @Override + public void prefetch(final Collection ranges) throws N5IOException { + delegate.prefetch(ranges); + } + /** * Returns a new ReadData that uses the given {@code OutputStreamOperator} to * encode this SegmentedReadData. From 4d2d201413fb6361e47747565ab101c5c136033f Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 21 Oct 2025 13:11:13 +0200 Subject: [PATCH 376/423] javadoc --- .../saalfeldlab/n5/readdata/ReadData.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index e3e79352b..67b912c68 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -34,6 +34,7 @@ import java.nio.ByteBuffer; import java.util.Collection; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.codec.DataCodec; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; /** @@ -48,10 +49,12 @@ *

      * {@code ReadData} may be lazy-loaded. For example, for {@code InputStream} and * {@code KeyValueAccess} sources, loading is deferred until the data is - * accessed (e.g., {@link #allBytes()}, {@link #writeTo(OutputStream)}). + * accessed (e.g., {@link #allBytes()}, {@link #writeTo(OutputStream)}), or + * explicitly {@link #materialize() meterialized}. *

      - * {@code ReadData} can be {@code encoded} and {@code decoded} with a {@code - * CodecInfo}, which will also be lazy if possible. + * {@code ReadData} can be {@link DataCodec#encode encoded} and {@link + * DataCodec#decode decoded} by a {@link DataCodec}, which will also be lazy if + * possible. */ public interface ReadData { @@ -241,7 +244,12 @@ default ReadData encode(OutputStreamOperator encoder) { } /** - * Like {@code UnaryOperator}, but {@code apply} throws {@code IOException}. + * {@code OutputStreamOperator} is {@link #apply applied} to an {@code + * OutputStream} to transform it into another(e.g., compressed) {@code + * OutputStream}. + *

      + * This is basically {@code UnaryOperator}, but {@link #apply} + * throws {@code IOException}. */ @FunctionalInterface interface OutputStreamOperator { From c8f0cad53778fa55b228e74bc16dd94c777e8e21 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 21 Oct 2025 13:24:44 +0200 Subject: [PATCH 377/423] clean up imports --- .../n5/shard/DefaultDatasetAccess.java | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 58992f87a..731f18fa5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -1,26 +1,19 @@ package org.janelia.saalfeldlab.n5.shard; - +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.TreeMap; +import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; import org.janelia.saalfeldlab.n5.codec.BlockCodec; -import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.TreeMap; -import java.util.stream.Collectors; - public class DefaultDatasetAccess implements DatasetAccess { private final NestedGrid grid; From d3b111a3f0a14f4edb8ddfab9d4e1ac8f94ca9ef Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 21 Oct 2025 13:41:30 +0200 Subject: [PATCH 378/423] Use SegmentLocation.at instead of constructor --- .../n5/readdata/segment/DefaultSegmentedReadData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java index 31319799a..6ed032354 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java @@ -133,7 +133,7 @@ static SegmentedReadData wrap(final ReadData readData) { public SegmentLocation location(final Segment segment) { if (segmentSource.equals(segment.source()) && segment instanceof SegmentLocation) { final SegmentLocation l = (SegmentLocation) segment; - return offset == 0 ? l : new DefaultSegmentLocation(l.offset() - offset, l.length()); + return offset == 0 ? l : SegmentLocation.at(l.offset() - offset, l.length()); } else { throw new IllegalArgumentException(); } From 6fb023c95250419816385601834a32b8c73608f0 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 21 Oct 2025 16:49:31 +0200 Subject: [PATCH 379/423] refactor: Make DefaultSegmentLocation a local class --- .../segment/DefaultSegmentLocation.java | 46 ------------------- .../n5/readdata/segment/SegmentLocation.java | 46 +++++++++++++++++++ 2 files changed, 46 insertions(+), 46 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentLocation.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentLocation.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentLocation.java deleted file mode 100644 index dc05775f3..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentLocation.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.janelia.saalfeldlab.n5.readdata.segment; - -class DefaultSegmentLocation implements SegmentLocation { - - private final long offset; - private final long length; - - public DefaultSegmentLocation(final long offset, final long length) { - this.offset = offset; - this.length = length; - } - - @Override - public long offset() { - return offset; - } - - @Override - public long length() { - return length; - } - - @Override - public final boolean equals(final Object o) { - if (!(o instanceof DefaultSegmentLocation)) - return false; - - final DefaultSegmentLocation that = (DefaultSegmentLocation) o; - return offset == that.offset && length == that.length; - } - - @Override - public int hashCode() { - int result = Long.hashCode(offset); - result = 31 * result + Long.hashCode(length); - return result; - } - - @Override - public String toString() { - return "SegmentLocation{" + - "offset=" + offset + - ", length=" + length + - '}'; - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java index 2c058667b..9d8880339 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java @@ -19,6 +19,52 @@ default long end() { } static SegmentLocation at(final long offset, final long length) { + + class DefaultSegmentLocation implements SegmentLocation { + + private final long offset; + private final long length; + + public DefaultSegmentLocation(final long offset, final long length) { + this.offset = offset; + this.length = length; + } + + @Override + public long offset() { + return offset; + } + + @Override + public long length() { + return length; + } + + @Override + public final boolean equals(final Object o) { + if (!(o instanceof DefaultSegmentLocation)) + return false; + + final DefaultSegmentLocation that = (DefaultSegmentLocation) o; + return offset == that.offset && length == that.length; + } + + @Override + public int hashCode() { + int result = Long.hashCode(offset); + result = 31 * result + Long.hashCode(length); + return result; + } + + @Override + public String toString() { + return "Range{" + + "offset=" + offset + + ", length=" + length + + '}'; + } + } + return new DefaultSegmentLocation(offset, length); } } From a4ea057148cc54f2d29ef24fb435a9af2d491ce1 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 21 Oct 2025 16:57:53 +0200 Subject: [PATCH 380/423] refactor: rename SegmentLocation to Range and move to readdata package --- .../SegmentLocation.java => Range.java} | 36 +++++++---- .../saalfeldlab/n5/readdata/ReadData.java | 3 +- .../n5/readdata/segment/Concatenate.java | 28 ++++----- .../segment/DefaultSegmentedReadData.java | 25 ++++---- .../readdata/segment/SegmentedReadData.java | 7 ++- .../segment/SliceTrackingReadData.java | 9 +-- .../n5/readdata/segment/Slices.java | 11 ++-- .../saalfeldlab/n5/shard/RawShardCodec.java | 8 +-- .../saalfeldlab/n5/shard/ShardIndex.java | 22 +++---- .../n5/readdata/segment/ConcatenateTest.java | 18 +++--- .../n5/readdata/segment/SegmentTest.java | 61 ++++++++++--------- .../n5/readdata/segment/SlicesTest.java | 23 +++---- 12 files changed, 133 insertions(+), 118 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/readdata/{segment/SegmentLocation.java => Range.java} (53%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java similarity index 53% rename from src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java rename to src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java index 9d8880339..bf2eef7c5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentLocation.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java @@ -1,31 +1,41 @@ -package org.janelia.saalfeldlab.n5.readdata.segment; +package org.janelia.saalfeldlab.n5.readdata; import java.util.Comparator; -// TODO: If we use this to describe slices, it should be renamed probably!? -// Ideas: Range, SliceLocation -public interface SegmentLocation { +/** + * A range specified as a {@link #offset}, {@link #length} pair. + */ +public interface Range { - Comparator COMPARATOR = Comparator - .comparingLong(SegmentLocation::offset) - .thenComparingLong(SegmentLocation::length); + Comparator COMPARATOR = Comparator + .comparingLong(Range::offset) + .thenComparingLong(Range::length); + /** + * @return start index (inclusive) + */ long offset(); + /** + * @return number of elements + */ long length(); + /** + * @return end index (exclusive) + */ default long end() { return offset() + length(); } - static SegmentLocation at(final long offset, final long length) { + static Range at(final long offset, final long length) { - class DefaultSegmentLocation implements SegmentLocation { + class DefaultRange implements Range { private final long offset; private final long length; - public DefaultSegmentLocation(final long offset, final long length) { + public DefaultRange(final long offset, final long length) { this.offset = offset; this.length = length; } @@ -42,10 +52,10 @@ public long length() { @Override public final boolean equals(final Object o) { - if (!(o instanceof DefaultSegmentLocation)) + if (!(o instanceof DefaultRange)) return false; - final DefaultSegmentLocation that = (DefaultSegmentLocation) o; + final DefaultRange that = (DefaultRange) o; return offset == that.offset && length == that.length; } @@ -65,6 +75,6 @@ public String toString() { } } - return new DefaultSegmentLocation(offset, length); + return new DefaultRange(offset, length); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 67b912c68..3f12e117a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -35,7 +35,6 @@ import java.util.Collection; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.codec.DataCodec; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; /** * An abstraction over {@code byte[]} data. @@ -224,7 +223,7 @@ default void writeTo(OutputStream outputStream) throws N5IOException, IllegalSta * @throws N5IOException * if any I/O error occurs */ - default void prefetch(final Collection ranges) throws N5IOException { + default void prefetch(final Collection ranges) throws N5IOException { } // ------------- Encoding / Decoding ---------------- diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java index fd956789e..1a1a4a4b0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java @@ -8,8 +8,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; class Concatenate implements SegmentedReadData { @@ -17,8 +17,8 @@ class Concatenate implements SegmentedReadData { private final List content; private final ReadData delegate; private final List segments; - private final List locations; - private final Map segmentToLocation; + private final List locations; + private final Map segmentToLocation; private boolean locationsBuilt; private long length; @@ -34,7 +34,7 @@ class Concatenate implements SegmentedReadData { // constructor for slices private Concatenate(final ReadData delegate, final List segments, - final List locations) { + final List locations) { content = null; this.delegate = delegate; this.segments = segments; @@ -62,8 +62,8 @@ private void ensureKnownSize() throws IllegalStateException { } segments.addAll(data.segments()); for (Segment segment : data.segments()) { - final SegmentLocation l = data.location(segment); - locations.add(SegmentLocation.at(l.offset() + offset, l.length())); + final Range l = data.location(segment); + locations.add(Range.at(l.offset() + offset, l.length())); } offset += data.length(); } @@ -78,9 +78,9 @@ private void ensureKnownSize() throws IllegalStateException { } @Override - public SegmentLocation location(final Segment segment) throws IllegalArgumentException { + public Range location(final Segment segment) throws IllegalArgumentException { ensureKnownSize(); - final SegmentLocation location = segmentToLocation.get(segment); + final Range location = segmentToLocation.get(segment); if (location == null) { throw new IllegalArgumentException(); } @@ -122,7 +122,7 @@ public long requireLength() throws N5IOException { @Override public SegmentedReadData slice(final Segment segment) throws IllegalArgumentException, N5IOException { ensureKnownSize(); - final SegmentLocation l = location(segment); + final Range l = location(segment); return slice(l.offset(), l.length()); } @@ -133,25 +133,25 @@ public SegmentedReadData slice(final long offset, final long length) throws N5IO final long sliceLength = delegateSlice.length(); // fromIndex: find first segment with offset >= sourceOffset - int fromIndex = Collections.binarySearch(locations, SegmentLocation.at(offset, -1), SegmentLocation.COMPARATOR); + int fromIndex = Collections.binarySearch(locations, Range.at(offset, -1), Range.COMPARATOR); if (fromIndex < 0) { fromIndex = -fromIndex - 1; } // toIndex: find first segment with offset >= sourceOffset + length - int toIndex = Collections.binarySearch(locations, SegmentLocation.at(offset + sliceLength, -1), SegmentLocation.COMPARATOR); + int toIndex = Collections.binarySearch(locations, Range.at(offset + sliceLength, -1), Range.COMPARATOR); if (toIndex < 0) { toIndex = -toIndex - 1; } // contained: find segments in [fromIndex, toIndex) with s.offset() + s.length() <= sourceOffset + length final List containedSegments = new ArrayList<>(); - final List containedSegmentLocations = new ArrayList<>(); + final List containedSegmentLocations = new ArrayList<>(); for (int i = fromIndex; i < toIndex; ++i) { - final SegmentLocation l = locations.get(i); + final Range l = locations.get(i); if (l.offset() + l.length() <= offset + sliceLength) { containedSegments.add(segments.get(i)); - containedSegmentLocations.add(SegmentLocation.at(l.offset() - offset, l.length())); + containedSegmentLocations.add(Range.at(l.offset() - offset, l.length())); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java index 6ed032354..41d38eb54 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java @@ -8,6 +8,7 @@ import java.util.Collections; import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; class DefaultSegmentedReadData implements SegmentedReadData { @@ -35,13 +36,13 @@ class DefaultSegmentedReadData implements SegmentedReadData { */ private final List segments; - private static class SegmentImpl implements Segment, SegmentLocation { + private static class SegmentImpl implements Segment, Range { private final SegmentedReadData source; private final long offset; private final long length; - public SegmentImpl(final SegmentedReadData source, final SegmentLocation location) { + public SegmentImpl(final SegmentedReadData source, final Range location) { this(source, location.offset(), location.length()); } @@ -95,14 +96,14 @@ private DefaultSegmentedReadData(final ReadData delegate, final List locations) { + static SegmentsAndData wrap(final ReadData readData, final List locations) { final List sortedSegments = new ArrayList<>(locations.size()); final DefaultSegmentedReadData data = new DefaultSegmentedReadData(readData, sortedSegments); - for (SegmentLocation l : locations) { + for (Range l : locations) { sortedSegments.add(new SegmentImpl(data, l)); } final List segments = new ArrayList<>(sortedSegments); - sortedSegments.sort(SegmentLocation.COMPARATOR); + sortedSegments.sort(Range.COMPARATOR); return new SegmentsAndData() { @@ -130,10 +131,10 @@ static SegmentedReadData wrap(final ReadData readData) { } @Override - public SegmentLocation location(final Segment segment) { - if (segmentSource.equals(segment.source()) && segment instanceof SegmentLocation) { - final SegmentLocation l = (SegmentLocation) segment; - return offset == 0 ? l : SegmentLocation.at(l.offset() - offset, l.length()); + public Range location(final Segment segment) { + if (segmentSource.equals(segment.source()) && segment instanceof Range) { + final Range l = (Range) segment; + return offset == 0 ? l : Range.at(l.offset() - offset, l.length()); } else { throw new IllegalArgumentException(); } @@ -180,7 +181,7 @@ public SegmentedReadData slice(final long offset, final long length) throws N5IO // fromIndex: find first segment with offset >= sourceOffset - int fromIndex = Collections.binarySearch(segments, SegmentLocation.at(sourceOffset, -1), SegmentLocation.COMPARATOR); + int fromIndex = Collections.binarySearch(segments, Range.at(sourceOffset, -1), Range.COMPARATOR); if (fromIndex < 0) { fromIndex = -fromIndex - 1; } @@ -190,7 +191,7 @@ public SegmentedReadData slice(final long offset, final long length) throws N5IO if (sliceLength < 0) { toIndex = segments.size(); } else { - toIndex = Collections.binarySearch(segments, SegmentLocation.at(sourceOffset + sliceLength, -1), SegmentLocation.COMPARATOR); + toIndex = Collections.binarySearch(segments, Range.at(sourceOffset + sliceLength, -1), Range.COMPARATOR); if (toIndex < 0) { toIndex = -toIndex - 1; } @@ -246,7 +247,7 @@ public void writeTo(final OutputStream outputStream) throws N5IOException, Illeg } @Override - public void prefetch(final Collection ranges) throws N5IOException { + public void prefetch(final Collection ranges) throws N5IOException { delegate.prefetch(ranges); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java index 687e70194..0bc505da2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java @@ -3,6 +3,7 @@ import java.util.Arrays; import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; public interface SegmentedReadData extends ReadData { @@ -27,7 +28,7 @@ static SegmentedReadData wrap(ReadData readData) { * matches the order of the given {@code locations} (while the segments in the * {@link SegmentsAndData#data()} are ordered by offset). */ - static SegmentsAndData wrap(ReadData readData, SegmentLocation... locations) { + static SegmentsAndData wrap(ReadData readData, Range... locations) { return wrap(readData, Arrays.asList(locations)); } @@ -37,7 +38,7 @@ static SegmentsAndData wrap(ReadData readData, SegmentLocation... locations) { * matches the order of the given {@code locations} (while the segments in the * {@link SegmentsAndData#data()} are ordered by offset). */ - static SegmentsAndData wrap(ReadData readData, List locations) { + static SegmentsAndData wrap(ReadData readData, List locations) { return DefaultSegmentedReadData.wrap(SliceTrackingReadData.wrap(readData), locations); } @@ -64,7 +65,7 @@ static SegmentedReadData concatenate(List readDatas) { * @throws IllegalArgumentException * if the segment is not contained in this ReadData */ - SegmentLocation location(Segment segment) throws IllegalArgumentException; + Range location(Segment segment) throws IllegalArgumentException; /** * @return all segments contained in this {@code ReadData}. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java index 57fc06e8c..81f9116d4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java @@ -7,11 +7,12 @@ import java.util.Collection; import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; class SliceTrackingReadData implements ReadData { - private static class Slice implements SegmentLocation { + private static class Slice implements Range { private final long offset; private final long length; @@ -127,11 +128,11 @@ public ReadData encode(final OutputStreamOperator encoder) { * if any I/O error occurs */ @Override - public void prefetch(final Collection ranges) throws N5IOException { + public void prefetch(final Collection ranges) throws N5IOException { long fromIndex = Long.MAX_VALUE; long toIndex = Long.MIN_VALUE; - for (final SegmentLocation slice : ranges) { + for (final Range slice : ranges) { if (!isCovered(slice)) { fromIndex = Math.min(fromIndex, slice.offset()); toIndex = Math.max(toIndex, slice.end()); @@ -143,7 +144,7 @@ public void prefetch(final Collection ranges) throws } } - private boolean isCovered(final SegmentLocation slice) { + private boolean isCovered(final Range slice) { return Slices.findContainingSlice(slices, slice) != null; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Slices.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Slices.java index 604d73847..3d60ded62 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Slices.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Slices.java @@ -3,6 +3,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import org.janelia.saalfeldlab.n5.readdata.Range; class Slices { @@ -24,10 +25,10 @@ private Slices() { * * @return */ - static T findContainingSlice(final List slices, final long offset, final long length) { + static T findContainingSlice(final List slices, final long offset, final long length) { // Find the slice s with largest s.offset <= offset. - final int i = Collections.binarySearch(slices, SegmentLocation.at(offset, 0), Comparator.comparingLong(SegmentLocation::offset)); + final int i = Collections.binarySearch(slices, Range.at(offset, 0), Comparator.comparingLong(Range::offset)); // Largest index of a slice with slice.offset <= offset. final int index = i < 0 ? -i - 2 : i; @@ -45,7 +46,7 @@ static T findContainingSlice(final List slices, f return slice; } - static T findContainingSlice(final List slices, final SegmentLocation range) { + static T findContainingSlice(final List slices, final Range range) { return findContainingSlice(slices, range.offset(), range.length()); } @@ -54,9 +55,9 @@ static T findContainingSlice(final List slices, f * Note, that the new {@code slice} is expected to not be fully contained in an existing slice! * This will insert {@code slice} into the list at the correct position ({@code slices} is ordered by slice offset), and remove all existing slices that are fully contained in the new {@code slice}. */ - static void addSlice(final List slices, final T slice) { + static void addSlice(final List slices, final T slice) { - final int i = Collections.binarySearch(slices, slice, Comparator.comparingLong(SegmentLocation::offset)); + final int i = Collections.binarySearch(slices, slice, Comparator.comparingLong(Range::offset)); final int from = i < 0 ? -i - 1 : i; int to = from; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java index adaeb864e..8ba859336 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java @@ -9,7 +9,7 @@ import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.segment.Segment; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; +import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; import org.janelia.saalfeldlab.n5.shard.ShardIndex.NDArray; @@ -51,7 +51,7 @@ public ReadData encode(final DataBlock shard) throws N5Exception.N5IOE final ReadData.OutputStreamWriter writer; if (indexLocation == START) { data.materialize(); - final NDArray locations = ShardIndex.locations(index, data); + final NDArray locations = ShardIndex.locations(index, data); final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, indexBlockSizeInBytes); final ReadData indexReadData = indexCodec.encode(indexDataBlock); writer = out -> { @@ -61,7 +61,7 @@ public ReadData encode(final DataBlock shard) throws N5Exception.N5IOE } else { // indexLocation == END writer = out -> { data.writeTo(out); - final NDArray locations = ShardIndex.locations(index, data); + final NDArray locations = ShardIndex.locations(index, data); final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, 0); final ReadData indexReadData = indexCodec.encode(indexDataBlock); indexReadData.writeTo(out); @@ -76,7 +76,7 @@ public DataBlock decode(final ReadData readData, final long[] gridPosi final long indexOffset = (indexLocation == START) ? 0 : (readData.requireLength() - indexBlockSizeInBytes); final ReadData indexReadData = readData.slice(indexOffset, indexBlockSizeInBytes); final DataBlock indexDataBlock = indexCodec.decode(indexReadData, new long[size.length]); - final NDArray locations = ShardIndex.fromDataBlock(indexDataBlock); + final NDArray locations = ShardIndex.fromDataBlock(indexDataBlock); final ShardIndex.SegmentIndexAndData segments = ShardIndex.segments(locations, readData); return new RawShardDataBlock(gridPosition, new RawShard(segments)); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index c0534675b..f2fa967a4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -9,7 +9,7 @@ import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.segment.Segment; -import org.janelia.saalfeldlab.n5.readdata.segment.SegmentLocation; +import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData.SegmentsAndData; @@ -110,26 +110,26 @@ static int[] getStrides(final int[] size) { */ private static final int LONGS_PER_BLOCK = 2; - static NDArray fromDataBlock( final DataBlock block ) { + static NDArray fromDataBlock( final DataBlock block ) { final long[] blockData = block.getData(); final int[] size = indexSizeFromBlockSize(block.getSize()); final int n = getNumElements(size); - final SegmentLocation[] locations = new SegmentLocation[n]; + final Range[] locations = new Range[n]; for (int i = 0; i < n; i++) { long offset = blockData[i * LONGS_PER_BLOCK]; long length = blockData[i * LONGS_PER_BLOCK + 1]; if (offset != EMPTY_INDEX_NBYTES && length != EMPTY_INDEX_NBYTES) { - locations[i] = SegmentLocation.at(offset, length); + locations[i] = Range.at(offset, length); } } return new NDArray<>(size, locations); } - static DataBlock toDataBlock( final NDArray locations, final long offset ) { + static DataBlock toDataBlock( final NDArray locations, final long offset ) { - final SegmentLocation[] data = locations.data; + final Range[] data = locations.data; final int[] blockSize = blockSizeFromIndexSize(locations.size); final long[] blockData = new long[data.length * 2]; @@ -181,10 +181,10 @@ static int[] indexSizeFromBlockSize(final int[] blockSize) { * {@code segments}. Returns a {@code NDArray} with entries * corresponding tho the {@code segments} entries. */ - static NDArray locations(final NDArray segments, final SegmentedReadData readData) { + static NDArray locations(final NDArray segments, final SegmentedReadData readData) { final Segment[] data = segments.data; - final SegmentLocation[] locations = new SegmentLocation[data.length]; + final Range[] locations = new Range[data.length]; for (int i = 0; i < data.length; ++i) { final Segment segment = data[i]; if ( segment != null ) { @@ -205,12 +205,12 @@ interface SegmentIndexAndData { * SegmentedReadData} with these segments and a {@code NDArray} * with segment entries corresponding to the {@code locations} entries. */ - static SegmentIndexAndData segments(final NDArray locations, final ReadData readData) { + static SegmentIndexAndData segments(final NDArray locations, final ReadData readData) { - final SegmentLocation[] locationsData = locations.data; + final Range[] locationsData = locations.data; final Segment[] segmentsData = new Segment[locationsData.length]; - final List presentLocations = new ArrayList<>(); + final List presentLocations = new ArrayList<>(); for (int i = 0; i < locationsData.length; i++) { if (locationsData[i] != null) { presentLocations.add(locationsData[i]); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenateTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenateTest.java index ae24adfae..2eb860f3b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenateTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenateTest.java @@ -1,10 +1,10 @@ package org.janelia.saalfeldlab.n5.readdata.segment; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.junit.Before; import org.junit.Test; @@ -31,10 +31,10 @@ public void createReadData() { @Test public void testConcatenate() { - final SegmentLocation[] locations = { - SegmentLocation.at(10, 30), - SegmentLocation.at(0, 10), - SegmentLocation.at(40, 20)}; + final Range[] locations = { + Range.at(10, 30), + Range.at(0, 10), + Range.at(40, 20)}; final SegmentedReadData.SegmentsAndData segmentsAndData0 = SegmentedReadData.wrap(readData, locations); final SegmentedReadData r0 = segmentsAndData0.data(); final List segments0 = segmentsAndData0.segments(); @@ -60,19 +60,19 @@ public void testConcatenate() { // } // System.out.println("c.location(segment) = " + c.location(segments1.get(0))); - final SegmentLocation l1 = c.location(segments0.get(1)); + final Range l1 = c.location(segments0.get(1)); assertEquals(0, l1.offset()); assertEquals(10, l1.length()); - final SegmentLocation l0 = c.location(segments0.get(0)); + final Range l0 = c.location(segments0.get(0)); assertEquals(10, l0.offset()); assertEquals(30, l0.length()); - final SegmentLocation l3 = c.location(segments1.get(0)); + final Range l3 = c.location(segments1.get(0)); assertEquals(40, l3.offset()); assertEquals(100, l3.length()); - final SegmentLocation l2 = c.location(segments0.get(2)); + final Range l2 = c.location(segments0.get(2)); assertEquals(140, l2.offset()); assertEquals(20, l2.length()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java index 83efabb1c..a53657337 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java @@ -2,6 +2,7 @@ import java.io.ByteArrayInputStream; import java.util.List; +import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.junit.Before; import org.junit.Test; @@ -28,22 +29,22 @@ public void createReadData() { @Test public void testWrap() { - final SegmentLocation[] locations = { - SegmentLocation.at(0, 10), - SegmentLocation.at(10, 10), - SegmentLocation.at(40, 20)}; + final Range[] locations = { + Range.at(0, 10), + Range.at(10, 10), + Range.at(40, 20)}; final SegmentedReadData r = SegmentedReadData.wrap(readData, locations).data(); assertEquals(3, r.segments().size()); - final SegmentLocation l0 = r.location(r.segments().get(0)); + final Range l0 = r.location(r.segments().get(0)); assertEquals(0, l0.offset()); assertEquals(10, l0.length()); - final SegmentLocation l1 = r.location(r.segments().get(1)); + final Range l1 = r.location(r.segments().get(1)); assertEquals(10, l1.offset()); assertEquals(10, l1.length()); - final SegmentLocation l2 = r.location(r.segments().get(2)); + final Range l2 = r.location(r.segments().get(2)); assertEquals(40, l2.offset()); assertEquals(20, l2.length()); } @@ -51,25 +52,25 @@ public void testWrap() { @Test public void testWrapOrder() { - final SegmentLocation[] locations = { - SegmentLocation.at(10, 10), - SegmentLocation.at(0, 10), - SegmentLocation.at(40, 20)}; + final Range[] locations = { + Range.at(10, 10), + Range.at(0, 10), + Range.at(40, 20)}; final SegmentedReadData.SegmentsAndData segmentsAndData = SegmentedReadData.wrap(readData, locations); final SegmentedReadData r = segmentsAndData.data(); final List segments = segmentsAndData.segments(); assertEquals(3, segments.size()); - final SegmentLocation l0 = r.location(segments.get(0)); + final Range l0 = r.location(segments.get(0)); assertEquals(10, l0.offset()); assertEquals(10, l0.length()); - final SegmentLocation l1 = r.location(segments.get(1)); + final Range l1 = r.location(segments.get(1)); assertEquals(0, l1.offset()); assertEquals(10, l1.length()); - final SegmentLocation l2 = r.location(segments.get(2)); + final Range l2 = r.location(segments.get(2)); assertEquals(40, l2.offset()); assertEquals(20, l2.length()); } @@ -77,21 +78,21 @@ public void testWrapOrder() { @Test public void testSlice() { - final SegmentLocation[] locations = { - SegmentLocation.at(0, 10), - SegmentLocation.at(10, 10), - SegmentLocation.at(40, 20)}; + final Range[] locations = { + Range.at(0, 10), + Range.at(10, 10), + Range.at(40, 20)}; final SegmentedReadData r = SegmentedReadData.wrap(readData, locations).data(); final SegmentedReadData s = r.slice(10, 60); assertEquals(60, s.length()); assertEquals(2, s.segments().size()); - final SegmentLocation l0 = s.location(s.segments().get(0)); + final Range l0 = s.location(s.segments().get(0)); assertEquals(0, l0.offset()); assertEquals(10, l0.length()); - final SegmentLocation l1 = s.location(s.segments().get(1)); + final Range l1 = s.location(s.segments().get(1)); assertEquals(30, l1.offset()); assertEquals(20, l1.length()); } @@ -99,17 +100,17 @@ public void testSlice() { @Test public void testSliceSegment() { - final SegmentLocation[] locations = { - SegmentLocation.at(0, 10), - SegmentLocation.at(10, 10), - SegmentLocation.at(40, 20)}; + final Range[] locations = { + Range.at(0, 10), + Range.at(10, 10), + Range.at(40, 20)}; final SegmentedReadData r = SegmentedReadData.wrap(readData, locations).data(); final SegmentedReadData s = r.slice(r.segments().get(2)); assertEquals(20, s.length()); assertEquals(1, s.segments().size()); - final SegmentLocation l0 = s.location(s.segments().get(0)); + final Range l0 = s.location(s.segments().get(0)); assertEquals(0, l0.offset()); assertEquals(20, l0.length()); } @@ -119,16 +120,16 @@ public void testWrapFully() { final SegmentedReadData r = SegmentedReadData.wrap(readDataUnknownLength); assertEquals(1, r.segments().size()); - final SegmentLocation l0 = r.location(r.segments().get(0)); + final Range l0 = r.location(r.segments().get(0)); assertEquals(0, l0.offset()); assertEquals(-1, l0.length()); final SegmentedReadData m = r.materialize(); - final SegmentLocation l0m = r.location(r.segments().get(0)); + final Range l0m = r.location(r.segments().get(0)); assertEquals(0, l0m.offset()); assertEquals(100, l0m.length()); - final SegmentLocation l0m2 = m.location(m.segments().get(0)); + final Range l0m2 = m.location(m.segments().get(0)); assertEquals(0, l0m2.offset()); assertEquals(100, l0m2.length()); } @@ -141,7 +142,7 @@ public void testSliceFullyWrapped() { assertEquals(1, s.segments().size()); - final SegmentLocation l0 = s.location(s.segments().get(0)); + final Range l0 = s.location(s.segments().get(0)); assertEquals(0, l0.offset()); assertEquals(100, l0.length()); @@ -163,7 +164,7 @@ public void testSliceSegmentFullyWrapped() { assertEquals(1, s.segments().size()); - final SegmentLocation l0 = s.location(s.segments().get(0)); + final Range l0 = s.location(s.segments().get(0)); assertEquals(0, l0.offset()); assertEquals(-1, l0.length()); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SlicesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SlicesTest.java index a005ef34a..3059f92af 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SlicesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SlicesTest.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import org.janelia.saalfeldlab.n5.readdata.Range; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -9,10 +10,10 @@ public class SlicesTest { - private List createSlices(final long[] offsets, final long[] lengths) { - final List slices = new ArrayList<>(); + private List createSlices(final long[] offsets, final long[] lengths) { + final List slices = new ArrayList<>(); for (int i = 0; i < offsets.length; ++i) { - slices.add(SegmentLocation.at(offsets[i], lengths[i])); + slices.add(Range.at(offsets[i], lengths[i])); } return slices; } @@ -25,10 +26,10 @@ public void testFindContaining() { // (6,4) [---------] // (8,6) [-----------] - final List slices = createSlices( + final List slices = createSlices( new long[] {2, 6, 8}, new long[] {6, 4, 6}); - SegmentLocation slice; + Range slice; // 0 1 2 3 4 5 6 7 8 9 A B C D E F // (1,1) [-] @@ -84,14 +85,14 @@ public void testAddSlice() { // (2,6) [-----------] // (6,4) [---------] // (8,6) [-----------] - final List initial = createSlices( + final List initial = createSlices( new long[] {2, 6, 8}, new long[] {6, 4, 6}); - List slices; + List slices; slices = new ArrayList<>(initial); - Slices.addSlice(slices, SegmentLocation.at(0, 1)); + Slices.addSlice(slices, Range.at(0, 1)); // 0 1 2 3 4 5 6 7 8 9 A B C D E F // (0,1) [-] // (2,6) [-----------] @@ -103,7 +104,7 @@ public void testAddSlice() { slices = new ArrayList<>(initial); - Slices.addSlice(slices, SegmentLocation.at(0, 16)); + Slices.addSlice(slices, Range.at(0, 16)); // 0 1 2 3 4 5 6 7 8 9 A B C D E F // (0,16)[-------------------------------] assertEquals(createSlices( @@ -112,7 +113,7 @@ public void testAddSlice() { slices = new ArrayList<>(initial); - Slices.addSlice(slices, SegmentLocation.at(2, 8)); + Slices.addSlice(slices, Range.at(2, 8)); // 0 1 2 3 4 5 6 7 8 9 A B C D E F // (2,8) [-----------------] // (8,6) [-----------] @@ -122,7 +123,7 @@ public void testAddSlice() { slices = new ArrayList<>(initial); - Slices.addSlice(slices, SegmentLocation.at(1, 10)); + Slices.addSlice(slices, Range.at(1, 10)); // 0 1 2 3 4 5 6 7 8 9 A B C D E F // (1,10) [---------------------] // (8,6) [-----------] From 81a4890c9f5a7b2a02db18c9d894393ff0a65eff Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 21 Oct 2025 13:03:22 -0400 Subject: [PATCH 381/423] fix: return empty list for readBlocks over nonexistent blocks, not null --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 4 - .../n5/shard/DefaultDatasetAccess.java | 10 +- .../saalfeldlab/n5/shard/ShardTest.java | 109 +++++++++++------- 3 files changed, 74 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index ff5c5256e..abeb461df 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -113,14 +113,10 @@ default List> readBlocks( final DatasetAttributes datasetAttributes, final List blockPositions) throws N5Exception { - try { final PositionValueAccess posKva = PositionValueAccess.fromKva( getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), p -> datasetAttributes.relativeBlockPath(p)); return datasetAttributes.getDatasetAccess().readBlocks(posKva, blockPositions); - } catch (N5Exception.N5NoSuchKeyException e) { - return null; - } } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 731f18fa5..6c4bae3d5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -71,13 +71,17 @@ public List> readBlocks(PositionValueAccess pva, List posit final NestedPosition firstBlock = blocksInSingleShard.get(0); final ReadData readData = pva.get(firstBlock.key()); - final List> shardBlocks = readShardRecursive(readData, blocksInSingleShard, outermostLevel); + final List> shardBlocks; + try { + shardBlocks = readShardRecursive(readData, blocksInSingleShard, outermostLevel); + } catch (N5NoSuchKeyException e) { + continue; + } blocks.addAll(shardBlocks); } //TODO Caleb: No guarantee of order; If we want that, we need to sort the result. - //TODO Caleb: Also no guarantee of uniqueness. May want to use a Set instead of a list if we care. return blocks; } @@ -189,6 +193,8 @@ private ReadData writeBlockRecursive( final NestedPositionDataBlock firstDataBlock = dataBlocksForShard.get(0); final long[] shardKey = firstDataBlock.key(); + //TODO Caleb: When writing all blocks in a shard, we don't need to read existing data. + // Also, could only materialize the index and skip reading if we are overwriting all existing blocks. final ReadData existingReadData = getExistingReadData(pva, shardKey); final ReadData writeShardReadData = writeShardRecursive(existingReadData, dataBlocksForShard, outermostLevel); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 840d04bf2..4d1249a5b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -52,8 +52,10 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Random; +import java.util.stream.Stream; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -257,54 +259,75 @@ public void writeReadBlocksTest() { } final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; - for (long[] blockIndex : newBlockIndices) { + final List newBlockIndexList = Arrays.asList(newBlockIndices); + final List> readBlocks = writer.readBlocks(dataset, datasetAttributes, newBlockIndexList); + for (int i = 0; i < newBlockIndices.length; i++) { + final long[] blockIndex = newBlockIndices[i]; final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])block.getData()); + final DataBlock blockFromReadBlocks = readBlocks.get(i); + Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])blockFromReadBlocks.getData()); + } + } + + @Test + public void readBlocksTest() { + + final N5Writer n5 = tempN5Factory.createTempN5Writer(); + final DatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + new int[]{2, 2} + ); + + final String dataset = "writeReadBlocks"; + final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; + final List> readBlocks = n5.readBlocks(dataset, datasetAttributes, Arrays.asList(newBlockIndices)); + System.out.println(readBlocks.size()); + } + + @Test + public void writeReadBlockTest() { + + final N5Writer writer = tempN5Factory.createTempN5Writer(); + final DatasetAttributes datasetAttributes = getTestAttributes(); + + final String dataset = "writeReadBlock"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); + writer.deleteBlock(dataset, 0, 0); //FIXME Caleb: We are abusing this here. It shouldn't delete the entire shard.. + + final int[] blockSize = datasetAttributes.getBlockSize(); + final DataType dataType = datasetAttributes.getDataType(); + final int numElements = 2 * 2; + + final HashMap writtenBlocks = new HashMap<>(); + + for (int idx1 = 1; idx1 >= 0; idx1--) { + for (int idx2 = 1; idx2 >= 0; idx2--) { + final long[] gridPosition = {idx1, idx2}; + final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(blockSize, gridPosition, numElements); + byte[] data = dataBlock.getData(); + for (int i = 0; i < data.length; i++) { + data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); + } + writer.writeBlock(dataset, datasetAttributes, dataBlock); + + final DataBlock block = writer.readBlock(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + + for (Map.Entry entry : writtenBlocks.entrySet()) { + final long[] otherGridPosition = entry.getKey(); + final byte[] otherData = entry.getValue(); + final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); + Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); + } + + writtenBlocks.put(gridPosition, data); + } } } -// @Test -// public void writeReadBlockTest() { -// -// final N5Writer writer = tempN5Factory.createTempN5Writer(); -// final DatasetAttributes datasetAttributes = getTestAttributes(); -// -// final String dataset = "writeReadBlock"; -// writer.remove(dataset); -// writer.createDataset(dataset, datasetAttributes); -// writer.deleteBlock(dataset, 0, 0); //FIXME Caleb: We are abusing this here. It shouldn't delete the entire shard.. -// -// final int[] blockSize = datasetAttributes.getBlockSize(); -// final DataType dataType = datasetAttributes.getDataType(); -// final int numElements = 2 * 2; -// -// final HashMap writtenBlocks = new HashMap<>(); -// -// for (int idx1 = 1; idx1 >= 0; idx1--) { -// for (int idx2 = 1; idx2 >= 0; idx2--) { -// final long[] gridPosition = {idx1, idx2}; -// final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(blockSize, gridPosition, numElements); -// byte[] data = dataBlock.getData(); -// for (int i = 0; i < data.length; i++) { -// data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); -// } -// writer.writeBlock(dataset, datasetAttributes, dataBlock); -// -// final DataBlock block = writer.readBlock(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); -// Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); -// -// for (Map.Entry entry : writtenBlocks.entrySet()) { -// final long[] otherGridPosition = entry.getKey(); -// final byte[] otherData = entry.getValue(); -// final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); -// Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); -// } -// -// writtenBlocks.put(gridPosition, data); -// } -// } -// } -// // @Test // public void writeReadShardTest() { // From f807bd767f86ec4f2e157a9b9754e79ffb3dcec3 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 21 Oct 2025 21:51:52 +0200 Subject: [PATCH 382/423] javadoc --- .../saalfeldlab/n5/readdata/Range.java | 4 ++ .../saalfeldlab/n5/readdata/ReadData.java | 16 ++++- .../readdata/segment/SegmentedReadData.java | 70 +++++++++++++++---- 3 files changed, 73 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java index bf2eef7c5..90e6e52dd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java @@ -7,6 +7,10 @@ */ public interface Range { + /** + * Order {@code Range}s by {@link #offset}. + * Ranges with the same offset are ordered by {@link #length}. + */ Comparator COMPARATOR = Comparator .comparingLong(Range::offset) .thenComparingLong(Range::length); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 3f12e117a..11224a30a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -98,8 +98,8 @@ default ReadData limit(final long length) throws N5IOException { * Returns a new {@link ReadData} representing a slice, or subset * of this ReadData. * - * @param offset the offset relative to this - * @param length of the returned ReadData + * @param offset the offset relative to this ReadData + * @param length length of the returned ReadData * @return a slice * @throws N5IOException an exception */ @@ -107,6 +107,18 @@ default ReadData slice(final long offset, final long length) throws N5IOExceptio return materialize().slice(offset, length); } + /** + * Returns a new {@link ReadData} representing a slice, or subset + * of this ReadData. + * + * @param range a range in this ReadData + * @return a slice + * @throws N5IOException an exception + */ + default ReadData slice(final Range range) throws N5IOException { + return slice(range.offset(), range.length()); + } + /** * Open a {@code InputStream} on this data. *

      diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java index 0bc505da2..f0d572705 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java @@ -6,6 +6,15 @@ import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; +/** + * A {@code ReadData} which keeps track of contained {@link Segment}s. + *

      + * A {@code Segment} refers to the data in a particular {@link Range}. + * (That range can be obtained using {@link #location(Segment)}). Segments are + * pulled along when {@link #slice(long, long)} slicing} or {@link #concatenate + * concatenating} {@code SegmentedReadData} (and will have appropriately offset + * {@link #location locations} in the slice/concatenation). + */ public interface SegmentedReadData extends ReadData { interface SegmentsAndData { @@ -25,8 +34,8 @@ static SegmentedReadData wrap(ReadData readData) { /** * Wrap {@code readData} and create segments at the given locations. The * order of segments in the returned {@link SegmentsAndData#segments()} list - * matches the order of the given {@code locations} (while the segments in the - * {@link SegmentsAndData#data()} are ordered by offset). + * matches the order of the given {@code locations} (while the {@link + * #segments} in the {@link SegmentsAndData#data()} are ordered by offset). */ static SegmentsAndData wrap(ReadData readData, Range... locations) { return wrap(readData, Arrays.asList(locations)); @@ -35,8 +44,8 @@ static SegmentsAndData wrap(ReadData readData, Range... locations) { /** * Wrap {@code readData} and create segments at the given locations. The * order of segments in the returned {@link SegmentsAndData#segments()} list - * matches the order of the given {@code locations} (while the segments in the - * {@link SegmentsAndData#data()} are ordered by offset). + * matches the order of the given {@code locations} (while the {@link + * #segments} in the {@link SegmentsAndData#data()} are ordered by offset). */ static SegmentsAndData wrap(ReadData readData, List locations) { return DefaultSegmentedReadData.wrap(SliceTrackingReadData.wrap(readData), locations); @@ -49,13 +58,13 @@ static SegmentedReadData concatenate(List readDatas) { /** - * Returns the {@code SegmentLocation} of {@code segment} in this {@code ReadData}. + * Returns the location of {@code segment} in this {@code ReadData}. *

      * Note that this {@code ReadData} is not necessarily the source of the segment. *

      - * The returned {@code SegmentLocation} may be {@code {offset=0, index=-1}}, which + * The returned {@code Range} may be {@code {offset=0, length=-1}}, which * means that the segment comprises this whole {@code ReadData} (and the length of - * this {@code ReadData} is not yet known. + * this {@code ReadData} is not yet known). * * @param segment * the segment id @@ -68,9 +77,11 @@ static SegmentedReadData concatenate(List readDatas) { Range location(Segment segment) throws IllegalArgumentException; /** + * Return all segments (fully) contained in this {@code ReadData}, ordered by location + * (that is, sorted by {@link Range#COMPARATOR}). + * * @return all segments contained in this {@code ReadData}. */ - // Order is the same as the SegmentLocations given at construction List segments(); @Override @@ -79,21 +90,50 @@ default SegmentedReadData limit(final long length) throws N5Exception.N5IOExcept } /** - * Return a {@code SegmentedReadData} wrapping a slice containing exactly the given segment. - * - * @param segment - * - * @return + * Return a {@code SegmentedReadData} wrapping a slice containing exactly + * the given segment. + *

      + * The {@link #location} of the given {@code segment} in this ReadData + * specifies the range to slice. * + * @param segment segment to slice + * @return a slice * @throws IllegalArgumentException * if the segment is not contained in this ReadData * @throws N5Exception.N5IOException */ SegmentedReadData slice(Segment segment) throws IllegalArgumentException, N5Exception.N5IOException; - // TODO: has all segments fully contained in requested slice. + /** + * Returns a new {@link SegmentedReadData} representing a slice, or subset + * of this ReadData. + *

      + * The {@link #segments} of the returned SegmentedReadData are all segments + * fully contained in the requested range. + * + * @param offset the offset relative to this ReadData + * @param length length of the returned ReadData + * @return a slice + * @throws N5Exception.N5IOException an exception + */ + @Override + SegmentedReadData slice(long offset, long length) throws N5Exception.N5IOException; + + /** + * Returns a new {@link SegmentedReadData} representing a slice, or subset + * of this ReadData. + *

      + * The {@link #segments} of the returned SegmentedReadData are all segments + * fully contained in the requested range. + * + * @param range a range in this ReadData + * @return a slice + * @throws N5Exception.N5IOException an exception + */ @Override - SegmentedReadData slice(final long offset, final long length) throws N5Exception.N5IOException; + default SegmentedReadData slice(final Range range) throws N5Exception.N5IOException { + return slice(range.offset(), range.length()); + } @Override SegmentedReadData materialize() throws N5Exception.N5IOException; From de5ec1a98f710d02018a4b48202142866cd49081 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Oct 2025 10:30:22 +0200 Subject: [PATCH 383/423] refactor: rename Concatenate to ConcatenatedReadData --- .../{Concatenate.java => ConcatenatedReadData.java} | 8 ++++---- .../n5/readdata/segment/SegmentedReadData.java | 2 +- ...ConcatenateTest.java => ConcatenatedReadDataTest.java} | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/{Concatenate.java => ConcatenatedReadData.java} (94%) rename src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/{ConcatenateTest.java => ConcatenatedReadDataTest.java} (98%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadData.java similarity index 94% rename from src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java rename to src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadData.java index 1a1a4a4b0..22b541676 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Concatenate.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadData.java @@ -12,7 +12,7 @@ import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; -class Concatenate implements SegmentedReadData { +class ConcatenatedReadData implements SegmentedReadData { private final List content; private final ReadData delegate; @@ -22,7 +22,7 @@ class Concatenate implements SegmentedReadData { private boolean locationsBuilt; private long length; - Concatenate(final List content) { + ConcatenatedReadData(final List content) { this.content = content; delegate = ReadData.from(os -> content.forEach(d -> d.writeTo(os))); segments = new ArrayList<>(); @@ -33,7 +33,7 @@ class Concatenate implements SegmentedReadData { } // constructor for slices - private Concatenate(final ReadData delegate, final List segments, + private ConcatenatedReadData(final ReadData delegate, final List segments, final List locations) { content = null; this.delegate = delegate; @@ -155,7 +155,7 @@ public SegmentedReadData slice(final long offset, final long length) throws N5IO } } - return new Concatenate(delegateSlice, containedSegments, containedSegmentLocations); + return new ConcatenatedReadData(delegateSlice, containedSegments, containedSegmentLocations); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java index f0d572705..5438a22b7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java @@ -52,7 +52,7 @@ static SegmentsAndData wrap(ReadData readData, List locations) { } static SegmentedReadData concatenate(List readDatas) { - return new Concatenate(readDatas); + return new ConcatenatedReadData(readDatas); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenateTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadDataTest.java similarity index 98% rename from src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenateTest.java rename to src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadDataTest.java index 2eb860f3b..4b847fb67 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenateTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadDataTest.java @@ -11,7 +11,7 @@ import static org.junit.Assert.assertEquals; -public class ConcatenateTest { +public class ConcatenatedReadDataTest { private ReadData readData; From d987cd461608fce2e3415c00175cefc63d8cc3f7 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Oct 2025 12:16:40 +0200 Subject: [PATCH 384/423] augment and document concatenate test --- .../segment/ConcatenatedReadDataTest.java | 193 ++++++++++++++---- 1 file changed, 153 insertions(+), 40 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadDataTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadDataTest.java index 4b847fb67..73ece4d91 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadDataTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadDataTest.java @@ -1,11 +1,13 @@ package org.janelia.saalfeldlab.n5.readdata.segment; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData.SegmentsAndData; import org.junit.Before; import org.junit.Test; @@ -13,70 +15,181 @@ public class ConcatenatedReadDataTest { - private ReadData readData; - - private ReadData readDataUnknownLength; + private final byte[] data = new byte[100]; @Before - public void createReadData() { + public void fillData() { - final byte[] data = new byte[100]; for (int i = 0; i < data.length; i++) { data[i] = (byte) i; } - readData = ReadData.from(data); - readDataUnknownLength = ReadData.from(new ByteArrayInputStream(data)); } - @Test - public void testConcatenate() { - - final Range[] locations = { + /** + * Create a SegmentedReadData with segments at (10,l=30), (0,l=10), and (40,l=20). + * The returned ReadData knows its length. + *

      + * 

      +	 * [0.....10.....20.....30.....40.....50.....60.....70.....80.....90.....]
+	 * [(-s1-)(---------s0--------)(-----s2-----)............................]
+	 *
      + */ + private SegmentsAndData createKnownLength() { + + return SegmentedReadData.wrap( + ReadData.from(data), Range.at(10, 30), Range.at(0, 10), - Range.at(40, 20)}; - final SegmentedReadData.SegmentsAndData segmentsAndData0 = SegmentedReadData.wrap(readData, locations); + Range.at(40, 20)); + } + + /** + * Create a SegmentedReadData with one segment spanning it completely. + * The returned ReadData doesn't know its length. + *

      + * 

      +	 * [0.....10.....20.....30.....40.....50.....60.....70.....80.....90.....]
+	 * [(------------------------------s3-----------------------------------)]
+	 *
      + */ + private SegmentsAndData createUnknownLength() { + + final SegmentedReadData srd = SegmentedReadData.wrap( + ReadData.from( + new ByteArrayInputStream(data))); + + return new SegmentsAndData() { + + @Override + public List segments() { + return Collections.singletonList(srd.segments().get(0)); + } + + @Override + public SegmentedReadData data() { + return srd; + } + }; + } + + + /** + * Create slices of known length and unknown length SegmentedReadData, and concatenate. + * Take slice (0,l=40) + *

      + * 

      +	 *
+	 * KNOWN_LENGTH:
+	 * [0.....10.....20.....30.....40.....50.....60.....70.....80.....90.....]
+	 * [(-s1-)(---------s0--------)(-----s2-----)............................]
+	 *
+	 * SLICE_0
+	 * [(-s1-)(---------s0--------)]
+	 *
+	 *                            SLICE_1
+	 *                            [(-----s2-----)]
+	 *
+	 * UNKNOWN_LENGTH:
+	 * [0.....10.....20.....30.....40.....50.....60.....70.....80.....90.....]
+	 * [(------------------------------s3-----------------------------------)]
+	 *
+	 * CONCATENATED:
+	 * [-------- SLICE_0 ----------][- UNKNOWN_LENGTH -][-- SLICE_1 ---]
+	 * [(-s1-)(---------s0--------)][(--...--s3--...--)][(-----s2-----)]
+	 *
+	 *
      + */ + private SegmentsAndData createConcatenate() { + + final SegmentsAndData segmentsAndData0 = createKnownLength(); final SegmentedReadData r0 = segmentsAndData0.data(); - final List segments0 = segmentsAndData0.segments(); - final SegmentedReadData r1 = SegmentedReadData.wrap(readDataUnknownLength); - assertEquals(1, r1.segments().size()); - final List segments1 = Collections.singletonList(r1.segments().get(0)); + final SegmentsAndData segmentsAndData1 = createUnknownLength(); + final SegmentedReadData r1 = segmentsAndData1.data(); + + final List segments = new ArrayList<>(); + segments.addAll(segmentsAndData0.segments()); + segments.addAll(segmentsAndData1.segments()); final List datas = new ArrayList<>(); datas.add(r0.slice(0,40)); datas.add(r1); - datas.add(r0.slice(segments0.get(2))); - final SegmentedReadData c = SegmentedReadData.concatenate(datas); + datas.add(r0.slice(segments.get(2))); + final SegmentedReadData concatenated = SegmentedReadData.concatenate(datas); - // TODO: create individual tests: - // Both materialize() and writeTo(OutputStream) should ensure that all SegmentLocations are known - // Otherwise we expect IllegalStateException - c.materialize(); -// c.writeTo(new ByteArrayOutputStream()); + return new SegmentsAndData() { -// for (Segment segment : segments0) { -// System.out.println("c.location(segment) = " + c.location(segment)); -// } -// System.out.println("c.location(segment) = " + c.location(segments1.get(0))); + @Override + public List segments() { + return segments; + } - final Range l1 = c.location(segments0.get(1)); - assertEquals(0, l1.offset()); - assertEquals(10, l1.length()); + @Override + public SegmentedReadData data() { + return concatenated; + } + }; + } - final Range l0 = c.location(segments0.get(0)); - assertEquals(10, l0.offset()); - assertEquals(30, l0.length()); + /** + * Check that segments in the concatenated ReadData are at the expected locations. + * The segments should be laid out like this: + *

      + * 

      +	 * CONCATENATED:
+	 * [0.....10.....20.....30.....40...             ...140.....150.....]
+	 * [-------- SLICE_0 ----------][- UNKNOWN_LENGTH -][--- SLICE_1 ---]
+	 * [(-s1-)(---------s0--------)][(--...--s3--...--)][(------s2-----)]
+	 *
      + *

      + */ + private static void checkSegmentLocations(final SegmentsAndData segmentsAndData) { + checkSegmentRange(segmentsAndData,1, Range.at(0, 10)); + checkSegmentRange(segmentsAndData,0, Range.at(10, 30)); + checkSegmentRange(segmentsAndData,3, Range.at(40, 100)); + checkSegmentRange(segmentsAndData,2, Range.at(140, 20)); + } - final Range l3 = c.location(segments1.get(0)); - assertEquals(40, l3.offset()); - assertEquals(100, l3.length()); + private static void checkSegmentRange(final SegmentsAndData data, final int segmentIndex, final Range expectedLocation) + { + final Range location = data.data().location(data.segments().get(segmentIndex)); + assertEquals(expectedLocation.offset(), location.offset()); + assertEquals(expectedLocation.length(), location.length()); + } + + // A concatenated ReadData containing unknown-length parts requires + // either materialize() or writeTo(OutputStream) to make those lengths + // known. Otherwise, trying to get segment locations from the + // concatenated ReadData fails with an IllegalStateException. + + @Test(expected = IllegalStateException.class) + public void testConcatenateUnmaterialized() { + + final SegmentsAndData segmentsAndData = createConcatenate(); + final SegmentedReadData c = segmentsAndData.data(); + final List s = segmentsAndData.segments(); + System.out.println(c.location(s.get(0))); + } - final Range l2 = c.location(segments0.get(2)); - assertEquals(140, l2.offset()); - assertEquals(20, l2.length()); + // Calling materialize() on the concatenated ReadData makes sure that all + // segment offsets are known. + @Test + public void testConcatenateMaterialize() { + final SegmentsAndData segmentsAndData = createConcatenate(); + segmentsAndData.data().materialize(); + checkSegmentLocations(segmentsAndData); } + // Calling writeTo(OutputStream) on the concatenated ReadData makes sure + // that all segment offsets are known. + + @Test + public void testConcatenateWriteTo() { + + final SegmentsAndData segmentsAndData = createConcatenate(); + segmentsAndData.data().writeTo(new ByteArrayOutputStream()); + checkSegmentLocations(segmentsAndData); + } } From e980a041f8689c857b40d3db6d4633fe5504de47 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Oct 2025 12:17:17 +0200 Subject: [PATCH 385/423] Add static Range.equals() to check two Ranges for equality --- .../org/janelia/saalfeldlab/n5/readdata/Range.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java index 90e6e52dd..51c1f8ef8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java @@ -32,6 +32,16 @@ default long end() { return offset() + length(); } + static boolean equals(final Range r0, final Range r1) { + if (r0 == null && r1==null) { + return true; + } else if (r0 == null || r1 == null) { + return false; + } else { + return r0.offset() == r1.offset() && r0.length() == r1.length(); + } + } + static Range at(final long offset, final long length) { class DefaultRange implements Range { From 387a10b8b32dad10ed97cfb2c87597b29b968698 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Oct 2025 13:00:36 +0200 Subject: [PATCH 386/423] javadoc --- .../segment/ConcatenatedReadData.java | 14 +++++++++++ .../segment/DefaultSegmentedReadData.java | 24 +++++++++++++++---- .../n5/readdata/segment/Segment.java | 2 +- .../readdata/segment/SegmentedReadData.java | 15 +++++++++++- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadData.java index 22b541676..66331a77c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/ConcatenatedReadData.java @@ -12,6 +12,20 @@ import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; +/** + * Implementation of a {@link SegmentedReadData} representing the concatenation + * of several {@code SegmentedReadData}s. + *

      + * {@code ConcatenatedReadData} contains the segments of all concatenated {@code + * SegmentedReadData}s with appropriately offset locations. + *

      + * In particular, it is also possible to concatenate {@code SegmentedReadData}s + * with (yet) unknown length. (This is useful for postponing compression of + * DataBlocks until they are actually written.) In that case, segment locations + * are only available, after all lengths become known. This happens when this + * {@code ConcatenatedReadData} (or all its constituents) is + * {@link #materialize() materialized} or {@link #writeTo(OutputStream) written}. + */ class ConcatenatedReadData implements SegmentedReadData { private final List content; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java index 41d38eb54..0dd22de35 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/DefaultSegmentedReadData.java @@ -11,6 +11,15 @@ import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; +/** + * Implementation of a {@link SegmentedReadData} wrapper around an existing {@code ReadData} delegate. + *

      + * It is used for + *

        + *
      • adding segment information to vanilla ReadData (see {@link SegmentedReadData#wrap(ReadData, Range...)}),
        • + *
      • representing slices into other {@code DefaultSegmentedReadData}.
        • + *
      locations) { final List sortedSegments = new ArrayList<>(locations.size()); final DefaultSegmentedReadData data = new DefaultSegmentedReadData(readData, sortedSegments); @@ -119,17 +134,16 @@ public SegmentedReadData data() { }; } - private DefaultSegmentedReadData(final ReadData delegate) { + /** + * Wrap the given {@code delegate} with a single segment fully containing it. + */ + DefaultSegmentedReadData(final ReadData delegate) { this.delegate = delegate; this.segmentSource = this; this.offset = 0; this.segments = Collections.singletonList(new EnclosingSegmentImpl(this)); } - static SegmentedReadData wrap(final ReadData readData) { - return new DefaultSegmentedReadData(readData); - } - @Override public Range location(final Segment segment) { if (segmentSource.equals(segment.source()) && segment instanceof Range) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java index 38d98ad73..75fdde9ec 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java @@ -1,7 +1,7 @@ package org.janelia.saalfeldlab.n5.readdata.segment; /** - * A particular segment in a source {@code ReadData}. + * A particular segment in a source {@link SegmentedReadData}. */ public interface Segment { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java index 5438a22b7..f7676c9de 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.readdata.segment; +import java.io.OutputStream; import java.util.Arrays; import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception; @@ -28,7 +29,7 @@ interface SegmentsAndData { * of {@link SegmentedReadData#segments()}. */ static SegmentedReadData wrap(ReadData readData) { - return DefaultSegmentedReadData.wrap(SliceTrackingReadData.wrap(readData)); + return new DefaultSegmentedReadData(SliceTrackingReadData.wrap(readData)); } /** @@ -51,6 +52,18 @@ static SegmentsAndData wrap(ReadData readData, List locations) { return DefaultSegmentedReadData.wrap(SliceTrackingReadData.wrap(readData), locations); } + /** + * Return a {@link SegmentedReadData} representing the concatenation of the + * given {@code readDatas}. The concatenation contains the segments of all + * concatenated {@code readData}s with appropriately offset locations. + *

      + * In particular, it is also possible to concatenate {@code SegmentedReadData}s + * with (yet) unknown length. (This is useful for postponing compression of + * DataBlocks until they are actually written.) In that case, segment locations + * are only available after all lengths become known. This happens when + * concatenation (or all its constituents) is {@link #materialize() + * materialized} or {@link #writeTo(OutputStream) written}. + */ static SegmentedReadData concatenate(List readDatas) { return new ConcatenatedReadData(readDatas); } From 80d9e01dab6e10802f5fff0df13f6b187b0de1ff Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 22 Oct 2025 13:39:45 -0400 Subject: [PATCH 387/423] fix: IdentityCodec implements DeterministicSizeDataCodec --- .../org/janelia/saalfeldlab/n5/codec/IdentityCodec.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java index c1dd36f9f..046b99b67 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -4,7 +4,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @NameConfig.Name(IdentityCodec.TYPE) -public class IdentityCodec implements DataCodec, DataCodecInfo { +public class IdentityCodec implements DeterministicSizeDataCodec, DataCodecInfo { private static final long serialVersionUID = 8354269325800855621L; @@ -32,4 +32,10 @@ public ReadData encode(ReadData readData) { return this; } + + @Override + public long encodedSize(long size) { + + return size; + } } From 6c99a00dbe75b306f9f2f6a241341a0a81db3b19 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 22 Oct 2025 13:41:23 -0400 Subject: [PATCH 388/423] wip!: DatasetAttributes changes * remove constructor with shard size * createDatasetAccess method now protected (for zarr) * rm defaultShardCodecInfo (n5 format does not use it) --- .../saalfeldlab/n5/DatasetAttributes.java | 33 +---------------- .../org/janelia/saalfeldlab/n5/N5Writer.java | 32 ++-------------- .../saalfeldlab/n5/AbstractN5Test.java | 2 +- .../saalfeldlab/n5/DatasetAttributesTest.java | 37 ++++++++++++++----- .../n5/http/HttpReaderFsWriter.java | 6 --- 5 files changed, 32 insertions(+), 78 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 1140bd090..619d5790c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -40,13 +40,10 @@ import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.shard.DatasetAccess; import org.janelia.saalfeldlab.n5.shard.DefaultDatasetAccess; -import org.janelia.saalfeldlab.n5.shard.DefaultShardCodecInfo; import org.janelia.saalfeldlab.n5.shard.ShardCodecInfo; import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; -import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; import java.io.Serializable; import java.lang.reflect.Type; @@ -160,25 +157,7 @@ public DatasetAttributes( this(dimensions, blockSize, dataType, new DataCodecInfo[0]); } - /** - * Constructs a DatasetAttributes instance with specified dimensions, block size, data type, and default codecs - * - * @param dimensions the dimensions of the dataset - * @param blockSize the size of the blocks in the dataset - * @param dataType the data type of the dataset - */ - public DatasetAttributes( - final long[] dimensions, - final int[] shardSize, - final int[] blockSize, - final DataType dataType) { - - // TODO add compression arg - this(dimensions, shardSize, dataType, - defaultShardCodecInfo(blockSize)); - } - - private DatasetAccess createDatasetAccess() { + protected DatasetAccess createDatasetAccess() { final int m = nestingDepth(blockCodecInfo); @@ -223,16 +202,6 @@ private static int nestingDepth(BlockCodecInfo info) { } } - protected static BlockCodecInfo defaultShardCodecInfo(int[] innerBlockSize) { - - return new DefaultShardCodecInfo( - innerBlockSize, - new N5BlockCodecInfo(), // TODO call default method - new DataCodecInfo[]{new RawCompression()}, - new RawBlockCodecInfo(), - new DataCodecInfo[]{new RawCompression()}, - IndexLocation.END); - } protected BlockCodecInfo defaultBlockCodecInfo() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 5bd85e6d9..aec867d40 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -221,42 +221,16 @@ default void createDataset( * @param blockSize the block size * @param dataType the data type * @param blockCodecInfo the block codec - * @param dataCodecs data codecs + * @param compression the compression */ default void createDataset( final String datasetPath, final long[] dimensions, final int[] blockSize, final DataType dataType, - final DataCodecInfo... dataCodecInfos) throws N5Exception { + final Compression compression) throws N5Exception { - // TODO default block codec? - // TODO better doc - createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, dataCodecInfos)); - } - - /** - * Creates a dataset. This does not create any data but the path and - * mandatory attributes only. - * - * @param datasetPath dataset path - * @param dimensions the dataset dimensions - * @param blockSize the block size - * @param dataType the data type - * @param blockCodecInfo the block codec - * @param dataCodecs data codecs - */ - default void createDataset( - final String datasetPath, - final long[] dimensions, - final int[] blockSize, - final DataType dataType, - final BlockCodecInfo blockCodecInfo, - final DataCodecInfo[] dataCodecInfos) throws N5Exception { - - // TODO default block codec? - // TODO better doc - createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, blockCodecInfo, dataCodecInfos)); + createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression)); } /** diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 2cf394c42..3e9daf2ab 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -235,7 +235,7 @@ public void testCreateDataset() { final DatasetAttributes info; try (N5Writer writer = createTempN5Writer()) { - writer.createDataset(datasetName, dimensions, blockSize, DataType.UINT64); + writer.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression()); assertTrue("Dataset does not exist", writer.exists(datasetName)); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java index e84f43e9a..9f2d52810 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java @@ -43,9 +43,12 @@ import java.util.stream.StreamSupport; import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.shard.DefaultShardCodecInfo; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; import org.janelia.saalfeldlab.n5.util.GridIterator; import org.janelia.saalfeldlab.n5.util.Position; import org.junit.Test; @@ -68,7 +71,7 @@ public void testValidateBlockShardSizesValid() { DataType dataType = DataType.UINT8; // This should not throw any exception - DatasetAttributes attrs = new DatasetAttributes(dimensions, shardSize, blockSize, dataType); + DatasetAttributes attrs = shardDatasetAttributes(dimensions, shardSize, blockSize, dataType); // assertEquals(shardSize, attrs.getShardSize()); assertEquals(blockSize, attrs.getBlockSize()); // assertArrayEquals(new int[]{1, 1, 1}, attrs.getBlocksPerShard()); @@ -76,7 +79,7 @@ public void testValidateBlockShardSizesValid() { // Test case 2: shard size is a multiple of block size shardSize = new int[]{128}; blockSize = new int[]{64}; - attrs = new DatasetAttributes(new long[]{128}, shardSize, blockSize, dataType); + attrs = shardDatasetAttributes(new long[]{128}, shardSize, blockSize, dataType); // assertEquals(shardSize, attrs.getShardSize()); assertEquals(blockSize, attrs.getBlockSize()); // assertArrayEquals(new int[]{2}, attrs.getBlocksPerShard()); @@ -84,7 +87,7 @@ public void testValidateBlockShardSizesValid() { // Test case 3: different multiples per dimension shardSize = new int[]{128, 256, 32, 2}; blockSize = new int[]{32, 64, 32, 1}; - attrs = new DatasetAttributes(new long[]{128, 128, 128, 128}, shardSize, blockSize, dataType ); + attrs = shardDatasetAttributes(new long[]{128, 128, 128, 128}, shardSize, blockSize, dataType ); // assertEquals(shardSize, attrs.getShardSize()); assertEquals(blockSize, attrs.getBlockSize()); // assertArrayEquals(new int[]{4, 4, 1, 2}, attrs.getBlocksPerShard()); @@ -92,12 +95,26 @@ public void testValidateBlockShardSizesValid() { // Test case 4: large multiples shardSize = new int[]{1024, 2048, 512}; blockSize = new int[]{32, 64, 16}; - attrs = new DatasetAttributes(dimensions, shardSize, blockSize, dataType); + attrs = shardDatasetAttributes(dimensions, shardSize, blockSize, dataType); // assertEquals(shardSize, attrs.getShardSize()); assertEquals(blockSize, attrs.getBlockSize()); // assertArrayEquals(new int[]{32, 32, 32}, attrs.getBlocksPerShard()); } + private static DatasetAttributes shardDatasetAttributes( + long[] dimensions, int[] shardSize, int[] blockSize, DataType dataType) { + + DefaultShardCodecInfo blockCodecInfo = new DefaultShardCodecInfo( + blockSize, + new N5BlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + IndexLocation.END); + + return new DatasetAttributes(dimensions, shardSize, dataType, blockCodecInfo); + } + /** * Test that validateBlockShardSizes method rejects invalid shard and block size combinations. */ @@ -110,37 +127,37 @@ public void testValidateBlockShardSizesInvalid() { // Block size too small IllegalArgumentException ex0 = assertThrows( IllegalArgumentException.class, - () -> new DatasetAttributes(dimensions, new int[]{1, 1, 1}, new int[]{1, 0, -1}, dataType)); + () -> shardDatasetAttributes(dimensions, new int[]{1, 1, 1}, new int[]{1, 0, -1}, dataType)); assertTrue(ex0.getMessage().contains("negative")); // Different number of dimensions IllegalArgumentException ex1 = assertThrows( IllegalArgumentException.class, - () -> new DatasetAttributes(dimensions, new int[]{64, 64}, new int[]{32, 32, 32}, dataType)); + () -> shardDatasetAttributes(dimensions, new int[]{64, 64}, new int[]{32, 32, 32}, dataType)); assertTrue(ex1.getMessage().contains("different length")); // Shard size smaller than block size IllegalArgumentException ex2 = assertThrows( IllegalArgumentException.class, - () -> new DatasetAttributes(dimensions, new int[]{32, 64, 64}, new int[]{64, 64, 64}, dataType)); + () -> shardDatasetAttributes(dimensions, new int[]{32, 64, 64}, new int[]{64, 64, 64}, dataType)); assertTrue(ex2.getMessage().contains("is smaller than previous")); // Shard size not a multiple of block size IllegalArgumentException ex3 = assertThrows( IllegalArgumentException.class, - () -> new DatasetAttributes(dimensions, new int[]{100, 100, 100}, new int[]{64, 64, 64}, dataType)); + () -> shardDatasetAttributes(dimensions, new int[]{100, 100, 100}, new int[]{64, 64, 64}, dataType)); assertTrue(ex3.getMessage().contains("not a multiple of previous level")); // Multiple violations - shard smaller than block in one dimension IllegalArgumentException ex4 = assertThrows( IllegalArgumentException.class, - () -> new DatasetAttributes(dimensions, new int[]{128, 32, 128}, new int[]{64, 64, 64}, dataType)); + () -> shardDatasetAttributes(dimensions, new int[]{128, 32, 128}, new int[]{64, 64, 64}, dataType)); assertTrue(ex4.getMessage().contains("is smaller than previous")); // Edge case - shard size of 0 assertThrows( IllegalArgumentException.class, - () -> new DatasetAttributes(dimensions, new int[]{0, 64, 64}, new int[]{64, 64, 64}, dataType)); + () -> shardDatasetAttributes(dimensions, new int[]{0, 64, 64}, new int[]{64, 64, 64}, dataType)); } // @Test diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index fd5a66170..4c7e62331 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -302,10 +302,4 @@ public HttpRead writer.writeBlocks(datasetPath, datasetAttributes, dataBlocks); } - - @Override public void createDataset(String datasetPath, long[] dimensions, int[] blockSize, DataType dataType, - BlockCodecInfo blockCodecInfo, DataCodecInfo[] dataCodecInfos) throws N5Exception { - - writer.createDataset(datasetPath, dimensions, blockSize, dataType, blockCodecInfo, dataCodecInfos); - } } From 9186e2d066caa7597c93ce72fee845d01874ff8b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 22 Oct 2025 13:42:12 -0400 Subject: [PATCH 389/423] fix: DefaultShardCodecInfo's serialization * default serialization should be for zarr --- .../saalfeldlab/n5/codec/CodecInfo.java | 3 +- .../n5/shard/DefaultShardCodecInfo.java | 90 +++++++++---------- .../saalfeldlab/n5/shard/ShardIndex.java | 5 +- 3 files changed, 47 insertions(+), 51 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java index 79221a1eb..a39032dab 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java @@ -1,11 +1,10 @@ package org.janelia.saalfeldlab.n5.codec; import java.io.Serializable; -import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; /** - * {@code CodecInfo}s can encode and decode {@link ReadData} objects. + * {@code CodecInfo}s are an untyped semantic layer for {@link BlockCodec}s and {@link DataCodec}s. *

      * Modeled after Codecs in * Zarr. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java index 25d7e5cda..7b7cc0473 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java @@ -1,41 +1,47 @@ package org.janelia.saalfeldlab.n5.shard; -import java.lang.reflect.Type; import java.util.Arrays; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.serialization.N5Annotations; import org.janelia.saalfeldlab.n5.serialization.NameConfig; import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; - /** * Default (and probably only) implementation of {@link ShardCodecInfo}. */ -// TODO rename? -@NameConfig.Name(value = "ShardingCodec") +@NameConfig.Name(value = "sharding_indexed") public class DefaultShardCodecInfo implements ShardCodecInfo { @Override public String getType() { - return "ShardingCodec"; + return "sharding_indexed"; } + @N5Annotations.ReverseArray + @NameConfig.Parameter(value = "chunk_shape") private final int[] innerBlockSize; - private final BlockCodecInfo innerBlockCodecInfo; - private final DataCodecInfo[] innerDataCodecInfos; - private final BlockCodecInfo indexBlockCodecInfo; - private final DataCodecInfo[] indexDataCodecInfos; + + @NameConfig.Parameter(value = "index_location") private final IndexLocation indexLocation; + @NameConfig.Parameter + private CodecInfo[] codecs; + + @NameConfig.Parameter(value = "index_codecs") + private CodecInfo[] indexCodecs; + + private transient final BlockCodecInfo innerBlockCodecInfo; + + private transient final DataCodecInfo[] innerDataCodecInfos; + + private transient final BlockCodecInfo indexBlockCodecInfo; + + private transient final DataCodecInfo[] indexDataCodecInfos; + DefaultShardCodecInfo() { // for serialization this(null, null, null, null, null, null); @@ -48,12 +54,16 @@ public DefaultShardCodecInfo( final BlockCodecInfo indexBlockCodecInfo, final DataCodecInfo[] indexDataCodecInfos, final IndexLocation indexLocation) { + this.innerBlockSize = innerBlockSize; this.innerBlockCodecInfo = innerBlockCodecInfo; this.innerDataCodecInfos = innerDataCodecInfos; this.indexBlockCodecInfo = indexBlockCodecInfo; this.indexDataCodecInfos = indexDataCodecInfos; this.indexLocation = indexLocation; + + codecs = concatenateCodecs(innerBlockCodecInfo, innerDataCodecInfos); + indexCodecs = concatenateCodecs(indexBlockCodecInfo, indexDataCodecInfos); } @Override @@ -86,6 +96,14 @@ public IndexLocation getIndexLocation() { return indexLocation; } + public CodecInfo[] getCodecs() { + return codecs; + } + + public CodecInfo[] getIndexCodecs() { + return indexCodecs; + } + @Override public RawShardCodec create(final int[] blockSize, final DataCodecInfo... codecs) { @@ -103,40 +121,16 @@ public RawShardCodec create(final int[] blockSize, final DataCodecInfo... codecs return new RawShardCodec(size, indexLocation, indexCodec); } - public static DefaultShardCodecInfoAdapter adapter = new DefaultShardCodecInfoAdapter(); - - public static class DefaultShardCodecInfoAdapter implements JsonDeserializer, JsonSerializer { - - @Override - public DefaultShardCodecInfo deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { - - if (!json.isJsonObject()) - return null; + private static CodecInfo[] concatenateCodecs(BlockCodecInfo blkInfo, DataCodecInfo[] dataInfos) { - JsonObject obj = json.getAsJsonObject(); - - int[] innerBlockSize = context.deserialize(obj.get("innerBlockSize"), int[].class); - BlockCodecInfo innerBlockCodecInfo = context.deserialize(obj.get("innerBlockCodecInfo"), BlockCodecInfo[].class); - DataCodecInfo[] innerDataCodecInfos = context.deserialize(obj.get("innerDataCodecInfos"), DataCodecInfo[].class); - BlockCodecInfo indexBlockCodecInfo = context.deserialize(obj.get("indexBlockCodecInfo"), BlockCodecInfo[].class); - DataCodecInfo[] indexDataCodecInfos = context.deserialize(obj.get("indexDataCodecInfos"), DataCodecInfo[].class); - IndexLocation indexLocation = IndexLocation.valueOf(obj.get("indexLocation").getAsString()); - - return new DefaultShardCodecInfo(innerBlockSize, innerBlockCodecInfo, innerDataCodecInfos, indexBlockCodecInfo, indexDataCodecInfos, indexLocation); + if (dataInfos == null) { + return new CodecInfo[]{blkInfo}; } - @Override - public JsonElement serialize(DefaultShardCodecInfo src, Type typeOfSrc, JsonSerializationContext context) { - - final JsonObject obj = new JsonObject(); - obj.add("innerBlockSize", context.serialize(src.innerBlockSize)); - obj.add("innerBlockCodecInfo", context.serialize(src.innerBlockCodecInfo)); - obj.add("innerDataCodecInfos", context.serialize(src.innerDataCodecInfos)); - obj.add("indexBlockCodecInfo", context.serialize(src.indexBlockCodecInfo)); - obj.add("indexDataCodecInfos", context.serialize(src.indexDataCodecInfos)); - obj.add("indexLocation", context.serialize(src.indexLocation)); - return obj; - } - } + final CodecInfo[] allCodecs = new CodecInfo[dataInfos.length + 1]; + allCodecs[0] = blkInfo; + System.arraycopy(dataInfos, 0, allCodecs, 1, dataInfos.length); + return allCodecs; + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java index f2fa967a4..d5d25eda7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -13,6 +13,8 @@ import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData.SegmentsAndData; +import com.google.gson.annotations.SerializedName; + public class ShardIndex { private ShardIndex() { @@ -20,7 +22,8 @@ private ShardIndex() { } public enum IndexLocation { - START, END + @SerializedName("start") START, + @SerializedName("end") END } /** From 7120b3875f06f132ae19f6d7d5ac75634f4699c5 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 22 Oct 2025 13:42:30 -0400 Subject: [PATCH 390/423] test: update getTestAttributes given dstAttribute changes --- .../org/janelia/saalfeldlab/n5/shard/ShardTest.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 4d1249a5b..76e948b61 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -139,11 +139,19 @@ public void removeTempWriters() { private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] blockSize) { + DefaultShardCodecInfo blockCodec = new DefaultShardCodecInfo( + blockSize, + new N5BlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + IndexLocation.END); + return new DatasetAttributes( dimensions, shardSize, - blockSize, - DataType.UINT8); + DataType.UINT8, + blockCodec); } private DatasetAttributes getTestAttributes() { @@ -610,7 +618,6 @@ public ShardedN5Writer(String basePath, GsonBuilder gsonBuilder) { gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); // gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, new TestDatasetAttributesAdapter()); gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBlockCodecInfo.byteOrderAdapter); - gsonBuilder.registerTypeHierarchyAdapter(DefaultShardCodecInfo.class, DefaultShardCodecInfo.adapter); gsonBuilder.disableHtmlEscaping(); gson = gsonBuilder.create(); } From 0b8ffed749064807ea8ecd4bbf97f672a8546b1f Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Oct 2025 22:06:37 +0200 Subject: [PATCH 391/423] javadoc --- .../saalfeldlab/n5/readdata/segment/Segment.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java index 75fdde9ec..3b2eea470 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Segment.java @@ -5,5 +5,17 @@ */ public interface Segment { + /** + * Returns the {@code SegmentedReadData} on which this segment is originally + * defined. (The segment is tracked through slices and concatenations, but + * the source will remain the same.) + *

      + * This is mostly just used internally to make {@code SegmentedReadData} + * implementations easier. The only real use for {@code source()} outside of + * that is to get a {@code ReadData} containing exactly this segment, using + * {@code segment.source().slice(segment)}. + * + * @return the {@code SegmentedReadData} on which this segment is originally defined + */ SegmentedReadData source(); } From 5eab7660bfb74cbac3fda4a773f0b330fd0d3ff4 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 22 Oct 2025 16:09:28 -0400 Subject: [PATCH 392/423] refactor/doc: NestedGrid * rename NestedGrid fields * add some basic javadoc --- .../janelia/saalfeldlab/n5/shard/Nesting.java | 77 +++++++++++-------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java index 16b454bb9..cd4a2f586 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java @@ -141,29 +141,26 @@ public String toString() { // TODO: should we have prefix()? suffix()? head()? tail()? } - /** - * TODO better property names + * A nested grid of blocks used to coordinate the relationships of shards and the blocks / chunks they contain. */ public static class NestedGrid { - // num levels - private final int m; + private final int numLevels; - // num dimensions - private final int n; + private final int numDimensions; - // s[i][d] is block size at level i relative to level 0 - private final int[][] s; + // relativeToBase[i][d] is block size at level i relative to level 0 + private final int[][] relativeToBase; - // r[i][d] is block size at level i relative to level i-1 - private final int[][] r; + // relativeToAdjacent[i][d] is block size at level i relative to level i-1 + private final int[][] relativeToAdjacent; private final int[][] blockSizes; /** * {@code blockSizes[l][d]} is the block size at level {@code l} in dimension {@code d}. - * Level 0 is the highest resolution (smallest block sizes). + * Level 0 contains the smallest blocks. blockSizes[l+1][d] must be a multiple of blockSizes[l][d]. * * @param blockSizes * block sizes for all levels and dimensions. @@ -178,21 +175,21 @@ public NestedGrid(int[][] blockSizes) { this.blockSizes = blockSizes; - m = blockSizes.length; - n = blockSizes[0].length; - s = new int[m][n]; - r = new int[m][n]; - for (int l = 0; l < m; ++l) { + numLevels = blockSizes.length; + numDimensions = blockSizes[0].length; + relativeToBase = new int[numLevels][numDimensions]; + relativeToAdjacent = new int[numLevels][numDimensions]; + for (int l = 0; l < numLevels; ++l) { final int k = Math.max(0, l - 1); if (blockSizes[l] == null) throw new IllegalArgumentException("blockSizes[" + l + "] null"); - if (blockSizes[l].length != n) + if (blockSizes[l].length != numDimensions) throw new IllegalArgumentException( - String.format("Block size at level %d has a different length (%d vs %d)", l, n, blockSizes[l].length)); + String.format("Block size at level %d has a different length (%d vs %d)", l, numDimensions, blockSizes[l].length)); - for (int d = 0; d < n; ++d) { + for (int d = 0; d < numDimensions; ++d) { if (blockSizes[l][d] <= 0 ) { throw new IllegalArgumentException( @@ -214,18 +211,18 @@ public NestedGrid(int[][] blockSizes) { l, blockSizes[l][d], blockSizes[k][d], d)); } - s[l][d] = blockSizes[l][d] / blockSizes[0][d]; - r[l][d] = blockSizes[l][d] / blockSizes[k][d]; + relativeToBase[l][d] = blockSizes[l][d] / blockSizes[0][d]; + relativeToAdjacent[l][d] = blockSizes[l][d] / blockSizes[k][d]; } } } public int numLevels() { - return m; + return numLevels; } public int numDimensions() { - return n; + return numDimensions; } public int[] getBlockSize(int level) { @@ -237,9 +234,9 @@ public void absolutePosition( final int sourceLevel, final long[] targetPos, final int targetLevel) { - final int[] sk = s[sourceLevel]; - final int[] si = s[targetLevel]; - for (int d = 0; d < n; ++d) { + final int[] sk = relativeToBase[sourceLevel]; + final int[] si = relativeToBase[targetLevel]; + for (int d = 0; d < numDimensions; ++d) { targetPos[d] = sourcePos[d] * sk[d] / si[d]; } } @@ -250,23 +247,39 @@ public void relativePosition( final long[] targetPos, final int targetLevel) { absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel); - if (targetLevel < m - 1) { - final int[] rj = r[targetLevel + 1]; - for (int d = 0; d < n; ++d) { + if (targetLevel < numLevels - 1) { + final int[] rj = relativeToAdjacent[targetLevel + 1]; + for (int d = 0; d < numDimensions; ++d) { targetPos[d] %= rj[d]; } } } + /** + * The absolute position of * source position for the given source level at the target level. + * + * @param sourcePos the source position j + * @param sourceLevel the source level + * @param targetLevel the target level + * @return absolute position at the target level + */ public long[] absolutePosition( final long[] sourcePos, final int sourceLevel, final int targetLevel) { - final long[] targetPos = new long[n]; + final long[] targetPos = new long[numDimensions]; absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel); return targetPos; } + /** + * The absolute position of the level 0 source position at + * the target level. + * + * @param sourcePos the source position j + * @param targetLevel the target level + * @return absolute position at the target level + */ public long[] absolutePosition( final long[] sourcePos, final int targetLevel) { @@ -277,7 +290,7 @@ public long[] relativePosition( final long[] sourcePos, final int sourceLevel, final int targetLevel) { - final long[] targetPos = new long[n]; + final long[] targetPos = new long[numDimensions]; relativePosition(sourcePos, sourceLevel, targetPos, targetLevel); return targetPos; } @@ -289,7 +302,7 @@ public long[] relativePosition( } public int[] relativeBlockSize(final int level) { - return r[level]; + return relativeToAdjacent[level]; } } From 8a491ef41a986d7c347dd2f7b019ee97cbe54221 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Oct 2025 22:29:51 +0200 Subject: [PATCH 393/423] typo --- src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 11224a30a..a1c193345 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -49,7 +49,7 @@ * {@code ReadData} may be lazy-loaded. For example, for {@code InputStream} and * {@code KeyValueAccess} sources, loading is deferred until the data is * accessed (e.g., {@link #allBytes()}, {@link #writeTo(OutputStream)}), or - * explicitly {@link #materialize() meterialized}. + * explicitly {@link #materialize() materialized}. *

      * {@code ReadData} can be {@link DataCodec#encode encoded} and {@link * DataCodec#decode decoded} by a {@link DataCodec}, which will also be lazy if From fc4a1ec97e2642a19a1cc977bb6502ef2998c659 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 22 Oct 2025 22:37:58 +0200 Subject: [PATCH 394/423] clean up Better to not have a default implementation for requireLength(). The details are different and tricky enough between ReadData implementations to encourage thinking about every case individually. --- .../janelia/saalfeldlab/n5/readdata/InputStreamReadData.java | 1 - .../java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java | 2 -- src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java | 1 - 3 files changed, 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java index d0073e444..8944c80a3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/InputStreamReadData.java @@ -50,7 +50,6 @@ public long length() { return (bytes != null) ? bytes.length() : length; } - // TODO: remove? Should this be the default implementation? @Override public long requireLength() throws N5IOException { long l = length(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java index 8c76dbbdf..7c08ce832 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/LazyReadData.java @@ -84,8 +84,6 @@ public long length() { return (bytes != null) ? bytes.length() : length; } - // TODO: remove? Should this be the default implementation? - // TODO: could just always return materialize().length()? @Override public long requireLength() throws N5IOException { long l = length(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index a1c193345..68b0d4579 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -79,7 +79,6 @@ default long length() { * if an I/O error occurs while trying to get the length */ long requireLength() throws N5IOException; - // TODO: default: {materialize(); return length();} /** * Returns a {@link ReadData} whose length is limited to the given value. From 8b9a85fe039c6dbd2b8d8fb977aa7a2c85b50f46 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 22 Oct 2025 17:05:28 -0400 Subject: [PATCH 395/423] test: more NestedGrid tests --- .../n5/shardstuff/NestedGridTest.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java index b3d286741..2910a50e5 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java @@ -1,5 +1,8 @@ package org.janelia.saalfeldlab.n5.shardstuff; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertThrows; + import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; import org.junit.Assert; import org.junit.Test; @@ -10,13 +13,23 @@ private static long absPosition1D(final NestedGrid grid, final int sourcePos, fi return grid.absolutePosition(new long[] {sourcePos}, targetLevel)[0]; } + @Test + public void testValidateInput() { + int[][] blockSizes = {{3}, {7}, {11}}; + assertThrows(IllegalArgumentException.class, () -> new NestedGrid(blockSizes)); + } + @Test public void testAbsolutePosition() { int[][] blockSizes = {{1}, {3}, {6}, {24}}; NestedGrid grid = new NestedGrid(blockSizes); Assert.assertEquals(38, absPosition1D(grid, 38, 0)); + + Assert.assertEquals(12, absPosition1D(grid, 36, 1)); + Assert.assertEquals(12, absPosition1D(grid, 37, 1)); Assert.assertEquals(12, absPosition1D(grid, 38, 1)); + Assert.assertEquals(6, absPosition1D(grid, 38, 2)); Assert.assertEquals(1, absPosition1D(grid, 38, 3)); } @@ -58,4 +71,18 @@ public void testRelativePositionChunkSize() { Assert.assertEquals(2, relPosition1D(grid, 38, 2)); Assert.assertEquals(1, relPosition1D(grid, 38, 3)); } + + @Test + public void testNd() { + + int[][] blockSizes = {{5, 7}, {5*3, 7*2}}; + NestedGrid grid = new NestedGrid(blockSizes); + System.out.println(grid); + assertArrayEquals(new long[]{1, 2}, grid.absolutePosition(new long[]{1, 2}, 0)); + assertArrayEquals(new long[]{99, 99}, grid.absolutePosition(new long[]{99, 99}, 0)); + + assertArrayEquals(new long[]{0, 0}, grid.absolutePosition(new long[]{0, 1}, 1)); + assertArrayEquals(new long[]{0, 1}, grid.absolutePosition(new long[]{0, 2}, 1)); + assertArrayEquals(new long[]{1, 1}, grid.absolutePosition(new long[]{3, 2}, 1)); + } } From d53b7a6de4276ab9caf03142d5e75733f7a01156 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 22 Oct 2025 17:56:30 -0400 Subject: [PATCH 396/423] test: additions to ShardTest --- .../saalfeldlab/n5/shard/ShardTest.java | 101 +++++++++++++++++- 1 file changed, 96 insertions(+), 5 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 76e948b61..5a85e8498 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -214,13 +214,23 @@ public void writeReadBlocksTest() { {dataset, "2", "2"} }; - // TODO check that these are the only keys - for (String[] key : keys) { final String shard = kva.compose(writer.getURI(), key); Assert.assertTrue("Shard at" + Arrays.toString(key) + "Does not exist", kva.exists(shard)); } + final String[][] someUnusedKeys = new String[][]{ + {dataset, "0", "1"}, + {dataset, "1", "1"}, + {dataset, "1", "2"}, + {dataset, "2", "1"} + }; + + for (String[] key : someUnusedKeys) { + final String shard = kva.compose(writer.getURI(), key); + Assert.assertFalse("Shard at" + Arrays.toString(key) + " exists but should not.", kva.exists(shard)); + } + final long[][] blockIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; for (long[] blockIndex : blockIndices) { final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); @@ -253,13 +263,22 @@ public void writeReadBlocksTest() { {dataset, "2", "2"} }; - // TODO check that other keys do not exist - for (String[] key : keys2) { final String shard = kva.compose(writer.getURI(), key); Assert.assertTrue("Shard at" + Arrays.toString(key) + "Does not exist", kva.exists(shard)); } + final String[][] someUnusedKeys2 = new String[][]{ + {dataset, "1", "1"}, + {dataset, "1", "2"}, + {dataset, "2", "1"} + }; + + for (String[] key : someUnusedKeys2) { + final String shard = kva.compose(writer.getURI(), key); + Assert.assertFalse("Shard at" + Arrays.toString(key) + " exists but should not.", kva.exists(shard)); + } + final long[][] oldBlockIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; for (long[] blockIndex : oldBlockIndices) { final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); @@ -277,6 +296,78 @@ public void writeReadBlocksTest() { Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])blockFromReadBlocks.getData()); } } + + @Test + public void writeShardDataSizeTest() { + + // note: this test depends on the use of raw compression + final N5Writer writer = tempN5Factory.createTempN5Writer(); + + int numBlocksPerShard = 16; + final int n5HeaderSizeBytes = 12; // 2 + 2 + 4*2 + final DatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + new int[]{2, 2} + ); + + final String dataset = "writeBlocksShardSize"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); + final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); + + final int[] blockSize = datasetAttributes.getBlockSize(); + final int numElements = blockSize[0] * blockSize[1]; + + final byte[] data = new byte[numElements]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)((100) + (10) + i); + } + + writer.writeBlocks( + dataset, + datasetAttributes, + /* shard (0, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + + /* shard (1, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), + + /* shard (2, 2) */ + new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) + ); + + final int indexSizeBytes = numBlocksPerShard * 16; // 8 bytes per long * + final int blockDataSizeBytes = numElements + n5HeaderSizeBytes; + + // shard 0,0 has 4 blocks so should be this size: + long shard00SizeBytes = indexSizeBytes + 4 * blockDataSizeBytes; + long shard10SizeBytes = indexSizeBytes + 2 * blockDataSizeBytes; + long shard22SizeBytes = indexSizeBytes + 1 * blockDataSizeBytes; + + final String[][] keys = new String[][]{ + {dataset, "0", "0"}, + {dataset, "1", "0"}, + {dataset, "2", "2"} + }; + + long[] shardSizes = new long[]{ + shard00SizeBytes, + shard10SizeBytes, + shard22SizeBytes + }; + + int i = 0; + for (String[] key : keys) { + final String shardPath = kva.compose(writer.getURI(), key); + Assert.assertEquals("shard at " + shardPath + " was the wrong size", shardSizes[i++], kva.size(shardPath)); + } + + } @Test public void readBlocksTest() { @@ -377,7 +468,7 @@ public void writeReadBlockTest() { // Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); // } // } -// + // @SuppressWarnings("unchecked") // @Test // public void testShardDelete() { From 685eabe960664881bd4a499d229f261c10364220 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 22 Oct 2025 22:03:31 -0400 Subject: [PATCH 397/423] feat: add NestedGrid.positionInSubGrid --- .../janelia/saalfeldlab/n5/shard/Nesting.java | 45 +++++++++++++++++-- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java index cd4a2f586..e694819b0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java @@ -1,6 +1,8 @@ package org.janelia.saalfeldlab.n5.shard; +import java.util.ArrayList; + // TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // // [ ] NestedGrid @@ -25,6 +27,9 @@ // TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ import java.util.Arrays; +import java.util.List; + +import org.janelia.saalfeldlab.n5.util.GridIterator; public class Nesting { @@ -36,9 +41,6 @@ public static void main(String[] args) { System.out.println("key = " + Arrays.toString(pos.key())); } - - - public static class NestedPosition implements Comparable { private final NestedGrid grid; @@ -304,7 +306,42 @@ public long[] relativePosition( public int[] relativeBlockSize(final int level) { return relativeToAdjacent[level]; } - } + public int[] absoluteBlockSize(final int level) { + return relativeToBase[level]; + } + + /** + * Given a block position at a particular level, returns a list of + * positions of all sub-blocks at a particular subLevel. + *

      + * Can be used to get a list of chunk positions for a shard with a + * particular position. + * + * @param position + * a block position + * @param level + * the blocks level + * @param subLevel + * the sub-level of positions to return + * @return the sub-block positions + */ + public List positionInSubGrid(long[] position, int level, int subLevel) { + + final long[] subPosition = new long[numDimensions()]; + absolutePosition(position, level, subPosition, subLevel); + + final int[] numElementsInSubGrid = absoluteBlockSize(numLevels() - 1); + final GridIterator git = new GridIterator(GridIterator.int2long(numElementsInSubGrid), subPosition); + + // TODO return NestedPositions instead? + final ArrayList positions = new ArrayList<>(); + while (git.hasNext()) + positions.add(git.next().clone()); + + return positions; + } + + } } From 7fe774905e05ec9d5432881298d67b5c128d5e34 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 24 Oct 2025 16:59:54 -0400 Subject: [PATCH 398/423] feat(test): change test size defaults so the dimensions are larger than the blocks; add tests for blocksize/dimensions edge cases --- .../saalfeldlab/n5/AbstractN5Test.java | 90 ++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 3e9daf2ab..01ea7f5a1 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -85,8 +85,8 @@ public abstract class AbstractN5Test { static protected final String groupName = "/test/group"; static protected final String[] subGroupNames = new String[]{"a", "b", "c"}; static protected final String datasetName = "/test/group/dataset"; - static protected final long[] dimensions = new long[]{10, 20, 30}; - static protected final int[] blockSize = new int[]{33, 22, 11}; + static protected final long[] dimensions = new long[]{6, 15, 35}; + static protected final int[] blockSize = new int[]{3, 5, 7}; static protected final int blockNumElements = blockSize[0] * blockSize[1] * blockSize[2]; static protected byte[] byteBlock; @@ -246,6 +246,92 @@ public void testCreateDataset() { assertEquals(DataType.UINT64, info.getDataType()); } + @Test + public void testBlocksLargerThanDimensions() { + + // Test case where block size is larger than dataset dimensions + final long[] smallDimensions = new long[]{2, 3, 4}; + final int[] largeBlockSize = new int[]{5, 7, 10}; + + try (final N5Writer n5 = createTempN5Writer()) { + n5.createDataset(datasetName, smallDimensions, largeBlockSize, DataType.UINT8, new RawCompression()); + final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + + // Create a block that is larger than the dataset dimensions + final int numElements = largeBlockSize[0] * largeBlockSize[1] * largeBlockSize[2]; + final byte[] data = new byte[numElements]; + for (int i = 0; i < numElements; i++) { + data[i] = (byte)(i % 256); + } + + final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(largeBlockSize, new long[]{0, 0, 0}, data); + n5.writeBlock(datasetName, attributes, dataBlock); + + // Read the block back + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + assertNotNull("Block should be readable", loadedDataBlock); + assertArrayEquals("Block size should match", largeBlockSize, loadedDataBlock.getSize()); + assertArrayEquals("Block data should match", data, (byte[])loadedDataBlock.getData()); + } + } + + @Test + public void testUnalignedBlocksTruncatedAtEnd() { + + // Test case where dimensions don't evenly divide by block size + final long[] unalignedDimensions = new long[]{5, 14, 33}; + final int[] testBlockSize = new int[]{3, 5, 7}; + + try (final N5Writer n5 = createTempN5Writer()) { + n5.createDataset(datasetName, unalignedDimensions, testBlockSize, DataType.INT32, new RawCompression()); + final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + + // Test writing to the last block in dimension 0 (should be truncated to size 2 instead of 3) + final int[] truncatedBlockSize0 = new int[]{2, 5, 7}; // [3-4] in dim 0 + final int numElements0 = truncatedBlockSize0[0] * truncatedBlockSize0[1] * truncatedBlockSize0[2]; + final int[] data0 = new int[numElements0]; + for (int i = 0; i < numElements0; i++) { + data0[i] = i + 1000; + } + final IntArrayDataBlock dataBlock0 = new IntArrayDataBlock(truncatedBlockSize0, new long[]{1, 0, 0}, data0); + n5.writeBlock(datasetName, attributes, dataBlock0); + + final DataBlock loadedBlock0 = n5.readBlock(datasetName, attributes, 1, 0, 0); + assertNotNull("Truncated block should be readable", loadedBlock0); + assertArrayEquals("Truncated block data should match", data0, (int[])loadedBlock0.getData()); + + // Test writing to the last block in dimension 1 (should be truncated to size 4 instead of 5) + final int[] truncatedBlockSize1 = new int[]{3, 4, 7}; // [10-13] in dim 1 + final int numElements1 = truncatedBlockSize1[0] * truncatedBlockSize1[1] * truncatedBlockSize1[2]; + final int[] data1 = new int[numElements1]; + for (int i = 0; i < numElements1; i++) { + data1[i] = i + 2000; + } + final IntArrayDataBlock dataBlock1 = new IntArrayDataBlock(truncatedBlockSize1, new long[]{0, 2, 0}, data1); + n5.writeBlock(datasetName, attributes, dataBlock1); + + final DataBlock loadedBlock1 = n5.readBlock(datasetName, attributes, 0, 2, 0); + assertNotNull("Truncated block should be readable", loadedBlock1); + assertArrayEquals("Truncated block data should match", data1, (int[])loadedBlock1.getData()); + + // Test writing to the last block in dimension 2 (should be truncated to size 5 instead of 7) + final int[] truncatedBlockSize2 = new int[]{3, 5, 5}; // [28-32] in dim 2 + final int numElements2 = truncatedBlockSize2[0] * truncatedBlockSize2[1] * truncatedBlockSize2[2]; + final int[] data2 = new int[numElements2]; + for (int i = 0; i < numElements2; i++) { + data2[i] = i + 3000; + } + final IntArrayDataBlock dataBlock2 = new IntArrayDataBlock(truncatedBlockSize2, new long[]{0, 0, 4}, data2); + n5.writeBlock(datasetName, attributes, dataBlock2); + + final DataBlock loadedBlock2 = n5.readBlock(datasetName, attributes, 0, 0, 4); + assertNotNull("Truncated block should be readable", loadedBlock2); + assertArrayEquals("Truncated block data should match", data2, (int[])loadedBlock2.getData()); + } + } + + + @Test public void testWriteReadByteBlock() { From a4f8b4c7fee2f2cdefbfd63e45cde2697c0dfb9d Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 28 Oct 2025 10:34:25 -0400 Subject: [PATCH 399/423] test: generalize ShardTest for n5-zarr re-use --- .../saalfeldlab/n5/shard/ShardTest.java | 90 +++++++++---------- 1 file changed, 43 insertions(+), 47 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 5a85e8498..32befe3c1 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -154,18 +154,11 @@ private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, blockCodec); } - private DatasetAttributes getTestAttributes() { + protected DatasetAttributes getTestAttributes() { return getTestAttributes(new long[]{8, 8}, new int[]{4, 4}, new int[]{2, 2}); } - private DatasetAttributes getTestAttributes3d() { - - final int[] blockSize = {33, 22, 11}; - final int[] shardSize = {blockSize[0] * 2, blockSize[1] * 2, blockSize[2] * 2}; - return getTestAttributes(new long[]{10, 20, 30}, shardSize, blockSize); - } - @Test public void writeReadBlocksTest() { @@ -208,28 +201,21 @@ public void writeReadBlocksTest() { final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); - final String[][] keys = new String[][]{ - {dataset, "0", "0"}, - {dataset, "1", "0"}, - {dataset, "2", "2"} + long[][] keys = new long[][]{ + {0, 0}, + {1, 0}, + {2, 2} }; - for (String[] key : keys) { - final String shard = kva.compose(writer.getURI(), key); - Assert.assertTrue("Shard at" + Arrays.toString(key) + "Does not exist", kva.exists(shard)); - } - - final String[][] someUnusedKeys = new String[][]{ - {dataset, "0", "1"}, - {dataset, "1", "1"}, - {dataset, "1", "2"}, - {dataset, "2", "1"} + final long[][] someUnusedKeys = new long[][]{ + {0, 1}, + {1, 1}, + {1, 2}, + {2, 1} }; - for (String[] key : someUnusedKeys) { - final String shard = kva.compose(writer.getURI(), key); - Assert.assertFalse("Shard at" + Arrays.toString(key) + " exists but should not.", kva.exists(shard)); - } + ensureKeysExist(kva, writer.getURI(), dataset, datasetAttributes, keys); + ensureKeysDoNotExist(kva, writer.getURI(), dataset, datasetAttributes, someUnusedKeys); final long[][] blockIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; for (long[] blockIndex : blockIndices) { @@ -241,6 +227,7 @@ public void writeReadBlocksTest() { for (int i = 0; i < data2.length; i++) { data2[i] = (byte)(10 + i); } + writer.writeBlocks( dataset, datasetAttributes, @@ -253,31 +240,23 @@ public void writeReadBlocksTest() { new ByteArrayDataBlock(blockSize, new long[]{0, 5}, data2), /* shard (2, 2) */ - new ByteArrayDataBlock(blockSize, new long[]{10, 10}, data2) - ); + new ByteArrayDataBlock(blockSize, new long[]{10, 10}, data2)); - final String[][] keys2 = new String[][]{ - {dataset, "0", "0"}, - {dataset, "1", "0"}, - {dataset, "0", "1"}, - {dataset, "2", "2"} + long[][] keys2 = new long[][]{ + {0, 0}, + {1, 0}, + {0, 1}, + {2, 2} }; - for (String[] key : keys2) { - final String shard = kva.compose(writer.getURI(), key); - Assert.assertTrue("Shard at" + Arrays.toString(key) + "Does not exist", kva.exists(shard)); - } - - final String[][] someUnusedKeys2 = new String[][]{ - {dataset, "1", "1"}, - {dataset, "1", "2"}, - {dataset, "2", "1"} + long[][] someUnusedKeys2 = new long[][]{ + {1, 1}, + {1, 2}, + {2, 1} }; - for (String[] key : someUnusedKeys2) { - final String shard = kva.compose(writer.getURI(), key); - Assert.assertFalse("Shard at" + Arrays.toString(key) + " exists but should not.", kva.exists(shard)); - } + ensureKeysExist(kva, writer.getURI(), dataset, datasetAttributes, keys2); + ensureKeysDoNotExist(kva, writer.getURI(), dataset, datasetAttributes, someUnusedKeys2); final long[][] oldBlockIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; for (long[] blockIndex : oldBlockIndices) { @@ -296,6 +275,24 @@ public void writeReadBlocksTest() { Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])blockFromReadBlocks.getData()); } } + + private void ensureKeysExist(KeyValueAccess kva, URI uri, String dataset, + DatasetAttributes datasetAttributes, long[][] keys) { + + for (long[] key : keys) { + final String shard = kva.compose(uri, dataset, datasetAttributes.relativeBlockPath(key)); + Assert.assertTrue("Shard at" + shard + "Does not exist", kva.exists(shard)); + } + } + + private void ensureKeysDoNotExist(KeyValueAccess kva, URI uri, String dataset, + DatasetAttributes datasetAttributes, long[][] keys) { + + for (long[] key : keys) { + final String shard = kva.compose(uri, dataset, datasetAttributes.relativeBlockPath(key)); + Assert.assertFalse("Shard at" + shard + " exists but should not.", kva.exists(shard)); + } + } @Test public void writeShardDataSizeTest() { @@ -382,7 +379,6 @@ public void readBlocksTest() { final String dataset = "writeReadBlocks"; final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; final List> readBlocks = n5.readBlocks(dataset, datasetAttributes, Arrays.asList(newBlockIndices)); - System.out.println(readBlocks.size()); } @Test From e9ece6e21d2fdd47ecd9461100b9ea04d72ba0a5 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 28 Oct 2025 10:34:47 -0400 Subject: [PATCH 400/423] doc: improve Nesting.positionInSubGrid doc --- src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java index e694819b0..892900ae4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java @@ -319,11 +319,11 @@ public int[] absoluteBlockSize(final int level) { * particular position. * * @param position - * a block position + * a position * @param level - * the blocks level + * the nesting level of the given position * @param subLevel - * the sub-level of positions to return + * the nesting sub-level of positions to return * @return the sub-block positions */ public List positionInSubGrid(long[] position, int level, int subLevel) { From ddcc821f153bfa997d212ef148bc3d53df999b85 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 29 Oct 2025 13:05:37 -0400 Subject: [PATCH 401/423] test: add extensible method that defines illegal characters --- .../java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 01ea7f5a1..1ab428600 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1699,13 +1699,17 @@ public void testWriterSeparation() { } } + protected String[] illegalChars() { + return new String[]{" ", "#", "%"}; + } + @Test public void testPathsWithIllegalUriCharacters() throws IOException, URISyntaxException { try (N5Writer writer = createTempN5Writer()) { try (N5Reader reader = createN5Reader(writer.getURI().toString())) { - final String[] illegalChars = {" ", "#", "%"}; + final String[] illegalChars = illegalChars(); for (final String illegalChar : illegalChars) { final String groupWithIllegalChar = "test" + illegalChar + "group"; assertThrows("list over group should throw prior to create", N5Exception.N5IOException.class, () -> writer.list(groupWithIllegalChar)); From 380964c6d4cae6bd991c80a6c8c51c92112062ca Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 16 Aug 2024 14:00:54 -0400 Subject: [PATCH 402/423] test: start of partial read benchmarks Signed-off-by: Caleb Hulbert --- pom.xml | 2 +- src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java | 2 +- src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java | 2 +- src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java | 2 +- src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 803d0dab5..2636e9052 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.scijava pom-scijava - 40.0.0 + 43.0.0 diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 8e61c543f..5fa871088 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -38,7 +38,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("bzip2") -@NameConfig.Name("bzip2-compression") +@NameConfig.Name("bzip2") public class Bzip2Compression implements Compression { private static final long serialVersionUID = -4873117458390529118L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index a870c954b..9c81fef76 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -42,7 +42,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("gzip") -@NameConfig.Name("gzip-compression") +@NameConfig.Name("gzip") public class GzipCompression implements Compression { private static final long serialVersionUID = 8630847239813334263L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index eb1d52f1a..ea01879df 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -35,7 +35,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("lz4") -@NameConfig.Name("lz4-compression") +@NameConfig.Name("lz4") public class Lz4Compression implements Compression { private static final long serialVersionUID = -9071316415067427256L; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index 2e48a1e32..6d3ebc86e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -34,7 +34,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("raw") -@NameConfig.Name("raw-compression") +@NameConfig.Name("raw") public class RawCompression implements Compression, DeterministicSizeDataCodec { private static final long serialVersionUID = 7526445806847086477L; From 1de7b706598640f06a39f330e6d9a6667075c2d8 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 29 Oct 2025 14:00:01 -0400 Subject: [PATCH 403/423] fix(test): don't specify relative paths for createTempN5Writer; in this case, default is fine --- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 1ab428600..6c8cb0a8d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -361,7 +361,7 @@ public void testWriteReadStringBlock() { final String[] stringBlock = new String[]{"", "a", "bc", "de", "fgh", ":-þ"}; for (final Compression compression : getCompressions()) { - try (final N5Writer n5 = createTempN5Writer("test.n5")) { + try (final N5Writer n5 = createTempN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final StringDataBlock dataBlock = new StringDataBlock(blockSize, new long[]{0L, 0L, 0L}, stringBlock); @@ -578,7 +578,7 @@ public void testWriteInvalidBlock() { public void testOverwriteBlock() { final Compression compression = getCompressions()[0]; - try (final N5Writer n5 = createTempN5Writer("test-overwrite")) { + try (final N5Writer n5 = createTempN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, DataType.INT32, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); From 162309cbcf6c544f5659262e37ffda42cce2a5da Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 29 Oct 2025 15:12:31 -0400 Subject: [PATCH 404/423] feat: add getNestedBlockGrid method to DatasetAttributes * getDatasetAccess no longer public --- .../janelia/saalfeldlab/n5/DatasetAttributes.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 619d5790c..0059aafa2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -406,10 +406,22 @@ public DataType getDataType() { * * @return the {@code BlockCodecInfo} for this dataset */ - public DatasetAccess getDatasetAccess() { + DatasetAccess getDatasetAccess() { return (DatasetAccess)access; } + + /** + * Returns the {@code NestedGrid} for this dataset, from which block and + * shard sizes are accessible. + * + * @return the NestedGrid + */ + public NestedGrid getNestedBlockGrid() { + + return getDatasetAccess().getGrid(); + } + public BlockCodecInfo getBlockCodecInfo() { From 2e5d86b83f65e33a1f7b52542682c1e1fb6f043b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 29 Oct 2025 15:14:29 -0400 Subject: [PATCH 405/423] test: update tests to work with less accessible DatasetAccess --- .../saalfeldlab/n5/codec/BlockCodecTests.java | 12 +++++++---- .../n5/shardstuff/RawShardTest.java | 20 +++++++++++++++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java index f83d2b0e3..cc005b5b7 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java @@ -2,7 +2,6 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.nio.ByteOrder; @@ -18,10 +17,12 @@ import org.janelia.saalfeldlab.n5.GzipCompression; import org.janelia.saalfeldlab.n5.IntArrayDataBlock; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; import org.janelia.saalfeldlab.n5.shard.DatasetAccess; import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; +import org.janelia.saalfeldlab.n5.shardstuff.RawShardTest; import org.janelia.saalfeldlab.n5.shardstuff.TestPositionValueAccess; import org.junit.Test; @@ -117,14 +118,15 @@ public void testEmptyBlock() throws Exception { final int[] blockSize = {0, 0}; final long[] gridPosition = {0, 0}; final N5BlockCodecInfo blockCodecInfo = new N5BlockCodecInfo(); - final DatasetAttributes attributes = new DatasetAttributes( + final RawShardTest.TestDatasetAttributes attributes = new RawShardTest.TestDatasetAttributes( new long[]{64, 64}, new int[]{8, 8}, DataType.UINT8, - blockCodecInfo); + blockCodecInfo, + new RawCompression()); final PositionValueAccess store = new TestPositionValueAccess(); - DatasetAccess access = attributes.getDatasetAccess(); + DatasetAccess access = attributes.datasetAccess(); // Test encode/decode final ByteArrayDataBlock emptyBlock = new ByteArrayDataBlock(blockSize, gridPosition, new byte[0]); @@ -284,4 +286,6 @@ private static void assertDataEquals(DataBlock expected, DataBlock actual) throw new IllegalArgumentException("Unknown data type"); } } + + } \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java index 5cfebf5d3..e3a0b05a8 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java @@ -46,14 +46,14 @@ public static void main(String[] args) { IndexLocation.START ); - DatasetAttributes attributes = new DatasetAttributes( + TestDatasetAttributes attributes = new TestDatasetAttributes( new long[] {}, level2ShardSize, DataType.INT8, c2, new RawCompression()); - final DatasetAccess datasetAccess = attributes.getDatasetAccess(); + final DatasetAccess datasetAccess = attributes.datasetAccess(); // TODO: N5Reader/Writer needs to provide a PositionValueAccess implementation on top of its KVA. // The read/write/deleteBlock methods would getDataAccess() from the DatasetAttributes and call it with that PositionValueAccess. @@ -126,5 +126,21 @@ private static DataBlock createDataBlock(int[] size, long[] gridPosition Arrays.fill(bytes, (byte) fillValue); return new ByteArrayDataBlock(size, gridPosition, bytes); } + + public static class TestDatasetAttributes extends DatasetAttributes { + + public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType dataType, BlockCodecInfo blockCodecInfo, + DataCodecInfo... dataCodecInfos) { + + super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); + } + + public DatasetAccess datasetAccess() { + + // to make this accessible for the test + return createDatasetAccess(); + } + + } } From 1ea9da21cf5f731e7856254c75ff24f29fda1b4b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 31 Oct 2025 16:16:59 -0400 Subject: [PATCH 406/423] doc: correct DatasetAttributes.getDatasetAccess --- .../java/org/janelia/saalfeldlab/n5/DatasetAttributes.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 0059aafa2..b3e1890c1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -402,15 +402,15 @@ public DataType getDataType() { } /** - * Get the {@link BlockCodecInfo} for this dataset. + * Get the {@link DatasetAccess} for this dataset. * - * @return the {@code BlockCodecInfo} for this dataset + * @return the {@code DatasetAccess} for this dataset */ DatasetAccess getDatasetAccess() { return (DatasetAccess)access; } - + /** * Returns the {@code NestedGrid} for this dataset, from which block and * shard sizes are accessible. From 5f23b09d592db214a4c4af2f34bd5135819e7ecc Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 3 Nov 2025 10:02:32 -0500 Subject: [PATCH 407/423] test: clean up ShardTest --- .../saalfeldlab/n5/shard/ShardTest.java | 401 +----------------- 1 file changed, 3 insertions(+), 398 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 32befe3c1..76b935d3c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -1,8 +1,6 @@ package org.janelia.saalfeldlab.n5.shard; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; -import org.janelia.saalfeldlab.n5.Compression; -import org.janelia.saalfeldlab.n5.CompressionAdapter; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; @@ -13,37 +11,22 @@ import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.NameConfigAdapter; import org.janelia.saalfeldlab.n5.RawCompression; -import org.janelia.saalfeldlab.n5.DatasetAttributes.DatasetAttributesAdapter; import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; -import org.janelia.saalfeldlab.n5.shard.DefaultShardCodecInfo; import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; -import org.janelia.saalfeldlab.n5.util.GridIterator; import org.junit.After; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonSerializationContext; -import com.google.gson.JsonSerializer; import java.io.File; -import java.lang.reflect.Type; import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteOrder; @@ -54,21 +37,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Random; -import java.util.stream.Stream; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - - -//@RunWith(Parameterized.class) +@RunWith(Parameterized.class) public class ShardTest { - - // TODO private static final boolean LOCAL_DEBUG = false; @@ -373,8 +344,7 @@ public void readBlocksTest() { final DatasetAttributes datasetAttributes = getTestAttributes( new long[]{24, 24}, new int[]{8, 8}, - new int[]{2, 2} - ); + new int[]{2, 2}); final String dataset = "writeReadBlocks"; final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; @@ -384,13 +354,12 @@ public void readBlocksTest() { @Test public void writeReadBlockTest() { - final N5Writer writer = tempN5Factory.createTempN5Writer(); + final GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)tempN5Factory.createTempN5Writer(); final DatasetAttributes datasetAttributes = getTestAttributes(); final String dataset = "writeReadBlock"; writer.remove(dataset); writer.createDataset(dataset, datasetAttributes); - writer.deleteBlock(dataset, 0, 0); //FIXME Caleb: We are abusing this here. It shouldn't delete the entire shard.. final int[] blockSize = datasetAttributes.getBlockSize(); final DataType dataType = datasetAttributes.getDataType(); @@ -423,265 +392,6 @@ public void writeReadBlockTest() { } } -// @Test -// public void writeReadShardTest() { -// -// final N5Writer writer = tempN5Factory.createTempN5Writer(); -// -// final DatasetAttributes datasetAttributes = getTestAttributes(); -// -// final String dataset = "writeReadShard"; -// writer.createDataset(dataset, datasetAttributes); -// writer.deleteBlock(dataset, 0, 0); -// -// final int[] blockSize = datasetAttributes.getBlockSize(); -// final DataType dataType = datasetAttributes.getDataType(); -// final int numElements = 2 * 2; -// -// final HashMap writtenBlocks = new HashMap<>(); -// -// final InMemoryShard shard = new InMemoryShard<>(datasetAttributes, new long[]{0, 0}); -// -// for (int idx1 = 1; idx1 >= 0; idx1--) { -// for (int idx2 = 1; idx2 >= 0; idx2--) { -// final long[] gridPosition = {idx1, idx2}; -// final DataBlock dataBlock = dataType.createDataBlock(blockSize, gridPosition, numElements); -// byte[] data = (byte[])dataBlock.getData(); -// for (int i = 0; i < data.length; i++) { -// data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); -// } -// shard.addBlock((DataBlock)dataBlock); -// writtenBlocks.put(gridPosition, data); -// } -// } -// -// writer.writeShard(dataset, datasetAttributes, shard); -// -// for (Map.Entry entry : writtenBlocks.entrySet()) { -// final long[] otherGridPosition = entry.getKey(); -// final byte[] otherData = entry.getValue(); -// final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); -// Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); -// } -// } - -// @SuppressWarnings("unchecked") -// @Test -// public void testShardDelete() { -// -// final Random rnd = new Random(88); -// try (GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)tempN5Factory.createTempN5Writer()) { -// final String datasetName = "testShardDelete"; -// -// // Create a sharded dataset -// final DatasetAttributes datasetAttributes = getTestAttributes3d(); -// int[] blockSize = datasetAttributes.getBlockSize(); -// writer.createDataset(datasetName, datasetAttributes); -// -// final int blockNumElements = Arrays.stream(blockSize).reduce(1, (x,y) -> x*y); -// final byte[] byteBlock = new byte[blockNumElements]; -// rnd.nextBytes(byteBlock); -// -// // Create blocks in different shards -// final long[] shardPosition1 = {0, 0, 0}; -// final long[] shardPosition2 = {0, 0, 1}; // Different shard in z dimension -// final long[] shardPositionNonExistent = {0, 0, 2}; // Different shard in z dimension -// -// // Create blocks within the first shard -// final long[] blockPosition1 = {0, 0, 0}; -// final long[] blockPosition2 = {1, 0, 0}; -// final long[] blockPosition3 = {0, 1, 0}; -// -// // Create a block in the second shard -// final long[] blockPosition4 = {0, 0, 2}; // This will be in -// // shardPosition2 -// -// final ByteArrayDataBlock block1 = new ByteArrayDataBlock(blockSize, blockPosition1, byteBlock); -// final ByteArrayDataBlock block2 = new ByteArrayDataBlock(blockSize, blockPosition2, byteBlock); -// final ByteArrayDataBlock block3 = new ByteArrayDataBlock(blockSize, blockPosition3, byteBlock); -// final ByteArrayDataBlock block4 = new ByteArrayDataBlock(blockSize, blockPosition4, byteBlock); -// -// // Write blocks to create shards -// writer.writeBlocks(datasetName, datasetAttributes, block1, block2, block3, block4); -// -// // Verify shards exist -// final Shard shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); -// final Shard shard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); -// final Shard shardDNE = writer.readShard(datasetName, datasetAttributes, shardPositionNonExistent); -// assertNotNull("Shard 1 should exist", shard1); -// assertNotNull("Shard 2 should exist", shard2); -// assertNull("Shard 3 should not exist", shardDNE); -// assertEquals("Shard 1 should contain 3 blocks", 3, shard1.getBlocks().size()); -// assertEquals("Shard 2 should contain 1 block", 1, shard2.getBlocks().size()); -// -// // Test deleteShard -// boolean deleted1 = writer.deleteShard(datasetName, shardPosition1); -// assertTrue("deleteShard should return true when shard exists", deleted1); -// -// // Verify shard is deleted -// final Shard deletedShard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); -// assertNull("Shard 1 should be deleted", deletedShard1); -// -// // Verify blocks in the deleted shard are gone -// assertNull("Block 1 should be deleted", writer.readBlock(datasetName, datasetAttributes, blockPosition1)); -// assertNull("Block 2 should be deleted", writer.readBlock(datasetName, datasetAttributes, blockPosition2)); -// assertNull("Block 3 should be deleted", writer.readBlock(datasetName, datasetAttributes, blockPosition3)); -// -// // Verify other shard is unaffected -// final Shard stillExistingShard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); -// assertNotNull("Shard 2 should still exist", stillExistingShard2); -// assertNotNull("Block 4 should still exist", writer.readBlock(datasetName, datasetAttributes, blockPosition4)); -// -// // Test deleting non-existent shard -// boolean deletedAgain = writer.deleteShard(datasetName, shardPosition1); -// assertFalse("deleteShard should return false when shard doesn't exist", deletedAgain); -// -// // Test deleting shard at invalid position -// final long[] invalidShardPosition = {100, 100, 100}; -// boolean deletedInvalid = writer.deleteShard(datasetName, invalidShardPosition); -// assertFalse("deleteShard should return false for non-existent shard position", deletedInvalid); -// } -// } -// -// @SuppressWarnings("unchecked") -// @Test -// public void testShardedBlockDelete() { -// -// final Random rnd = new Random(88); -// try (GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)tempN5Factory.createTempN5Writer()) { -// final String datasetName = "testShardBlockDelete"; -// -// // Create a sharded dataset -// final DatasetAttributes datasetAttributes = getTestAttributes3d(); -// int[] blockSize = datasetAttributes.getBlockSize(); -// writer.createDataset(datasetName, datasetAttributes); -// -// final int blockNumElements = Arrays.stream(blockSize).reduce(1, (x,y) -> x*y); -// final byte[] byteBlock = new byte[blockNumElements]; -// rnd.nextBytes(byteBlock); -// -// // Create blocks in different shards -// final long[] shardPosition1 = {0, 0, 0}; -// final long[] shardPosition2 = {0, 0, 1}; // Different shard in z dimension -// final long[] shardPositionNonExistent = {0, 0, 2}; // Different shard in z dimension -// -// // Create blocks within the first shard -// final long[] blockPosition1 = {0, 0, 0}; -// final long[] blockPosition2 = {1, 0, 0}; -// final long[] blockPosition3 = {0, 1, 0}; -// -// // Create a block in the second shard -// final long[] blockPosition4 = {0, 0, 2}; // This will be in shardPosition2 -// -// final ByteArrayDataBlock block1 = new ByteArrayDataBlock(blockSize, blockPosition1, byteBlock); -// final ByteArrayDataBlock block2 = new ByteArrayDataBlock(blockSize, blockPosition2, byteBlock); -// final ByteArrayDataBlock block3 = new ByteArrayDataBlock(blockSize, blockPosition3, byteBlock); -// final ByteArrayDataBlock block4 = new ByteArrayDataBlock(blockSize, blockPosition4, byteBlock); -// -// // Write blocks to create shards -// writer.writeBlocks(datasetName, datasetAttributes, block1, block2, block3, block4); -// -// // Verify shards exist -// Shard shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); -// Shard shard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); -// Shard shardDNE = writer.readShard(datasetName, datasetAttributes, shardPositionNonExistent); -// assertNotNull("Shard 1 should exist", shard1); -// assertNotNull("Shard 2 should exist", shard2); -// assertNull("Shard 3 should not exist", shardDNE); -// assertEquals("Shard 1 should contain 3 blocks", 3, shard1.getBlocks().size()); -// assertEquals("Shard 2 should contain 1 block", 1, shard2.getBlocks().size()); -// -// // Test delete one block from a multi-block shard -// boolean deleted1 = writer.deleteBlock(datasetName, blockPosition1); -// assertTrue("deleteBlock1", deleted1); -// shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); -// assertNotNull("Shard 1 should still exist", shard1); -// DataBlock blk1Read = writer.readBlock(datasetName, datasetAttributes, blockPosition1); -// DataBlock blk2Read = writer.readBlock(datasetName, datasetAttributes, blockPosition2); -// DataBlock blk3Read = writer.readBlock(datasetName, datasetAttributes, blockPosition3); -// assertNull("Block 1 should not exist", blk1Read); -// assertNotNull("Block 2 should exist", blk2Read); -// assertNotNull("Block 3 should exist", blk3Read); -// -// // Test delete one block from a multi-block shard -// boolean deleted2 = writer.deleteBlock(datasetName, blockPosition2); -// assertTrue("deleteBlock2", deleted2); -// shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); -// assertNotNull("Shard 1 should still exist", shard1); -// blk2Read = writer.readBlock(datasetName, datasetAttributes, blockPosition2); -// blk3Read = writer.readBlock(datasetName, datasetAttributes, blockPosition3); -// assertNull("Block 2 should not exist", blk2Read); -// assertNotNull("Block 3 should exist", blk3Read); -// -// // Test delete last block from a multi-block shard -// boolean deleted3 = writer.deleteBlock(datasetName, blockPosition3); -// assertTrue("deleteBlock3", deleted3); -// shard1 = writer.readShard(datasetName, datasetAttributes, shardPosition1); -// assertNull("Shard 1 should not exist", shard1); -// blk3Read = writer.readBlock(datasetName, datasetAttributes, blockPosition3); -// assertNull("Block 3 should not exist", blk3Read); -// -// // Test delete last block from a multi-block shard -// boolean deleted4 = writer.deleteBlock(datasetName, blockPosition4); -// assertTrue("deleteBlock4", deleted4); -// shard2 = writer.readShard(datasetName, datasetAttributes, shardPosition2); -// assertNull("Shard 2 should not exist", shard2); -// DataBlock blk4Read = writer.readBlock(datasetName, datasetAttributes, blockPosition4); -// assertNull("Block 2 should not exist", blk4Read); -// } -// } -// -// @Test -// @Ignore("Nested sharding not supported ") -// public void writeReadNestedShards() { -// -// int[] blockSize = new int[]{4, 4}; -// int N = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y); -// -// final N5Writer writer = tempN5Factory.createTempN5Writer(); -// final DatasetAttributes datasetAttributes = getNestedShardCodecsAttributes(blockSize); -// writer.createDataset("nestedShards", datasetAttributes); -// -// final byte[] data = new byte[N]; -// Arrays.fill(data, (byte)4); -// -// writer.writeBlocks("nestedShards", datasetAttributes, -// new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), -// new ByteArrayDataBlock(blockSize, new long[]{0, 2}, data), -// new ByteArrayDataBlock(blockSize, new long[]{2, 1}, data) -// ); -// -// assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 1, 1).getData()); -// assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 0, 2).getData()); -// assertArrayEquals(data, (byte[])writer.readBlock("nestedShards", datasetAttributes, 2, 1).getData()); -// } -// -// private DatasetAttributes getNestedShardCodecsAttributes(int[] blockSize) { -// -// final int[] innerShardSize = new int[]{2 * blockSize[0], 2 * blockSize[1]}; -// final int[] shardSize = new int[]{4 * blockSize[0], 4 * blockSize[1]}; -// final long[] dimensions = GridIterator.int2long(shardSize); -// -// // TODO: its not even clear how we build this given -// // this constructor. Is the block size of the sharded dataset attributes -// // the innermost (block) size or the intermediate shard size? -// // probably better to forget about this class - only use DatasetAttributes -// // and detect shading in another way -// final ShardingCodec innerShard = new ShardingCodec(innerShardSize, -// new CodecInfo[]{new N5BlockCodecInfo()}, -// new DeterministicSizeCodecInfo[]{new RawBlockCodecInfo(indexByteOrder), new Crc32cChecksumCodec()}, -// IndexLocation.START); -// -// return new DatasetAttributes( -// dimensions, shardSize, blockSize, DataType.UINT8, -// new ShardingCodec( -// blockSize, -// new CodecInfo[]{innerShard}, -// new DeterministicSizeCodecInfo[]{new RawBlockCodecInfo(indexByteOrder), new Crc32cChecksumCodec()}, -// IndexLocation.END) -// ); -// } - /** * An N5Writer that serializing the sharding codecs, enabling testing of * shard functionality, despite the fact that the N5 format does not support @@ -691,8 +401,6 @@ public static class ShardedN5Writer extends N5FSWriter { Gson gson; -// TestDatasetAttributesAdapter adapter = new TestDatasetAttributesAdapter(); - public ShardedN5Writer(String basePath) { this(basePath, new GsonBuilder()); @@ -703,7 +411,6 @@ public ShardedN5Writer(String basePath, GsonBuilder gsonBuilder) { super(basePath); gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); -// gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, new TestDatasetAttributesAdapter()); gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBlockCodecInfo.byteOrderAdapter); gsonBuilder.disableHtmlEscaping(); gson = gsonBuilder.create(); @@ -715,108 +422,6 @@ public Gson getGson() { // the super constructor needs the gson instance, unfortunately return gson == null ? super.gson : gson; } - -// @Override -// public DatasetAttributes createDatasetAttributes(final JsonElement attributes) { -// -// final JsonDeserializationContext context = new JsonDeserializationContext() { -// -// @Override -// public T deserialize(JsonElement json, Type typeOfT) throws JsonParseException { -// -// return getGson().fromJson(json, typeOfT); -// } -// }; -// -// return adapter.deserialize(attributes, DatasetAttributes.class, context); -// } } -// public static class TestDatasetAttributesAdapter extends DatasetAttributesAdapter { -// -// @Override -// public DatasetAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { -// -// if (json == null || !json.isJsonObject()) -// return null; -// final JsonObject obj = json.getAsJsonObject(); -// final boolean validKeySet = obj.has(DatasetAttributes.DIMENSIONS_KEY) -// && obj.has(DatasetAttributes.BLOCK_SIZE_KEY) -// && obj.has(DatasetAttributes.DATA_TYPE_KEY) -// && (obj.has(DatasetAttributes.CODEC_KEY) || obj.has(DatasetAttributes.COMPRESSION_KEY)); -// -// if (!validKeySet) -// return null; -// -// final long[] dimensions = context.deserialize(obj.get(DatasetAttributes.DIMENSIONS_KEY), long[].class); -// final int[] blockSize = context.deserialize(obj.get(DatasetAttributes.BLOCK_SIZE_KEY), int[].class); -// final DataType dataType = context.deserialize(obj.get(DatasetAttributes.DATA_TYPE_KEY), DataType.class); -// -// final CodecInfo[] codecs; -// if (obj.has(DatasetAttributes.CODEC_KEY)) { DefaultShardCodecInfo shardCodecInfo = new DefaultShardCodecInfo( -// blockSize, -// new N5BlockCodecInfo(), -// new DataCodecInfo[]{new RawCompression()}, -// new RawBlockCodecInfo(), -// new DataCodecInfo[]{new RawCompression()}, -// IndexLocation.END); -// codecs = context.deserialize(obj.get(DatasetAttributes.CODEC_KEY), CodecInfo[].class); -// } else if (obj.has(DatasetAttributes.COMPRESSION_KEY)) { -// final Compression compression -// = CompressionAdapter.getJsonAdapter().deserialize(obj.get(DatasetAttributes.COMPRESSION_KEY), Compression.class, context); -// codecs = new CodecInfo[]{compression}; -// } else { -// return null; -// } -// final BlockCodecInfo blockCodecInfo; -// final DataCodecInfo[] dataCodecInfos; -// if (codecs[0] instanceof BlockCodecInfo) { -// blockCodecInfo = (BlockCodecInfo)codecs[0]; -// dataCodecInfos = new DataCodecInfo[codecs.length - 1]; -// System.arraycopy(codecs, 1, dataCodecInfos, 0, dataCodecInfos.length); -// } else { -// blockCodecInfo = null; -// dataCodecInfos = (DataCodecInfo[])codecs; -// } -// return new DatasetAttributes(dimensions, blockSize, dataType, blockCodecInfo, dataCodecInfos); -// } -// -// @Override -// public JsonElement serialize(DatasetAttributes src, Type typeOfSrc, JsonSerializationContext context) { -// -// final JsonObject obj = new JsonObject(); -// obj.add(DatasetAttributes.DIMENSIONS_KEY, context.serialize(src.getDimensions())); -// obj.add(DatasetAttributes.BLOCK_SIZE_KEY, context.serialize(src.getBlockSize())); -// -// obj.add(DatasetAttributes.DATA_TYPE_K// public static class TestDatasetAttributesAdapter extends DatasetAttributesAdapter {EY, context.serialize(src.getDataType())); -// -// -// final DataCodecInfo[] byteCodecs = src.getDataCodecInfos(); -// final BlockCodecInfo blockCodecInfo = src.getBlockCodecInfo(); -// final CodecInfo[] allCodecs = new CodecInfo[byteCodecs.length + 1]; -// allCodecs[0] = blockCodecInfo; -// System.arraycopy(byteCodecs, 0, allCodecs, 1, byteCodecs.length); -// -// -// obj.add(DatasetAttributes.CODEC_KEY, context.serialize(concatenateCodecs(src))); -// return obj; -// -//// final JsonElement out = context.serialize(src); -//// return out; -// } -// } -// -// -// private static CodecInfo[] concatenateCodecs(final DatasetAttributes attributes) { -// -// final DataCodecInfo[] byteCodecs = attributes.getDataCodecInfos(); -// final BlockCodecInfo blockCodecInfo = attributes.getBlockCodecInfo(); -// final CodecInfo[] allCodecs = new CodecInfo[byteCodecs.length + 1]; -// allCodecs[0] = blockCodecInfo; -// System.arraycopy(byteCodecs, 0, allCodecs, 1, byteCodecs.length); -// -// return allCodecs; -// } - - } From 0dff84989df182e8a75449efc6f8f5e514585aa0 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 3 Nov 2025 10:18:55 -0500 Subject: [PATCH 408/423] feat!/test: clean up N5Writer delete implementation * refactor tests in shardstuff --- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 86 +------------------ .../saalfeldlab/n5/shard/DatasetAccess.java | 2 +- .../n5/shard/DefaultDatasetAccess.java | 17 +++- .../n5/shard/PositionValueAccess.java | 29 +++++-- .../saalfeldlab/n5/codec/BlockCodecTests.java | 4 +- .../{shardstuff => shard}/NestedGridTest.java | 2 +- .../{shardstuff => shard}/RawShardTest.java | 6 +- .../TestPositionValueAccess.java | 6 +- 8 files changed, 48 insertions(+), 104 deletions(-) rename src/test/java/org/janelia/saalfeldlab/n5/{shardstuff => shard}/NestedGridTest.java (98%) rename src/test/java/org/janelia/saalfeldlab/n5/{shardstuff => shard}/RawShardTest.java (95%) rename src/test/java/org/janelia/saalfeldlab/n5/{shardstuff => shard}/TestPositionValueAccess.java (87%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 12595dbd0..050f7a626 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -258,35 +258,6 @@ default boolean remove(final String path) throws N5Exception { return true; } - /** - * Delete a shard at the specified position. - * - * @param path - * the dataset path - * @param shardPosition - * the position of the shard to delete - * @return true if the shard existed was successfully deleted. - * @throws N5Exception - * if an error occurs during deletion - */ - default boolean deleteShard( - final String path, - final long... shardPosition) throws N5Exception { - - final String shardPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), shardPosition); - if (getKeyValueAccess().isFile(shardPath)) { - try { - getKeyValueAccess().delete(shardPath); - return true; - } catch (final Exception e) { - throw new N5Exception("The shard at " + - Arrays.toString(shardPosition) + - " could not be deleted.", e); - } - } - return false; - } - @Override default boolean deleteBlock( final String path, @@ -294,60 +265,11 @@ default boolean deleteBlock( final String normalPath = N5URI.normalizeGroupPath(path); final DatasetAttributes datasetAttributes = getDatasetAttributes(normalPath); + final PositionValueAccess posKva = PositionValueAccess.fromKva( + getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), + p -> datasetAttributes.relativeBlockPath(p)); - if (datasetAttributes == null) { - return false; // Dataset doesn't exist - return true for consistency - } + return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition); - // TODO -// if (datasetAttributes.isSharded()) { -// // For sharded datasets, we need to: -// // 1. Find which shard contains this block -// // 2. Read the shard -// // 3. Remove the block from the shard -// // 4. Write the shard back (or delete if empty) -// -// final long[] shardPosition = datasetAttributes.getShardPositionForBlock(gridPosition); -// final Shard shard = readShard(normalPath, datasetAttributes, shardPosition); -// -// if (shard == null) -// return false; // Shard doesn't exist, so block doesn't exist - -// // return false for consistency -// -// final int[] relativePosition = shard.getRelativeBlockPosition(gridPosition); -// if (!shard.blockExists(relativePosition)) -// return false; -// -// // Convert to InMemoryShard to manipulate blocks -// final InMemoryShard inMemoryShard = InMemoryShard.fromShard(shard); -// -// // Get all blocks except the one to remove -// final List> remainingBlocks = new ArrayList<>(); -// for (DataBlock block : inMemoryShard.getBlocks()) { -// if (!Arrays.equals(block.getGridPosition(), gridPosition)) { -// remainingBlocks.add(block); -// } -// } -// -// if (remainingBlocks.isEmpty()) { -// // If no blocks remain, delete the entire shard -// return deleteShard(normalPath, shardPosition); -// } else { -// // Create new shard with remaining blocks -// final InMemoryShard newShard = new InMemoryShard<>(datasetAttributes, shardPosition); -// for (DataBlock block : remainingBlocks) { -// newShard.addBlock(block); -// } -// -// // Write the updated shard -// writeShard(normalPath, datasetAttributes, newShard); -// return true; -// } -// -// } else { - // For non-sharded datasets, deleting the key deletes the block - // and deleteShard deletes the key for gridPosition - return deleteShard(path, gridPosition); -// } } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index 82ab75634..541c1cd8d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -19,7 +19,7 @@ public interface DatasetAccess { void writeBlock(PositionValueAccess kva, DataBlock dataBlock) throws N5IOException; - void deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; + boolean deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; List> readBlocks(PositionValueAccess kva, List positions); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 6c4bae3d5..e14e835a1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -1,12 +1,14 @@ package org.janelia.saalfeldlab.n5.shard; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.TreeMap; import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; import org.janelia.saalfeldlab.n5.codec.BlockCodec; @@ -250,19 +252,26 @@ private ReadData writeShardRecursive( } @Override - public void deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { + public boolean deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { final NestedPosition position = new NestedPosition(grid, gridPosition); final long[] key = position.key(); if (grid.numLevels() == 1) { // for non-sharded dataset, don't bother getting the value, just remove the key. - kva.remove(key); + try { + return kva.remove(key); + } catch (final Exception e) { + throw new N5Exception("The shard at " + Arrays.toString(key) + " could not be deleted.", e); + } } else { final ReadData existingData = kva.get(key); final ReadData modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); - if (modifiedData == null) { - kva.remove(key); + if (existingData != null && modifiedData == null) { + return kva.remove(key); } else if (modifiedData != existingData) { kva.put(key, modifiedData); + return true; + } else { + return false; } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java index 7576ad8fc..0f9e2892c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java @@ -23,7 +23,7 @@ public interface PositionValueAccess { void put(long[] key, ReadData data) throws N5Exception.N5IOException; - void remove(long[] key) throws N5Exception.N5IOException; + boolean remove(long[] key) throws N5Exception.N5IOException; public static PositionValueAccess fromKva( final KeyValueAccess kva, @@ -52,9 +52,20 @@ class KvaPositionValueAccess implements PositionValueAccess { this.blockPositionToPath = blockPositionToPath; } - // TODO this duplicates GsonKeyValueReader.absoluteDataBlockPath - // is this where we want the logic? - private String absolutePath( final long... gridPosition) { + /** + * Constructs the path for a shard or data block at a given + * grid position. + *
      + * If the gridPosition passed in refers to shard position in a sharded + * dataset, this will return the path to the shard key. + * + * @param normalPath + * normalized dataset path + * @param gridPosition + * to the target data block + * @return the absolute path to the data block ad gridPosition + */ + protected String absolutePath( final long... gridPosition) { return kva.compose(uri, normalPath, blockPositionToPath.apply(gridPosition)); } @@ -75,8 +86,14 @@ public void put(long[] key, ReadData data) throws N5IOException { } @Override - public void remove(long[] key) throws N5IOException { - kva.delete( absolutePath(key)); + public boolean remove(long[] gridPosition) throws N5IOException { + + final String key = absolutePath(gridPosition); + if (!kva.isFile(key)) + return false; + + kva.delete(key); + return true; } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java index cc005b5b7..3ff34059f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java @@ -22,8 +22,8 @@ import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; import org.janelia.saalfeldlab.n5.shard.DatasetAccess; import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; -import org.janelia.saalfeldlab.n5.shardstuff.RawShardTest; -import org.janelia.saalfeldlab.n5.shardstuff.TestPositionValueAccess; +import org.janelia.saalfeldlab.n5.shard.RawShardTest; +import org.janelia.saalfeldlab.n5.shard.TestPositionValueAccess; import org.junit.Test; public class BlockCodecTests { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java similarity index 98% rename from src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java rename to src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java index 2910a50e5..d213c0713 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/NestedGridTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java @@ -1,4 +1,4 @@ -package org.janelia.saalfeldlab.n5.shardstuff; +package org.janelia.saalfeldlab.n5.shard; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertThrows; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java similarity index 95% rename from src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java rename to src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java index e3a0b05a8..eaa895420 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/RawShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java @@ -1,4 +1,4 @@ -package org.janelia.saalfeldlab.n5.shardstuff; +package org.janelia.saalfeldlab.n5.shard; import java.util.Arrays; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; @@ -10,10 +10,6 @@ import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.shard.DatasetAccess; -import org.janelia.saalfeldlab.n5.shard.DefaultShardCodecInfo; -import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; -import org.janelia.saalfeldlab.n5.shard.ShardCodecInfo; import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; public class RawShardTest { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/TestPositionValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java similarity index 87% rename from src/test/java/org/janelia/saalfeldlab/n5/shardstuff/TestPositionValueAccess.java rename to src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java index 5af490285..29e609395 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shardstuff/TestPositionValueAccess.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java @@ -1,4 +1,4 @@ -package org.janelia.saalfeldlab.n5.shardstuff; +package org.janelia.saalfeldlab.n5.shard; import java.util.Arrays; import java.util.HashMap; @@ -25,8 +25,8 @@ public void put(final long[] key, final ReadData data) { } @Override - public void remove(final long[] key) throws N5IOException { - map.remove(new Key(key)); + public boolean remove(final long[] key) throws N5IOException { + return map.remove(new Key(key)) != null; } private static class Key { From ebca3401df63870376e32f6362fcd26f90a396c8 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 3 Nov 2025 10:20:11 -0500 Subject: [PATCH 409/423] BREAING!: rm GsonKeyValueN5Reader.absoluteDataBlockPath * moved to PositionValueAccess --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 37 +++---------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index abeb461df..054b7c4db 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -98,10 +98,16 @@ default DataBlock readBlock( final long... gridPosition) throws N5Exception { try { + + // method that creates a PositionValueAccess? + // that enables their re-use! + final PositionValueAccess posKva = PositionValueAccess.fromKva( getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), p -> datasetAttributes.relativeBlockPath(p)); + return datasetAttributes.getDatasetAccess().readBlock(posKva, gridPosition); + } catch (N5Exception.N5NoSuchKeyException e) { return null; } @@ -125,37 +131,6 @@ default String[] list(final String pathName) throws N5Exception { return getKeyValueAccess().listDirectories(absoluteGroupPath(pathName)); } - /** - * Constructs the path for a shard or data block in a dataset at a given - * grid position. - *
      - * If the gridPosition passed in refers to shard position in a sharded - * dataset, this will return the path to the shard key. - *

      - * The returned path is - * - * 

      -	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
-	 *
      - *

      - * This is the file into which the data block will be stored. - * - * @param normalPath - * normalized dataset path - * @param gridPosition - * to the target data block - * @return the absolute path to the data block ad gridPosition - */ - // TODO: revise javadoc -> see development branch - // TODO: rename to reflect that it may also refer to Shards - default String absoluteDataBlockPath( - final String normalPath, - final long... gridPosition) { - - final String relativeBlockPath = getDatasetAttributes(normalPath).relativeBlockPath(gridPosition); - return getKeyValueAccess().compose(getURI(), normalPath, relativeBlockPath); - } - /** * Constructs the absolute path (in terms of this store) for the group or * dataset. From ef2de58ec7141a02e15cd5a33543ae67df99f609 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 3 Nov 2025 10:56:58 -0500 Subject: [PATCH 410/423] refactor!: PositionValueAccess takes DatasetAttributes * not Function * because the function is DatasetAttributes.relativeBlockPath --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 18 +++++------------- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 16 ++++------------ .../n5/shard/PositionValueAccess.java | 14 +++++++------- 3 files changed, 16 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 054b7c4db..8ede4da3a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -98,15 +98,9 @@ default DataBlock readBlock( final long... gridPosition) throws N5Exception { try { - - // method that creates a PositionValueAccess? - // that enables their re-use! - - final PositionValueAccess posKva = PositionValueAccess.fromKva( - getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), - p -> datasetAttributes.relativeBlockPath(p)); - - return datasetAttributes.getDatasetAccess().readBlock(posKva, gridPosition); + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), + datasetAttributes); + return datasetAttributes. getDatasetAccess().readBlock(posKva, gridPosition); } catch (N5Exception.N5NoSuchKeyException e) { return null; @@ -119,10 +113,8 @@ default List> readBlocks( final DatasetAttributes datasetAttributes, final List blockPositions) throws N5Exception { - final PositionValueAccess posKva = PositionValueAccess.fromKva( - getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), - p -> datasetAttributes.relativeBlockPath(p)); - return datasetAttributes.getDatasetAccess().readBlocks(posKva, blockPositions); + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), datasetAttributes); + return datasetAttributes. getDatasetAccess().readBlocks(posKva, blockPositions); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 050f7a626..d0a9c0e66 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -217,9 +217,7 @@ default void writeBlocks( final DataBlock... dataBlocks) throws N5Exception { try { - final PositionValueAccess posKva = PositionValueAccess.fromKva( - getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), - p -> datasetAttributes.relativeBlockPath(p)); + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), datasetAttributes); datasetAttributes.getDatasetAccess().writeBlocks(posKva, Arrays.asList(dataBlocks)); } catch (final UncheckedIOException e) { throw new N5IOException( @@ -234,10 +232,8 @@ default void writeBlock( final DataBlock dataBlock) throws N5Exception { try { - final PositionValueAccess posKva = PositionValueAccess.fromKva( - getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), - p -> datasetAttributes.relativeBlockPath(p)); - datasetAttributes.getDatasetAccess().writeBlock(posKva, dataBlock); + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); + datasetAttributes. getDatasetAccess().writeBlock(posKva, dataBlock); } catch (final UncheckedIOException e) { throw new N5IOException( "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, @@ -265,11 +261,7 @@ default boolean deleteBlock( final String normalPath = N5URI.normalizeGroupPath(path); final DatasetAttributes datasetAttributes = getDatasetAttributes(normalPath); - final PositionValueAccess posKva = PositionValueAccess.fromKva( - getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), - p -> datasetAttributes.relativeBlockPath(p)); - + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition); - } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java index 0f9e2892c..f39819067 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java @@ -3,8 +3,8 @@ import java.io.IOException; import java.io.OutputStream; import java.net.URI; -import java.util.function.Function; +import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; @@ -29,9 +29,9 @@ public static PositionValueAccess fromKva( final KeyValueAccess kva, final URI uri, final String normalPath, - final Function blockPositionToPath) { + final DatasetAttributes attributes) { - return new KvaPositionValueAccess(kva, uri, normalPath, blockPositionToPath); + return new KvaPositionValueAccess(kva, uri, normalPath, attributes); } class KvaPositionValueAccess implements PositionValueAccess { @@ -39,17 +39,17 @@ class KvaPositionValueAccess implements PositionValueAccess { private final KeyValueAccess kva; private final URI uri; private final String normalPath; - private final Function blockPositionToPath; + private final DatasetAttributes attributes; KvaPositionValueAccess(final KeyValueAccess kva, final URI uri, final String normalPath, - final Function blockPositionToPath) { + final DatasetAttributes attributes) { this.kva = kva; this.uri = uri; this.normalPath = normalPath; - this.blockPositionToPath = blockPositionToPath; + this.attributes = attributes; } /** @@ -66,7 +66,7 @@ class KvaPositionValueAccess implements PositionValueAccess { * @return the absolute path to the data block ad gridPosition */ protected String absolutePath( final long... gridPosition) { - return kva.compose(uri, normalPath, blockPositionToPath.apply(gridPosition)); + return kva.compose(uri, normalPath, attributes.relativeBlockPath(gridPosition)); } @Override From 06a1e2d60a6ecbe52f114c92120631825229a29e Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 3 Nov 2025 11:06:31 -0500 Subject: [PATCH 411/423] test: rm unused testWriteReadShardOnUnshardedDataset --- .../saalfeldlab/n5/AbstractN5Test.java | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 6c8cb0a8d..60025e453 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1735,50 +1735,6 @@ public void testPathsWithIllegalUriCharacters() throws IOException, URISyntaxExc } } - @Test - public void testWriteReadShardOnUnshardedDataset() { - // TODO -// try (N5Writer writer = createTempN5Writer()) { -// final String datasetName = "testWriteShardOnUnshardedDataset"; -// final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64, new RawCompression()); -// writer.createDataset(datasetName, datasetAttributes); -// -// -// final DataBlock block0 = (DataBlock) DataType.UINT64.createDataBlock(blockSize, new long[]{0,0,0}); -// final DataBlock block1 = (DataBlock) DataType.UINT64.createDataBlock(blockSize, new long[]{1,0,0}); -// -// final InMemoryShard writeShard = new InMemoryShard<>(datasetAttributes, new long[]{0, 0, 0}); -// boolean added0 = writeShard.addBlock(block0); -// boolean added1 = writeShard.addBlock(block1); -// -// assertTrue("Block 0 should be added to shard", added0); -// assertFalse("Block 1 should not be added to shard because it's in a different shard position", added1); -// -// final List> writeBlocks = writeShard.getBlocks(); -// assertEquals("block as shard should not have the second block which is outside this shard", 1, writeBlocks.size()); -// assertEquals(block0, writeBlocks.get(0)); -// -// writer.writeShard(datasetName, datasetAttributes, writeShard); -// final DataBlock readAsBlock0 = writer.readBlock(datasetName, datasetAttributes, block0.getGridPosition()); -// assertBlockEquals(block0, readAsBlock0); -// -// assertFalse("Block 1 should not exist because it's not contained in the written shard", writer.blockExists(datasetName, datasetAttributes, block1.getGridPosition())); -// -// final Shard readShard = writer.readShard(datasetName, datasetAttributes, writeShard.getGridPosition()); -// Assert.assertArrayEquals("shard read position should be same as write position", writeShard.getGridPosition(), readShard.getGridPosition()); -// Assert.assertArrayEquals("shard position should be the same as block position when unsharded", block0.getGridPosition(), readShard.getGridPosition()); -// Assert.assertArrayEquals("shard size should equal block size when unsharded", readShard.getBlockSize(), readShard.getSize()); -// -// -// final List> readBlocks = readShard.getBlocks(); -// assertEquals("read shard should contain one block", 1, readBlocks.size()); -// final DataBlock readAsShardBlock0 = readBlocks.get(0); -// -// assertBlockEquals(block0, readAsShardBlock0); -// -// } - } - public static void assertBlockEquals(final DataBlock expected, final DataBlock actual) { assertEquals("Datablocks are different type", expected.getClass(), actual.getClass()); From 619f308b7ee9d0dd3d85d4e9e0d128ea466a0ecb Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 3 Nov 2025 15:01:39 -0500 Subject: [PATCH 412/423] style!: rm unused method registerGson --- .../java/org/janelia/saalfeldlab/n5/GsonUtils.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 9d03a8a88..3d3db1aeb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -53,18 +53,6 @@ */ public interface GsonUtils { -// static Gson registerGson(final GsonBuilder gsonBuilder) { -// -// gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); -// gsonBuilder.registerTypeHierarchyAdapter(DataCodecInfo.class, NameConfigAdapter.getJsonAdapter(DataCodecInfo.class)); -// gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); -// gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, DatasetAttributes.getJsonAdapter()); -// gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBlockCodecInfo.byteOrderAdapter); -// gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); -// gsonBuilder.disableHtmlEscaping(); -// return gsonBuilder.create(); -// } - /** * Reads the attributes json from a given {@link Reader}. * From 1bb10bb098c67ced598d957ce0f7395843f29525 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 3 Nov 2025 15:02:22 -0500 Subject: [PATCH 413/423] style!: remove unused classes * Slices * SliceTrackingReadData --- .../segment/SliceTrackingReadData.java | 151 ------------------ .../n5/readdata/segment/Slices.java | 77 --------- .../n5/readdata/segment/SlicesTest.java | 134 ---------------- 3 files changed, 362 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Slices.java delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SlicesTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java deleted file mode 100644 index 81f9116d4..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SliceTrackingReadData.java +++ /dev/null @@ -1,151 +0,0 @@ -package org.janelia.saalfeldlab.n5.readdata.segment; - -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.readdata.Range; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - -class SliceTrackingReadData implements ReadData { - - private static class Slice implements Range { - - private final long offset; - private final long length; - private final ReadData data; - - Slice(final long offset, final long length, final ReadData data) { - this.offset = offset; - this.length = length; - this.data = data; - } - - @Override - public long offset() { - return offset; - } - - @Override - public long length() { - return length; - } - - @Override - public String toString() { - return "{" + offset + ", " + length + '}'; - } - } - - private final List slices = new ArrayList<>(); - - /** - * The {@code ReadData} providing our data. - */ - private final ReadData delegate; - - private SliceTrackingReadData(final ReadData delegate) { - this.delegate = delegate; - } - - static ReadData wrap(final ReadData readData) { - return new SliceTrackingReadData(readData); - } - - @Override - public long length() { - return delegate.length(); - } - - @Override - public long requireLength() throws N5IOException { - return delegate.requireLength(); - } - - @Override - public ReadData slice(final long offset, final long length) throws N5IOException { - final Slice containing = Slices.findContainingSlice(slices, offset, length); - if (containing != null) { - return containing.data.slice(offset - containing.offset, length); - } else { - final ReadData data = delegate.slice(offset, length); - Slices.addSlice(slices, new Slice(offset, length, data)); - return data; - } - } - - @Override - public InputStream inputStream() throws N5IOException, IllegalStateException { - return delegate.inputStream(); - } - - @Override - public byte[] allBytes() throws N5IOException, IllegalStateException { - return delegate.allBytes(); - } - - @Override - public ByteBuffer toByteBuffer() throws N5IOException, IllegalStateException { - return delegate.toByteBuffer(); - } - - @Override - public ReadData materialize() throws N5IOException { - delegate.materialize(); - if (slices.size() != 1 || slices.get(0).data != delegate) { - slices.clear(); - slices.add(new Slice(0, length(), delegate)); - } - return this; - } - - @Override - public void writeTo(final OutputStream outputStream) throws N5IOException, IllegalStateException { - delegate.writeTo(outputStream); - } - - @Override - public ReadData encode(final OutputStreamOperator encoder) { - return delegate.encode(encoder); - } - - /** - * Indicates that the given slices will be subsequently read. - * {@code ReadData} implementations (optionally) may take steps to prepare - * for these subsequent slices. - *

      - * Minimal implementation: Find offset and length covering all ranges that - * are not yet fully covered by existing slices. Then materialize the slice - * covering that range. - * - * @param ranges - * slice ranges to prefetch - * - * @throws N5IOException - * if any I/O error occurs - */ - @Override - public void prefetch(final Collection ranges) throws N5IOException { - - long fromIndex = Long.MAX_VALUE; - long toIndex = Long.MIN_VALUE; - for (final Range slice : ranges) { - if (!isCovered(slice)) { - fromIndex = Math.min(fromIndex, slice.offset()); - toIndex = Math.max(toIndex, slice.end()); - } - } - - if (fromIndex < toIndex) { - slice(fromIndex, toIndex - fromIndex).materialize(); - } - } - - private boolean isCovered(final Range slice) { - - return Slices.findContainingSlice(slices, slice) != null; - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Slices.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Slices.java deleted file mode 100644 index 3d60ded62..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/Slices.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.janelia.saalfeldlab.n5.readdata.segment; - -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import org.janelia.saalfeldlab.n5.readdata.Range; - -class Slices { - - private Slices() { - // utility class. should not be instantiated. - } - - /** - * Pre-conditions: - *

        - *
      1. Slices are ordered by offset.
        2. - *
      2. If two slices overlap, no slice is fully contained within the other. (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
        3. - *
      - * - * @param slices - * ordered list of slices - * @param offset - * @param length - * - * @return - */ - static T findContainingSlice(final List slices, final long offset, final long length) { - // Find the slice s with largest s.offset <= offset. - - final int i = Collections.binarySearch(slices, Range.at(offset, 0), Comparator.comparingLong(Range::offset)); - - // Largest index of a slice with slice.offset <= offset. - final int index = i < 0 ? -i - 2 : i; - if (index < 0) { - // We find no overlapping slice, because - // slices[0].offset is already too large. - return null; - } - - final T slice = slices.get(index); - if (slice.end() < offset + length) { - return null; - } - - return slice; - } - - static T findContainingSlice(final List slices, final Range range) { - return findContainingSlice(slices, range.offset(), range.length()); - } - - /** - * Add a new {@code slice} to the {@code slice} list. - * Note, that the new {@code slice} is expected to not be fully contained in an existing slice! - * This will insert {@code slice} into the list at the correct position ({@code slices} is ordered by slice offset), and remove all existing slices that are fully contained in the new {@code slice}. - */ - static void addSlice(final List slices, final T slice) { - - final int i = Collections.binarySearch(slices, slice, Comparator.comparingLong(Range::offset)); - final int from = i < 0 ? -i - 1 : i; - - int to = from; - while (to < slices.size() && slices.get(to).end() <= slice.end()) { - ++to; - } - - if (from == to) { - // empty range: just insert - slices.add(from, slice); - } else { - // overwrite the first element in range, remove the rest - slices.set(from, slice); - slices.subList(from + 1, to).clear(); - } - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SlicesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SlicesTest.java deleted file mode 100644 index 3059f92af..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SlicesTest.java +++ /dev/null @@ -1,134 +0,0 @@ -package org.janelia.saalfeldlab.n5.readdata.segment; - -import java.util.ArrayList; -import java.util.List; -import org.janelia.saalfeldlab.n5.readdata.Range; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - - -public class SlicesTest { - - private List createSlices(final long[] offsets, final long[] lengths) { - final List slices = new ArrayList<>(); - for (int i = 0; i < offsets.length; ++i) { - slices.add(Range.at(offsets[i], lengths[i])); - } - return slices; - } - - @Test - public void testFindContaining() { - - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (2,6) [-----------] - // (6,4) [---------] - // (8,6) [-----------] - - final List slices = createSlices( - new long[] {2, 6, 8}, - new long[] {6, 4, 6}); - Range slice; - - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (1,1) [-] - slice = Slices.findContainingSlice(slices, 1, 1); - assertEquals(null, slice); - - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (2,1) [-] - slice = Slices.findContainingSlice(slices, 2, 1); - assertEquals(2, slice.offset()); - - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (2,6) [-----------] - slice = Slices.findContainingSlice(slices, 2, 6); - assertEquals(2, slice.offset()); - - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (2,7) [-------------] - slice = Slices.findContainingSlice(slices, 2, 7); - assertEquals(null, slice); - - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (6,4) [-------] - slice = Slices.findContainingSlice(slices, 6, 4); - assertEquals(6, slice.offset()); - - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (8,2) [---] - slice = Slices.findContainingSlice(slices, 8, 2); - assertEquals(8, slice.offset()); - - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (12,2) [---] - slice = Slices.findContainingSlice(slices, 12, 2); - assertEquals(8, slice.offset()); - - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (12,3) [-----] - slice = Slices.findContainingSlice(slices, 12, 3); - assertEquals(null, slice); - - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (14,1) [-] - slice = Slices.findContainingSlice(slices, 14, 1); - assertEquals(null, slice); - } - - - @Test - public void testAddSlice() { - - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (2,6) [-----------] - // (6,4) [---------] - // (8,6) [-----------] - final List initial = createSlices( - new long[] {2, 6, 8}, - new long[] {6, 4, 6}); - List slices; - - - slices = new ArrayList<>(initial); - Slices.addSlice(slices, Range.at(0, 1)); - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (0,1) [-] - // (2,6) [-----------] - // (6,4) [---------] - // (8,6) [-----------] - assertEquals(createSlices( - new long[] {0, 2, 6, 8}, - new long[] {1, 6, 4, 6}), slices); - - - slices = new ArrayList<>(initial); - Slices.addSlice(slices, Range.at(0, 16)); - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (0,16)[-------------------------------] - assertEquals(createSlices( - new long[] {0}, - new long[] {16}), slices); - - - slices = new ArrayList<>(initial); - Slices.addSlice(slices, Range.at(2, 8)); - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (2,8) [-----------------] - // (8,6) [-----------] - assertEquals(createSlices( - new long[] {2, 8}, - new long[] {8, 6}), slices); - - - slices = new ArrayList<>(initial); - Slices.addSlice(slices, Range.at(1, 10)); - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - // (1,10) [---------------------] - // (8,6) [-----------] - assertEquals(createSlices( - new long[] {1, 8}, - new long[] {10, 6}), slices); - } -} From 552d416f07197325a5341d8f2c69b97bd8aa7521 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 3 Nov 2025 15:07:59 -0500 Subject: [PATCH 414/423] chore!: rm tentative blockExists method stub --- src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 2eb932b38..e7004cdf3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -363,12 +363,6 @@ default T readSerializedBlock( } } - default boolean blockExists(final String pathName, final DatasetAttributes attributes, final long[] blockPosition) { - - // TODO implement me? or remove method? - return true; - } - /** * Test whether a group or dataset exists at a given path. * From 94b0918ea7d9e35cba11e5bcc47e631ce99c98b1 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 3 Nov 2025 15:33:14 -0500 Subject: [PATCH 415/423] Squashed commit of the following: Co-authored-by: Caleb Hulbert Co-authored-by: tpietzsch commit 552d416f07197325a5341d8f2c69b97bd8aa7521 Author: John Bogovic Date: Mon Nov 3 15:07:59 2025 -0500 chore!: rm tentative blockExists method stub commit 1bb10bb098c67ced598d957ce0f7395843f29525 Author: John Bogovic Date: Mon Nov 3 15:02:22 2025 -0500 style!: remove unused classes * Slices * SliceTrackingReadData commit 619f308b7ee9d0dd3d85d4e9e0d128ea466a0ecb Author: John Bogovic Date: Mon Nov 3 15:01:39 2025 -0500 style!: rm unused method registerGson commit 06a1e2d60a6ecbe52f114c92120631825229a29e Author: John Bogovic Date: Mon Nov 3 11:06:31 2025 -0500 test: rm unused testWriteReadShardOnUnshardedDataset commit ef2de58ec7141a02e15cd5a33543ae67df99f609 Author: John Bogovic Date: Mon Nov 3 10:56:58 2025 -0500 refactor!: PositionValueAccess takes DatasetAttributes * not Function * because the function is DatasetAttributes.relativeBlockPath commit ebca3401df63870376e32f6362fcd26f90a396c8 Author: John Bogovic Date: Mon Nov 3 10:20:11 2025 -0500 BREAING!: rm GsonKeyValueN5Reader.absoluteDataBlockPath * moved to PositionValueAccess commit 0dff84989df182e8a75449efc6f8f5e514585aa0 Author: John Bogovic Date: Mon Nov 3 10:18:55 2025 -0500 feat!/test: clean up N5Writer delete implementation * refactor tests in shardstuff commit 5f23b09d592db214a4c4af2f34bd5135819e7ecc Author: John Bogovic Date: Mon Nov 3 10:02:32 2025 -0500 test: clean up ShardTest commit 1ea9da21cf5f731e7856254c75ff24f29fda1b4b Author: John Bogovic Date: Fri Oct 31 16:16:59 2025 -0400 doc: correct DatasetAttributes.getDatasetAccess commit 2e5d86b83f65e33a1f7b52542682c1e1fb6f043b Author: John Bogovic Date: Wed Oct 29 15:14:29 2025 -0400 test: update tests to work with less accessible DatasetAccess commit 162309cbcf6c544f5659262e37ffda42cce2a5da Author: John Bogovic Date: Wed Oct 29 15:12:31 2025 -0400 feat: add getNestedBlockGrid method to DatasetAttributes * getDatasetAccess no longer public commit 1de7b706598640f06a39f330e6d9a6667075c2d8 Author: Caleb Hulbert Date: Wed Oct 29 14:00:01 2025 -0400 fix(test): don't specify relative paths for createTempN5Writer; in this case, default is fine commit 380964c6d4cae6bd991c80a6c8c51c92112062ca Author: John Bogovic Date: Fri Aug 16 14:00:54 2024 -0400 test: start of partial read benchmarks Signed-off-by: Caleb Hulbert commit ddcc821f153bfa997d212ef148bc3d53df999b85 Author: John Bogovic Date: Wed Oct 29 13:05:37 2025 -0400 test: add extensible method that defines illegal characters commit 152aef6b8f50f6a3153b69e0a0cf9ec219c01efe Merge: e9ece6e 7fe7749 Author: John Bogovic Date: Tue Oct 28 10:35:00 2025 -0400 Merge branch 'development' of github.com:saalfeldlab/n5 into development commit e9ece6e21d2fdd47ecd9461100b9ea04d72ba0a5 Author: John Bogovic Date: Tue Oct 28 10:34:47 2025 -0400 doc: improve Nesting.positionInSubGrid doc commit a4f8b4c7fee2f2cdefbfd63e45cde2697c0dfb9d Author: John Bogovic Date: Tue Oct 28 10:34:25 2025 -0400 test: generalize ShardTest for n5-zarr re-use commit 7fe774905e05ec9d5432881298d67b5c128d5e34 Author: Caleb Hulbert Date: Fri Oct 24 16:59:54 2025 -0400 feat(test): change test size defaults so the dimensions are larger than the blocks; add tests for blocksize/dimensions edge cases commit 574127b32b4cd4f37430901edfdc70c4e68c086c Merge: 14cd4f5 8442671 Author: John Bogovic Date: Thu Oct 23 15:19:48 2025 -0400 Merge branch 'feat/attributesKey' into development # Conflicts: # src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java # src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java # src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java # src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java commit 844267138974525b795e0573ba8b136641f9e1fa Merge: f1e82a7 41fb0e5 Author: John Bogovic Date: Thu Oct 23 14:37:44 2025 -0400 Merge branch 'master' into feat/attributesKey commit 14cd4f5762ca1e91aa3778104d5cb35f43e911d0 Merge: 685eabe 41fb0e5 Author: John Bogovic Date: Thu Oct 23 13:57:46 2025 -0400 Merge remote-tracking branch 'origin/master' into development # Conflicts: # src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccessReadData.java # src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java # src/test/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentTest.java commit 685eabe960664881bd4a499d229f261c10364220 Author: John Bogovic Date: Wed Oct 22 22:03:31 2025 -0400 feat: add NestedGrid.positionInSubGrid commit d53b7a6de4276ab9caf03142d5e75733f7a01156 Author: John Bogovic Date: Wed Oct 22 17:56:30 2025 -0400 test: additions to ShardTest commit 26576f248c324fd7054e96d6010a2dabc83405b5 Merge: 8b9a85f fc4a1ec Author: John Bogovic Date: Wed Oct 22 17:05:32 2025 -0400 Merge branch 'development' of github.com:saalfeldlab/n5 into development commit 8b9a85fe039c6dbd2b8d8fb977aa7a2c85b50f46 Author: John Bogovic Date: Wed Oct 22 17:05:28 2025 -0400 test: more NestedGrid tests commit fc4a1ec97e2642a19a1cc977bb6502ef2998c659 Author: tpietzsch Date: Wed Oct 22 22:37:58 2025 +0200 clean up Better to not have a default implementation for requireLength(). The details are different and tricky enough between ReadData implementations to encourage thinking about every case individually. commit 8a491ef41a986d7c347dd2f7b019ee97cbe54221 Author: tpietzsch Date: Wed Oct 22 22:29:51 2025 +0200 typo commit 5eab7660bfb74cbac3fda4a773f0b330fd0d3ff4 Author: John Bogovic Date: Wed Oct 22 16:09:28 2025 -0400 refactor/doc: NestedGrid * rename NestedGrid fields * add some basic javadoc commit 0b8ffed749064807ea8ecd4bbf97f672a8546b1f Author: tpietzsch Date: Wed Oct 22 22:06:37 2025 +0200 javadoc commit 15c5768a1e606fd5ab850e7cffe07d42dd1ab03c Merge: 7120b38 387a10b Author: John Bogovic Date: Wed Oct 22 13:42:38 2025 -0400 Merge branch 'development' of github.com:saalfeldlab/n5 into development commit 7120b3875f06f132ae19f6d7d5ac75634f4699c5 Author: John Bogovic Date: Wed Oct 22 13:42:30 2025 -0400 test: update getTestAttributes given dstAttribute changes commit 9186e2d066caa7597c93ce72fee845d01874ff8b Author: John Bogovic Date: Wed Oct 22 13:42:12 2025 -0400 fix: DefaultShardCodecInfo's serialization * default serialization should be for zarr commit 6c99a00dbe75b306f9f2f6a241341a0a81db3b19 Author: John Bogovic Date: Wed Oct 22 13:41:23 2025 -0400 wip!: DatasetAttributes changes * remove constructor with shard size * createDatasetAccess method now protected (for zarr) * rm defaultShardCodecInfo (n5 format does not use it) commit 80d9e01dab6e10802f5fff0df13f6b187b0de1ff Author: John Bogovic Date: Wed Oct 22 13:39:45 2025 -0400 fix: IdentityCodec implements DeterministicSizeDataCodec commit 387a10b8b32dad10ed97cfb2c87597b29b968698 Author: tpietzsch Date: Wed Oct 22 13:00:36 2025 +0200 javadoc commit e980a041f8689c857b40d3db6d4633fe5504de47 Author: tpietzsch Date: Wed Oct 22 12:17:17 2025 +0200 Add static Range.equals() to check two Ranges for equality commit d987cd461608fce2e3415c00175cefc63d8cc3f7 Author: tpietzsch Date: Wed Oct 22 12:16:40 2025 +0200 augment and document concatenate test commit de5ec1a98f710d02018a4b48202142866cd49081 Author: tpietzsch Date: Wed Oct 22 10:30:22 2025 +0200 refactor: rename Concatenate to ConcatenatedReadData commit f807bd767f86ec4f2e157a9b9754e79ffb3dcec3 Author: tpietzsch Date: Tue Oct 21 21:51:52 2025 +0200 javadoc commit 81a4890c9f5a7b2a02db18c9d894393ff0a65eff Author: Caleb Hulbert Date: Tue Oct 21 13:03:22 2025 -0400 fix: return empty list for readBlocks over nonexistent blocks, not null commit a4ea057148cc54f2d29ef24fb435a9af2d491ce1 Author: tpietzsch Date: Tue Oct 21 16:57:53 2025 +0200 refactor: rename SegmentLocation to Range and move to readdata package commit 6fb023c95250419816385601834a32b8c73608f0 Author: tpietzsch Date: Tue Oct 21 16:49:31 2025 +0200 refactor: Make DefaultSegmentLocation a local class commit d3b111a3f0a14f4edb8ddfab9d4e1ac8f94ca9ef Author: tpietzsch Date: Tue Oct 21 13:41:30 2025 +0200 Use SegmentLocation.at instead of constructor commit c8f0cad53778fa55b228e74bc16dd94c777e8e21 Author: tpietzsch Date: Tue Oct 21 13:24:44 2025 +0200 clean up imports commit 4d2d201413fb6361e47747565ab101c5c136033f Author: tpietzsch Date: Tue Oct 21 13:11:13 2025 +0200 javadoc commit 5a1d57c5ce0803d59033c478310091495d96a239 Author: tpietzsch Date: Tue Sep 23 21:31:47 2025 +0200 DefaultSegmentedReadData forwards prefetch() to delegate commit eeca2a4f02b8aab864c1d7823655ff2dcf3e2f4f Author: tpietzsch Date: Tue Sep 23 21:28:00 2025 +0200 Add prefetch() to ReadData interface default implementation does nothing commit efc6b5770d7e5a007e48d76dabc740591455a1e7 Author: tpietzsch Date: Tue Sep 23 21:21:25 2025 +0200 Remove redundant modifier commit 7c5b609ef74a23975a06475f51c4d41073e205ed Author: tpietzsch Date: Tue Oct 21 12:29:36 2025 +0200 clean up commit 10fc3df0da66da5ac3a29ebec4c957d9bddbda4d Author: John Bogovic Date: Fri Oct 17 15:01:07 2025 -0400 refactor: rm old shard classes, shardstuff -> shard commit 24a75d6006b4341862336bcfc753774b5f7fe887 Author: John Bogovic Date: Fri Oct 17 14:22:57 2025 -0400 test: AbstractN5Test fix random seed commit 1f13153f0301ccc7cb23c97f04b05eace6b261d3 Author: John Bogovic Date: Fri Oct 17 14:20:27 2025 -0400 refactor: rename ShardedDatasetAccess to DefaultDatasetAccess * move creation into DatasetAttributes commit 16cd811e3d41944b73a6bdbcc3bb82948a5413b2 Author: John Bogovic Date: Thu Oct 16 13:04:24 2025 -0400 feat: StringDataBlock uses UTF_8 encoding commit ace0b16a439ac9387cb89ad5b6ef1a4f905d4476 Author: John Bogovic Date: Thu Oct 16 10:52:42 2025 -0400 test: remove hard-coded compressor for testOverwriteBlock commit 61e6075ab9bb112bf0aad86f2b0cf09d10f933cc Author: John Bogovic Date: Thu Oct 16 10:52:25 2025 -0400 fix: NameConfig.Names should not collid with CompressionType name * because Compression is also a DataCodecInfo * causing serialization issues for zarr commit 6e2dfd364acb68c859e281d3c4f7b514f05e33db Author: John Bogovic Date: Tue Oct 14 16:41:38 2025 -0400 feat: DatasetAttributes can map grid positions to relative path * necessary for zarr's dimensionSeparator * use this in PositionValueAccess commit f5385ba686be1123dde7871f791dc04c6a45b739 Author: John Bogovic Date: Tue Oct 14 16:34:38 2025 -0400 feat: add block / data codecinfo getters * useful for zarr commit e73ecd3ee41b88e6e02fbf2ff07ba20bd339696b Author: John Bogovic Date: Tue Oct 14 16:34:06 2025 -0400 DatasetAttributes clean up commit 98d7618080fca32357f382822bb99a38ee297a11 Author: John Bogovic Date: Tue Oct 14 16:32:32 2025 -0400 n5.list returns a sorted list commit feb70afebd049746b95c954952bc5b11a47b9341 Author: John Bogovic Date: Tue Oct 14 16:32:12 2025 -0400 createDataset takes Block/DataCodecInfos * and has a default BlockCodecInfo * guard against null dataCodecInfos commit a379ed400bc17351b54d85019b787aa7e08de5ab Author: John Bogovic Date: Tue Oct 14 16:29:44 2025 -0400 remove read/write Shard implementations commit 901642c0d78b41e5d7084fa898ad2cb3e503c9c0 Author: John Bogovic Date: Tue Oct 14 16:24:14 2025 -0400 test: ignore testWriteInvalidBlock for the moment commit 493db08e279ebe577cdf90f001c8b717495835aa Author: John Bogovic Date: Tue Oct 14 16:23:03 2025 -0400 remove read/write/delete Shard interface methods commit c940ebbb5b641c2d40f12792edab3ce694a68807 Author: John Bogovic Date: Tue Oct 14 16:17:22 2025 -0400 style: rm unused dependencies commit 7751fade975c1222aa9c4969bbae687c34a1fedd Merge: 3dcb8f0 ff994d4 Author: John Bogovic Date: Wed Oct 8 18:07:10 2025 -0400 Merge branch 'wip/nesting-into-development' of github.com:saalfeldlab/n5 into wip/nesting-into-development commit 3dcb8f0c28271c54e845d74641aae63e6ca81a63 Author: John Bogovic Date: Wed Oct 8 18:06:28 2025 -0400 test: start backward compatibility tests * check that old data can be read * check that written data matches some specfied version * adds legacy data from previous major versions * adds scripts to create data for future versions commit ff994d4c4bf7b222d856fcfd6aea7669806d0a61 Author: Caleb Hulbert Date: Mon Oct 6 16:11:02 2025 -0400 feat: nested implementation of readBlocks/writeBlocks commit 2091dad4ed8682579d3a37c2dd71176262d9d2ee Author: John Bogovic Date: Mon Oct 6 16:13:38 2025 -0400 make DatasetAccess transient commit e1725361db21fb66d019135492291ff9e7856fff Author: John Bogovic Date: Mon Oct 6 16:09:41 2025 -0400 partially working ShardTest commit 8f7f5da363ee52caa292c8cae982562d4547ee07 Author: John Bogovic Date: Mon Oct 6 16:09:30 2025 -0400 feat: serialization for DefaultShardCodecInfo and RawCompression commit 6232eda77537e3ba82276de3e64cd837de9be0c2 Author: John Bogovic Date: Fri Sep 26 08:24:33 2025 -0400 fix: ShardedDatasetAccess set existingData to null if no key commit e61b513f0a0fa164663c75693ecac65c30c2e357 Author: John Bogovic Date: Wed Sep 24 21:54:57 2025 -0400 fix: null check on existingData in writeBlockRecursive is not enough * when the existingData is a LazyReadData commit 1db5b73211e25f1b596d58724cf1f7afb3cca52b Author: John Bogovic Date: Wed Sep 24 21:53:55 2025 -0400 fix: ShardedDatasetAccess create method with validation commit 3d97e94131531eead8604acb21af6ac89fe25401 Author: John Bogovic Date: Wed Sep 24 15:50:17 2025 -0400 wip: tmp comment shard related things commit 07b4b811cbcf7d809a97c2e4e3ac1697d65a19a6 Author: John Bogovic Date: Wed Sep 24 15:50:01 2025 -0400 fix: DatasetAttributes build dataCodecInfos from shard size * tmp commenting out some shard methods commit 10b87c2166f1499f4fe10bc4cb2003463f4bf75d Author: John Bogovic Date: Wed Sep 24 15:42:00 2025 -0400 feat: add NestedGrid.getBlockSize commit ec27cb595471dd146ba319260deca06245c4b4be Author: John Bogovic Date: Wed Sep 24 15:38:06 2025 -0400 fix: Nesting blockSize size checks were backwards commit 1861cc1891ca3bd4b79c3dc80dc0ae5263918024 Author: John Bogovic Date: Wed Sep 24 12:02:42 2025 -0400 fix: use NestedGrid for error checking * before building BlockCodecs commit 552d7123332f94819aea30b78fdd5d4c9fbc2edb Author: John Bogovic Date: Wed Sep 24 11:58:00 2025 -0400 fix: NestedGrid validates blockSizes commit 8cba4f9face221e30f0a3b4bec98890f273d00ba Merge: 3ac4f2f 39f1c8a Author: John Bogovic Date: Tue Sep 23 15:24:19 2025 -0400 Merge remote-tracking branch 'origin/nesting' into wip/nesting-into-development commit 3ac4f2fa01e3e7034a9d46a34b0e033663b1b00a Merge: f255fd8 6dec879 Author: John Bogovic Date: Tue Sep 23 15:23:48 2025 -0400 Merge branch 'nesting' into development # Conflicts: # src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java # src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java # src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java # src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java # src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java commit 39f1c8a4c69acef654c21180f2899e7ad29c3fd5 Author: tpietzsch Date: Mon Sep 22 22:45:21 2025 +0200 minimal prefetch implementation commit 97098890e371a5bf166c5fd76596bc56e43ab2a3 Author: tpietzsch Date: Mon Sep 22 22:36:09 2025 +0200 Add (empty) prefetch method commit cc1312a808543b01b06b5025f4e8a0c53e3a3ee6 Author: tpietzsch Date: Mon Sep 22 11:31:52 2025 +0200 WIP: Always use SliceTrackingReadData in SegmentedReadData wrapper commit 2082f86e564970e0769f4334146b3f67d43709df Author: tpietzsch Date: Mon Sep 22 21:51:29 2025 +0200 WIP: Add SliceTrackingReadData commit e8ce75be41fded5cb34c047758ac421222aba4bf Author: tpietzsch Date: Mon Sep 22 21:51:05 2025 +0200 Slice prefetching math and tests commit db1928dbf6bef9240eaf2ab3d26a83ff521e4950 Author: tpietzsch Date: Mon Sep 22 21:47:32 2025 +0200 Revise SegmentLocation Add TODO for potential renaming: SegmentLocation will be uased in other instances that require a (offset, length) pair. Therefore, it should probably be renamed to "Range" or similar. Add convenience method end() == offset() + length(). Add equals/hashCode for DefaultSegmentLocation. commit 6dec87911fc7c7b9bc20727179ba020195409914 Author: John Bogovic Date: Wed Sep 17 20:00:46 2025 -0400 feat!: use DatasetAccess instead of BlockCodec * rm getBlockCodecInfo and getBlockCodec from DatasetAttrubutes * add getDatasetAccess * use getDatasetAccess in readBlock / writeBlock * add PositionValueAccess wrapping KeyValueAccess commit 373c0ecf3438e121b1e87ca202be1ec4272715de Author: John Bogovic Date: Wed Sep 17 19:58:49 2025 -0400 fix/test: ReadData test for length vs requiredLength commit 6b284e0746a47946df44fa0848cb9a39c2ab0b8c Author: tpietzsch Date: Sat Sep 13 14:38:54 2025 +0200 RawCompression is a DeterministicSizeDataCodec commit 73042ccb6c4bd209335bd11d60d24a6e79f6a44b Author: tpietzsch Date: Sat Sep 13 14:36:52 2025 +0200 Concatenation of DeterministicSizeDataCodec commit ce464735a092f78ed4e9dbc61902777db38d9aa3 Author: tpietzsch Date: Fri Sep 12 23:41:14 2025 +0200 Add BlockDodec.encodedSize(int[]) and implementations commit 500f600dcbf20e946baadebab430c9d3a0d13261 Author: tpietzsch Date: Fri Sep 12 23:17:40 2025 +0200 clean up commit f9645d8762d708e17956fa1eb6b14c1161aa0d61 Author: tpietzsch Date: Fri Sep 12 23:17:33 2025 +0200 MAke ShardIndex package-private commit ca0c9f87f42b88ac38dab7108eaf3b6bca8c6ac9 Author: tpietzsch Date: Fri Sep 12 23:17:18 2025 +0200 add TODO commit cba45a4c97ca24a48cfc809c9db1131bcbaaf9b8 Author: tpietzsch Date: Mon Sep 8 22:39:33 2025 +0200 refactor commit 76c83c041a48823357b9fbed8304494c2fbf66c2 Author: tpietzsch Date: Mon Sep 8 22:37:28 2025 +0200 refactor commit dad4adc4ba1d8fe715458b1983a55afcd32076d8 Author: tpietzsch Date: Mon Sep 8 22:17:11 2025 +0200 refactor commit 9627749976b618f6fce798b80db42d63661ad3eb Author: tpietzsch Date: Mon Sep 8 22:03:00 2025 +0200 Clean up commit 6531f00b3d878f19a3dd62faa9e706d20dccae4d Author: tpietzsch Date: Mon Sep 8 21:59:17 2025 +0200 Add some TODOs commit 35166d8980a9adccafd88f0772edfb51d9e30d22 Author: tpietzsch Date: Mon Sep 8 21:50:46 2025 +0200 WIP deleteBlock commit e233eab29c7cfd2ca86ade6336b283ddd60354c7 Author: tpietzsch Date: Mon Sep 8 18:10:47 2025 +0200 WIP testing commit 09000189aadddce45b6176035a0cc72898fb6c93 Author: tpietzsch Date: Mon Sep 8 17:27:49 2025 +0200 bugfix: ReadData should know its length() after writeTo(OutputStream) commit 0e00feaae6bebc2845de6bb27be09c28ffc549e1 Author: tpietzsch Date: Mon Sep 8 17:23:59 2025 +0200 WIP testing commit 8fc92447f9121bf35ce300b6edc663ad85f46054 Author: tpietzsch Date: Mon Sep 8 16:40:37 2025 +0200 WIP commit b482a4b320bf1b6d417b92872830cb94b45d4c69 Author: tpietzsch Date: Mon Sep 8 13:24:16 2025 +0200 WIP DatasetAccess commit 353d2ac314c077c5696780c8c7f10b046a6ea12f Author: tpietzsch Date: Mon Sep 8 10:28:07 2025 +0200 WIP commit a9d9500f230cc28bb935508eca1643a6b5fc6e50 Author: tpietzsch Date: Sun Sep 7 22:12:10 2025 +0200 DefaultShardCodecInfo commit 524b2e75055d6d3c71a98031a9a7a623fbb9d5bf Author: tpietzsch Date: Sun Sep 7 22:02:53 2025 +0200 WIP dummy CodecInfos commit d3cae93f63732c5dfdfc3e5af653348a8ab3365f Author: tpietzsch Date: Sun Sep 7 22:02:35 2025 +0200 WIP index CodecInfo and location commit 9b501dd383704a140d0d48536a5f0fa00bd086cd Author: tpietzsch Date: Sun Sep 7 20:33:12 2025 +0200 refactor commit 2b1a38d38a9b1306052dea1f7eaba820fd519cad Author: tpietzsch Date: Sat Sep 6 22:57:46 2025 +0200 clean up commit 926348ce13d86b8792582bf6ee2b496eaa80127a Author: tpietzsch Date: Sat Sep 6 22:56:48 2025 +0200 WIP RawShard encoding commit 69f27ed2590b8c1a8f0441f1986a82fff79b455e Author: tpietzsch Date: Sat Sep 6 22:52:25 2025 +0200 WIP RawShard encoding commit 9f56850e25ab0baece538f283a7f731521fd9c9f Author: tpietzsch Date: Sat Sep 6 22:21:37 2025 +0200 refactor commit 8f7e4c301f1268b83dffb6fbc9ecb0796eafd3a7 Author: tpietzsch Date: Sat Sep 6 21:54:00 2025 +0200 javadoc commit e0289b7c2f724f823ecf742f87ec2ff4c02a3954 Author: tpietzsch Date: Sat Sep 6 21:44:12 2025 +0200 WIP RawShard deconding commit 816924cdc243de60eee6860710714452bf7ac619 Author: tpietzsch Date: Sat Sep 6 21:43:28 2025 +0200 javadoc commit 30d9e4069bd6b97dab4b44d7f2b447a92861dc92 Author: tpietzsch Date: Sat Sep 6 21:43:03 2025 +0200 clean up commit 55a0962a501634811cf0c27f5b70c643be90f323 Author: tpietzsch Date: Sat Sep 6 17:14:11 2025 +0200 WIP commit 7f87a746d96c9eb828e6e59d80d23971fafd7eb1 Author: tpietzsch Date: Fri Sep 5 13:14:25 2025 +0200 WIP ShardIndex to/from Segments commit 455afce7be403d9101b0701f9e34070de7cdbdab Author: tpietzsch Date: Fri Sep 5 12:54:59 2025 +0200 WIP commit d29a4f2e9bb219470bd9a811335052b7e72f670a Author: tpietzsch Date: Fri Sep 5 12:54:56 2025 +0200 Revise DataCodecInfo to take DataType and blockSize instead of DatasetAttributes This will make more sense when instantiating from nested ShardCodecs commit 0b2c658b9de74291439b6af9e8779b73505b4b70 Author: tpietzsch Date: Fri Sep 5 12:52:54 2025 +0200 Segment.source() is the SegmentedReadData that originally defined it Idea is that this allows to slice() the Segment from its source(), so we don't need to keep the sources around explicitly in shard implementation. commit c4ea774f623ecfc2200c803249a125e9b650ab05 Author: tpietzsch Date: Fri Sep 5 10:06:56 2025 +0200 WIP requireLength commit 44d5a75c70f9501f24e932a05dc4c472d4c2f462 Author: tpietzsch Date: Wed Sep 3 23:21:18 2025 +0200 Add ReadData.requireLength commit d719fbbf68d8ae0700a04e1c12ef85e3c698227f Author: tpietzsch Date: Thu Sep 4 22:11:43 2025 +0200 Add ReadData.materialize javadoc (implementation note) commit 9d641f358564ed27ea62ed88d79420ae9fe1fafb Author: tpietzsch Date: Thu Sep 4 10:41:03 2025 +0200 WIP handle ReadData.slice with length < 0 commit c6019fdbd2e1e7cd9b0f9be16fedb0e2b2aa4997 Author: tpietzsch Date: Wed Sep 3 22:58:34 2025 +0200 WIP Concatenate commit 686da94f0e441015b1fc1a8a6f87f9f3a28992b6 Author: tpietzsch Date: Tue Sep 2 21:42:26 2025 +0200 WIP clean up commit f503f55ceac628ce2f03a379604bf59560dcc492 Author: tpietzsch Date: Tue Sep 2 21:30:17 2025 +0200 WIP slicing commit 02013f44e7e4d3020d3e2fafd0d6f93db5a18b1b Author: tpietzsch Date: Tue Sep 2 12:59:48 2025 +0200 WIP commit e32e7b07303488520735276a676e4eaa3f2b2c98 Author: tpietzsch Date: Tue Sep 2 12:49:15 2025 +0200 Revise ReadData implementations to return itself from materialize() commit ed349e22d6e1940feca022379a64f098985b882c Author: tpietzsch Date: Tue Sep 2 11:32:18 2025 +0200 WIP SegmentedReadData commit 77048145abf9f1ceac8bc975e4119d51f9ba1088 Author: tpietzsch Date: Mon Sep 1 09:06:13 2025 +0200 wip commit d041601408be9486b7b26fd8568370f03aaeb4e7 Author: tpietzsch Date: Sun Aug 31 22:43:58 2025 +0200 WIP SegmentedReadData commit 8d2232dfda0931cad1f34fca34315133e912e892 Author: tpietzsch Date: Sat Aug 30 22:09:12 2025 +0200 WIP nested decoding commit 637cb52bb6b5c8a22f0903dbaed4a4bb11fc14ef Author: tpietzsch Date: Fri Aug 29 22:19:52 2025 +0200 WIP nested decoding again commit 37b51ce51200adc95109a0459372d8d691137afd Author: tpietzsch Date: Fri Aug 29 17:24:57 2025 +0200 WIP NestedPosition commit f255fd8e0bad4f013e5a2b819eed6e4b5c1cc520 Merge: 1dd88cb 4364b82 Author: Caleb Hulbert Date: Fri Aug 29 11:14:14 2025 -0400 Merge branch 'master' into development commit 1dd88cbddca2555f2a0d7e4af18c6a393768aec8 Author: Caleb Hulbert Date: Fri Aug 29 09:52:03 2025 -0400 refactor: large renaming to align with master before merge commit 4b65d41bd45812dad2292899a4836005bb91d520 Author: tpietzsch Date: Fri Aug 29 12:30:02 2025 +0200 WIP NestedGrid commit 94fdf95350dcee9f938c8cab603aaa772d4acc1f Author: tpietzsch Date: Fri Aug 29 08:25:50 2025 +0200 move WIP to separate package commit d9e2f373c22a4bae9fef34cc0581a405def3fc91 Author: tpietzsch Date: Thu Aug 28 21:56:47 2025 +0200 WIP commit a97e604c0fb805bf3513335b8d850fde6f8f49da Author: tpietzsch Date: Thu Aug 28 18:28:38 2025 +0200 WIP commit 72ede5419ff40b43883bedad94b3cd0d0643de73 Author: tpietzsch Date: Thu Aug 28 17:40:42 2025 +0200 WIP commit f8abd61922874d32ccd50f9b0f1b664f3949d11e Author: tpietzsch Date: Thu Aug 28 16:59:37 2025 +0200 WIP commit 0f84cb5bb4842c0b5aa53c6cc6611e5f7f85441e Author: tpietzsch Date: Thu Aug 28 13:39:36 2025 +0200 WIP commit 2f2e2fe0a0c46fb56d4e0fd7eeac34d865f112ce Author: tpietzsch Date: Thu Aug 28 11:24:29 2025 +0200 WIP BlockCodecInfo commit 4e3d353c3201bff2aea68a3a0584728fc34331c2 Author: tpietzsch Date: Thu Aug 28 11:24:02 2025 +0200 Remove superfluous modifiers commit 609e4f2c0fa8f87fe796ceb9d832f598fdb01d93 Author: tpietzsch Date: Wed Aug 27 17:35:42 2025 +0200 WIP commit f1e82a7cb3a5a8fbbe08c40a63d83e3837cae48e Merge: 7b9dee4 4364b82 Author: John Bogovic Date: Wed Aug 27 10:37:30 2025 -0400 Merge remote-tracking branch 'origin/master' into feat/attributesKey # Conflicts: # src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java # src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java commit d78f161858986d5b30acd4aa08b312da7528b477 Author: tpietzsch Date: Wed Aug 27 11:58:14 2025 +0200 temp commit 7b9dee43f54c5941eb2805f00ff5134f3f48fc60 Author: John Bogovic Date: Tue Aug 26 16:50:44 2025 -0400 feat: make registerGson an instance method * so that it can be overrided by ZarrV3Reader * remove static registerGson in GsonUtils commit bce0138f64cd4fe495d05326b6d5d933acbdb03a Author: John Bogovic Date: Tue Aug 26 16:49:57 2025 -0400 RawBytes type should be "bytes" * so that it can be used for zarr v3 commit 51e510ddf9d77f04d6cedc898fcbf1e9a93a99ae Author: John Bogovic Date: Tue Aug 26 16:47:33 2025 -0400 fix: remove attribute should use setAttributes * not writeAttributes * this change allows zarr v3 to re-use this method commit 7915ed364405b4e8db79d4fb58d01ce827d7884b Author: John Bogovic Date: Wed Aug 20 15:37:46 2025 -0400 feat: getAttributesKey method * enables zarr to re-use the implementation here re: attributes commit 75772b8720a76595dc6f5fdaf39e67b78cf19c78 Author: John Bogovic Date: Wed Aug 20 15:36:28 2025 -0400 feat: DatasetAttributes.getCodecs * downstream implementations (zarr) need access to the bytesCodecs commit 0fa24086bb05ab5ce7c3adb9cdb7803e09cd5934 Merge: a9e9822 45b9d71 Author: Caleb Hulbert Date: Wed Aug 13 18:58:41 2025 -0400 Merge branch 'master' into development # Conflicts: # src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java # src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java # src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java # src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java # src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java # src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java # src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java # src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodec.java # src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java # src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytes.java commit a9e982205b3f28ba5648605e0d07e9e714d4f9aa Author: John Bogovic Date: Wed Jul 9 16:02:53 2025 -0400 feat: add GridIterator.positionToIndex * useful in n5-imglib2 commit 6ed67a8a67baee5b1e1a72c4bb410ddc8dae40d3 Author: John Bogovic Date: Tue Jul 8 13:24:56 2025 -0400 test: TestN5ShardWriter should not register Compression adapter commit 939cb77e27738b0689634e1e0554cb04a795914f Author: John Bogovic Date: Tue Jul 8 13:24:38 2025 -0400 fix: InMemoryShard.fromShard returns null for null input commit 1f93f685517069f6c189cde58fe181ed09c723c5 Author: John Bogovic Date: Tue Jul 8 13:24:15 2025 -0400 fix: ShardingCodec.getCodecs commit 7c5bd7e6a8cacc6785d74aeeac4112bebc6aff05 Author: John Bogovic Date: Mon Jul 7 19:08:13 2025 -0400 fix/test: serialization of DatasetAttributes * ensure that n5-format serialization is unchanged * separate test of sharding functionality commit 2be0d47d23ac256906658d69aad95df1d3750d51 Author: John Bogovic Date: Mon Jul 7 19:04:49 2025 -0400 refactor: mark getCompression() as deprecated * instead of private * to make updates a little easier commit d31ce223d63b0955e5567895b57b912dc0ed4f26 Author: John Bogovic Date: Thu Jul 3 09:34:00 2025 -0400 feat/doc: add ShardIndex.setEmpty * add documentation commit 3ea57cc0f825e8ff77246646c639dd1e423e8571 Author: John Bogovic Date: Wed Jul 2 16:27:36 2025 -0400 feat: add GridIterator.nextInt * shards index their blocks with int[] * avoids creating many int[]s commit e22e50515e5aae9b91d6207bb66d3651cc5002fe Author: John Bogovic Date: Wed Jul 2 14:49:46 2025 -0400 refactor!: blockPositions for shard methods are shardRelative * rename and better document methods taking blockPositions * refactoring * update tests commit 936091defc91ecbd7201192383e3b0e3f56665e2 Author: John Bogovic Date: Tue Jul 1 13:59:36 2025 -0400 refactor/test: remove ShardProperties * move methods to DatasetAttributes * rm ShardPropertiesTests, add DatasetAttributesTest commit e1468f194b5d0da17ded1fa452b574f76bf96440 Author: John Bogovic Date: Mon Jun 30 16:10:28 2025 -0400 fix/test: fix N5Codecs.getSize * split BytesTests to ArrayCodecTests and BytesCodecTests * better test for CodecSerialization commit 77e86e42afaeb39dccf311361137cb9e2311a4c9 Author: John Bogovic Date: Sat Jun 28 18:08:04 2025 -0400 feat!: add N5Writer.deleteShard * add tests for sharded and unsharded datasets * change return value behavior for delete methods commit 9e115342652a14198ce75315e3862295d03d72ac Author: John Bogovic Date: Sat Jun 28 18:04:51 2025 -0400 feat: add Shard.blockExists commit a541c2f03bec381b0cb41bf02f40460cebce0ea5 Author: Caleb Hulbert Date: Fri Jun 27 13:08:02 2025 -0400 chore: cleanup for PR commit d0158c2011b746a1fb242c61e27682632f40de9d Author: Caleb Hulbert Date: Fri Jun 27 10:24:05 2025 -0400 chore!: remove AsTypeCodec and related logic commit a5c44d46f5060093b57e25605e7c8ce91d98ed1c Merge: b1c785b 3955cdf Author: Caleb Hulbert Date: Fri Jun 27 10:12:17 2025 -0400 Merge branch 'master' into development commit b1c785b7ee6c3df62c33843c0dc9367f75b2bfb6 Author: Caleb Hulbert Date: Tue Jun 24 15:26:41 2025 -0400 doc: update README.md commit 6ed4d8fce687c93d75767a8f00a15a4d78d41c26 Merge: 5c9a4f0 7724f25 Author: Caleb Hulbert Date: Tue Jun 24 15:22:38 2025 -0400 Merge branch 'master' into development # Conflicts: # src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java # src/main/java/org/janelia/saalfeldlab/n5/readdata/KeyValueAccessReadData.java # src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java commit 5c9a4f008608084bbc392fb9cd9ed04c17ce5190 Author: Caleb Hulbert Date: Fri Jun 13 17:11:08 2025 -0400 wip: shard merge after staged PR merges commit fccd7e64440c5d6a6c9e822d5f60becfb3bf254a Merge: 41d5281 fbae8ac Author: Caleb Hulbert Date: Fri Jun 13 16:53:33 2025 -0400 Merge branch 'wip/codecsReadData' into development commit 41d52813fa38a4218552fdc5ce9204d901ffe7d9 Merge: ba7c3e0 725c0b2 Author: Caleb Hulbert Date: Fri Jun 13 15:36:37 2025 -0400 Merge branch 'splittable-readdata' into development commit ba7c3e079a56a4ee2c280072d68225d30d40a634 Merge: 0f69cfd 322b2b7 Author: Caleb Hulbert Date: Fri Jun 13 15:25:55 2025 -0400 Merge branch 'wip/codecs' into development commit fbae8ac5e298c94f118fe4965c8a69ed214462b1 Author: Caleb Hulbert Date: Thu Jun 12 11:23:21 2025 -0400 test: BlockAsShard AbstractN5 Test commit 481f75afa324dd07af1cbaa0760b276a4c034855 Author: Caleb Hulbert Date: Thu Jun 12 11:05:46 2025 -0400 misc: cleanup more blockAsShard stuff commit bf0920c5626b1562f79cbcf1915e893f5cc1c781 Author: Caleb Hulbert Date: Thu Jun 12 11:05:07 2025 -0400 feat: annotation to indicate a non-serializable NameConfig commit fff741b38a5ca4c4f7893aa7cb6136080ef78782 Author: Caleb Hulbert Date: Thu Jun 12 10:31:19 2025 -0400 test: write shard for http fs tests commit 3c20ea017956e123fecf13f636cd32732a8051f9 Author: Caleb Hulbert Date: Thu Jun 12 10:22:08 2025 -0400 refactor: static readFromShard commit 74e1f700cb1fdf0c4b4bfa3d5da07f991f9be822 Author: Caleb Hulbert Date: Thu Jun 12 10:21:19 2025 -0400 test: make test data smaller commit 6c4156c1092337b47a626790bf8082615d7b0fa2 Author: Caleb Hulbert Date: Thu Jun 12 09:38:43 2025 -0400 feat: more block as shard logic commit 214466087b3a735fc1de492798236c9bffea3629 Author: Caleb Hulbert Date: Tue Jun 10 16:17:46 2025 -0400 feat: support writing blocks through shards on non-sharded datasets commit 2acc2816a88377f6518b20ad24760bd7cde73d6c Author: Caleb Hulbert Date: Mon Jun 2 14:23:41 2025 -0400 refactor: migrate IOExceptions to N5IOExceptions for ReadData and related logic commit 5d13af6d8b9f757da34dc3c1fe04cedac38533b2 Author: Caleb Hulbert Date: Mon Jun 2 11:01:12 2025 -0400 refactor: extract Pattern matching to be reused for attribute path normalization commit d7c9d35e64f44546cbc740eee0fe06c87cbfdfd3 Author: John Bogovic Date: Tue May 27 13:38:17 2025 -0400 test: writing / reading of invalid blocks commit ceaf5c30848df5cc5ae225491f9cadadce7b3b53 Author: Caleb Hulbert Date: Tue May 27 13:29:20 2025 -0400 chore: remove duplicate/outdated license header commit 4f29c338c6bd276b5e7812d796104ff01ac3f729 Author: Caleb Hulbert Date: Tue May 27 11:53:07 2025 -0400 fix: getArrayCodec not ShardingCodec commit c8c433585c209a76a2e1e187bd5ab30d4e3faa79 Author: Caleb Hulbert Date: Tue May 27 11:51:54 2025 -0400 fix: NameConfig for Bzip2 commit db0e1a3cdfc249dbf6261d8673f4ed0df0c67a3c Author: Caleb Hulbert Date: Tue May 27 11:51:39 2025 -0400 feat: Shard as a the base case for writing commit c3a87cbb906f7303f1b18b099c1d3e6a000fa7d8 Author: Caleb Hulbert Date: Tue May 27 11:50:10 2025 -0400 refactor: remove deprecated api methods, some cleanup commit 19af7cc5411c3a5da1b929e87f49eb4686b9826d Author: John Bogovic Date: Fri May 23 17:16:27 2025 -0400 test: add CodecTests * testing ConcatenatedBytesCodec commit 2243a4f5277a0c24e2f949fae88de25133fdbf8c Author: John Bogovic Date: Fri May 23 17:13:03 2025 -0400 perf: FileSystemKva's ReadData uses lockForReading commit b9130deb09368268140ad3189f6e0768d10d188e Author: John Bogovic Date: Fri May 23 17:11:51 2025 -0400 refactor: hide KeyValueAccessSplittableReadData commit 65345d9f1e0ae308c10039eb95102081c6d529d7 Author: John Bogovic Date: Fri May 23 17:10:48 2025 -0400 doc: SplittableReadData.slice commit d8504eb922b8e8f65e2a00dddf9e13c7d872147b Author: Caleb Hulbert Date: Fri May 23 16:59:40 2025 -0400 refactor: branch sharding logic on ShardingCodec commit 15737489dd17dd631801b48240fd45be27159856 Author: Caleb Hulbert Date: Fri May 23 16:59:16 2025 -0400 fix: N5BlockCodec adds header size to encoded size commit a56c80b1ef930a50f04ef56e623374225a91bd22 Author: John Bogovic Date: Fri May 23 14:38:53 2025 -0400 wip: refactor KvaSplittableReadData * add abstract KeyValueAccessSplittableReadData * ReadData.length does now throw IOException commit 3242dc716e63f950e657c8ff7f866f524f7f99b8 Author: Caleb Hulbert Date: Fri May 23 10:49:22 2025 -0400 fix(test): index should have RawBytesCodec commit 521b6aad987ace8866eab06af4074bd5d4318c36 Author: Caleb Hulbert Date: Fri May 23 10:49:05 2025 -0400 feat,BREAKING: wrap IO as N5IO and rethrow; don't throw IOException in KVA for size commit 6edda5f01d7a9cd060543db3de049fd2e8b6bcac Author: John Bogovic Date: Fri May 23 10:35:57 2025 -0400 test: FileSplittableReadData knows its length commit 11cc2e2901968eb4269741beaf6d5f32d9968068 Author: John Bogovic Date: Fri May 23 10:21:22 2025 -0400 rm ReadDataBenchmarksKvaReadFully commit 25b2f5c292f2fa876cb6cb0b2f44eac11a0dcd33 Merge: 3486b0f c274213 Author: John Bogovic Date: Fri May 23 10:20:33 2025 -0400 Merge branch 'wip/codecsReadData' of github.com:saalfeldlab/n5 into wip/codecsReadData commit 3486b0f47ee425a70860044c4ce88d9b326ebb26 Author: John Bogovic Date: Fri May 23 10:20:29 2025 -0400 wip: FileSplittableReadData now wraps a kva commit c27421382b60b4e4cccc19596acf69019556845e Author: Caleb Hulbert Date: Fri May 23 10:13:52 2025 -0400 fix: write ShardIndex before closing output stream fix(test): shardIndex writeRead commit 1c5f0c1165010b5febac7181c651b91b6a2f5dae Author: Caleb Hulbert Date: Fri May 23 09:37:48 2025 -0400 chore: feedback commit 2cc81aab194c64dd9455efc675b5e1bca8da0f14 Author: Caleb Hulbert Date: Fri May 23 09:37:33 2025 -0400 refactor(test): for sharding commit b56a3945594ebb24e495bf551c521330ed6e41cd Author: Caleb Hulbert Date: Fri May 23 09:37:13 2025 -0400 refactor: Shards should use ReadData, not previous SplitData implementation refactor: reduce scope of shard capability. Only support full Shard writes (InMemoryShard) and partial shard reads (VirtualShard) commit f3949376f6b4f4d7692978edbeb68600caaa8eb6 Author: Caleb Hulbert Date: Fri May 23 09:34:26 2025 -0400 feat: ReadDataSplittableReadData and KVA#createReadData; commit 3d27db9903b109508ff36e632429227494a58e8e Author: Caleb Hulbert Date: Fri May 23 09:33:19 2025 -0400 refactor: rename to ArrayCodec#initialize commit 967df11f8ee859eb1b76b4865d55354fa7d7ebe6 Merge: d8aedbd 02e0488 Author: John Bogovic Date: Thu May 22 11:09:05 2025 -0400 Merge branch 'wip/codecsReadData' of github.com:saalfeldlab/n5 into wip/codecsReadData # Conflicts: # src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java commit d8aedbdf1277fd87232f1580a85dc707726d80cc Author: John Bogovic Date: Thu May 22 10:22:28 2025 -0400 perf/style: http kva, implement range request * remove duplicate license header * add constants commit 02e0488f7e51f1fd802d54cb11d175da6ee2f5ac Merge: 0023d4e 424c101 Author: Caleb Hulbert Date: Thu May 22 10:05:56 2025 -0400 Merge remote-tracking branch 'origin/wip/codecsReadData-splittable' into wip/codecsReadData commit 0023d4e5ade62cf7e64f63cec75b6628ae89a28d Author: Caleb Hulbert Date: Thu May 22 09:49:33 2025 -0400 fix(test): add N5BlockCodec to shard test commit 424c1018550f6b4ea3f5ef748fd97986f160f005 Author: John Bogovic Date: Wed May 21 21:04:21 2025 -0400 perf: use range request for http KeyValueAccess commit 24817915bc14d4159628521bda1378037097c708 Author: John Bogovic Date: Wed May 21 16:27:04 2025 -0400 test: more read data benchmarks commit fecd95db597f84dadfc7ae9dbca326753a8fd18e Author: John Bogovic Date: Wed May 21 16:26:26 2025 -0400 test: more ReadDataTests commit 1564aa572ba4a45721aab1205c63b970a2aabd34 Author: John Bogovic Date: Wed May 21 16:25:55 2025 -0400 feat: add FileSplittableReadData commit c7a119ec632360db9a82954d120aa297a8a3fefb Author: John Bogovic Date: Wed May 21 13:42:39 2025 -0400 test: fix ShardProperties tests commit 20b5c003e92df59aab50cfdbe49f1777e67714bd Author: John Bogovic Date: Tue May 20 16:55:55 2025 -0400 feat: add splittable read data * add a test commit 38017fff277c9d11abfb9ec2ceb4773562df34d4 Author: John Bogovic Date: Tue May 20 16:34:23 2025 -0400 fix: AbstractInputStreamReadData close stream commit 31e0a9a3c92448731a5ed14a16864ea39f6d0707 Author: John Bogovic Date: Tue May 20 16:30:27 2025 -0400 test: add ReadData benchmark commit 1ba662d72b29c688c4c57cb45d10946c11ecd6a6 Author: Caleb Hulbert Date: Tue May 20 14:34:50 2025 -0400 chore: remove duplicate license headers commit 95e5aff2a020d6de752f87f1a2072f3f783a5749 Merge: c1485a9 9502d93 Author: Caleb Hulbert Date: Tue May 20 14:23:12 2025 -0400 Merge branch 'master' into wip/codecsReadData commit c1485a929f4cb5d6ffbb2e1576cc81d6fc50e6b3 Merge: 0dcade4 b4ca260 Author: Caleb Hulbert Date: Mon May 12 14:24:35 2025 -0400 Merge branch 'codecsReadData' into zarr3-readdata commit b4ca260c4df3c405620ca994cdee161122a9bc99 Author: Caleb Hulbert Date: Mon May 12 13:56:23 2025 -0400 tmp commit 0dcade45bf3a53466aca68c8a1850c1881171e9d Author: Caleb Hulbert Date: Tue Apr 8 14:58:13 2025 -0400 feat: add ZarrStringDataCodec support commit d4ebfefe2be06d87e92efdd4739f5d04c14498b2 Author: Caleb Hulbert Date: Tue Apr 8 14:57:50 2025 -0400 refactor: rename encodeBlockHeader -> createBlockHeader commit 2e056286a27668056a485d058727eb1bacbf68b8 Author: Caleb Hulbert Date: Tue Apr 8 14:57:30 2025 -0400 refactor: dont expose N5Codecs internals commit ab8b83b3e41512dce7a41bee0340dbf7b4169da4 Author: Caleb Hulbert Date: Tue Apr 8 14:56:34 2025 -0400 refactor: remove currently unused N5BlockCodec. Something like this may be needed when multiple codecs are supported commit 70ec77710cc1ec9453efa5c98611ae9a64060948 Author: Caleb Hulbert Date: Tue Apr 8 14:55:32 2025 -0400 revert: keep protected constructor with DataBlockCodec parameter refactor: inline createDataBlockCodec from constructor params commit 0f8b2c59e2dcd60f65593f380b61813e0f9ae151 Author: Caleb Hulbert Date: Tue Apr 8 14:53:32 2025 -0400 doc: retain javadoc from before refactor commit 0db21be4fc6b4f325cf43d9cd93cc1b4ce36daec Author: Caleb Hulbert Date: Tue Apr 8 14:52:52 2025 -0400 revert: add back createDataBlock logic in DataType commit 7b713cd7f93a76a9536799ae44852433b5655d0c Author: Caleb Hulbert Date: Wed Mar 5 16:59:15 2025 -0500 refactor: DatasetAttributes responsible for DataBlockCodec creation N5BlockCodec uses dataType (and potentially other DatasetAttributes to wrap the desired DataBlockCodec commit ae85a55844d8090ce0e17b26fc0f65d22169ef4b Author: Caleb Hulbert Date: Wed Mar 5 16:55:21 2025 -0500 refactor: add AbstractDataBlock to extract shared logic between Default/String/Object blocks commit f948f578479501e0e8a45e91904427428a7dc864 Author: Caleb Hulbert Date: Wed Mar 5 16:54:56 2025 -0500 feat: Add StringDataCodec, ObjectDataCodec, StringDataBlockCodec, ObjectDataBlockCodec DataCodecs now creat the access object and return it during `deserialize` commit b03e897db8d2d89bc480c8595eaef93225f5b2cc Author: Caleb Hulbert Date: Wed Mar 5 16:48:34 2025 -0500 refactor: move DataBlockFactory/DataBlockCodecFactory to N5Codecs commit 46c7eab42fc70a475fc9bc2880aa6f2d00b22995 Author: Caleb Hulbert Date: Wed Mar 5 16:40:52 2025 -0500 refactor: don't pass `decodedLength` to ReadData commit 2648d380f070320cff8f3083565d5ccd4b0e9076 Author: tpietzsch Date: Wed Feb 26 23:01:50 2025 +0100 Remove unnecessary flush()s, instead close OutputStream where it is constructed commit 4db22cfd276944c6b0003110e9b82961a2f6e9b8 Author: tpietzsch Date: Wed Feb 26 22:13:31 2025 +0100 rename ChunkHeader to BlockHeader commit 31f351e14274209157815e80b7dfed2a747b71f7 Author: tpietzsch Date: Wed Feb 19 10:47:19 2025 +0100 fix javadoc error commit 1154bf9f5510ecce82e89509d1fe2fb837df1de2 Author: tpietzsch Date: Wed Feb 19 10:35:29 2025 +0100 Remove SplittableReadData interface commit f191b7d62d4759ccd8ab83b118594dde30db1bb5 Author: tpietzsch Date: Wed Feb 19 10:27:12 2025 +0100 refactor, add ReadData.materialize() This is in preparation for moving SplittableReadData into a separate PR commit d8b7c1f4596c80fbe225ea7a9cb5aa35f8f85d49 Author: tpietzsch Date: Sun Feb 16 16:29:02 2025 +0100 Let DatasetAttributes provide the DataBlockCodec avoids the Compression argument to encode/decode methods commit ffea7a8842442050404b4b3a558d514e3077462e Author: tpietzsch Date: Sun Feb 16 16:04:33 2025 +0100 refactor commit 4290e0b414df26ae694cf82448d68c9f1c48e9eb Author: tpietzsch Date: Sun Feb 16 16:03:54 2025 +0100 refactor commit fa88b72515272a146d4c897fa5282a5069a808de Author: tpietzsch Date: Sun Feb 16 12:02:19 2025 +0100 typo commit 72986216b5e699c0956e27b38c37af1d770d62c6 Author: tpietzsch Date: Sun Feb 16 11:35:34 2025 +0100 convenience DataCodec methods to get codec with approprioate endianness commit 2a0cd24854eba5552163bf7fe9b9e22c8426f6a7 Author: tpietzsch Date: Sat Feb 15 13:09:05 2025 +0100 refactor commit fedfbc7f4977099a883e7246eb85018c0883b019 Author: tpietzsch Date: Sat Feb 15 00:27:09 2025 +0100 refactor commit a4fb38e54d75f7270f00865b78f085cdd9822e66 Author: tpietzsch Date: Sat Feb 15 00:15:03 2025 +0100 use only array type in DataBlockCodec generics commit d8d07195b88948c1e8114574020885c7e736910f Author: tpietzsch Date: Fri Feb 14 23:17:26 2025 +0100 Use ProxyOutputStream. FilterOutputStream is slow commit 1be51a5d2321b26bbd6187ce6b5c76571a1b0b35 Author: tpietzsch Date: Fri Feb 14 22:55:43 2025 +0100 WIP use Codecs, remove serialization methods from DataBlock commit 6cd52d0771b1bc5cd25919cc969781eb73c75eea Author: tpietzsch Date: Fri Feb 14 22:12:29 2025 +0100 WIP Codecs commit e52d0fc743e516106fbbedb5127d1ec3c62c53ba Author: tpietzsch Date: Fri Feb 14 17:46:49 2025 +0100 WIP Codecs commit 8683dfa3e775e119149813347e70ab2646ddf0c9 Author: tpietzsch Date: Fri Feb 14 17:03:51 2025 +0100 WIP Codecs commit 0d66ee69dd7866e379ff3eef1ab0df1953d60cde Author: tpietzsch Date: Fri Feb 14 17:03:15 2025 +0100 Add LazyReadData and OutputStreamWriter When data is requested from the LazyReadData, the LazyReadData will ask its OutputStreamWriter to write the data to a ByteArrayOutputStream. When the LazyReadData itself is written to an OutputStream, it will pass that OutputStream to its OutputStreamWriter (without loading the data into a byte[] array first). commit 2bcdab883ed583d9c5ba67474184e218427c0ccf Author: tpietzsch Date: Thu Feb 13 09:37:44 2025 +0100 idea commit 018d31186f269f1d7efc0d5701e3ef96d1e65928 Author: tpietzsch Date: Thu Feb 13 13:32:22 2025 +0100 remove BytesCodec interface and put encode/decode into Compression for now commit a2e9596706265abf2a40d73337f1539ef2409427 Author: tpietzsch Date: Thu Feb 13 13:14:17 2025 +0100 ReadData.encode/decode methods forwarding to BytesCodec commit b70e6b5ba1b2a91e857c61ee07811f45777b12f4 Author: tpietzsch Date: Thu Feb 13 13:11:33 2025 +0100 Use OutputStream wrapper to intercept close() in EncodedReadData We still need a custom interface "OutputStreamOperator" because we want to throw IOException and UnaryOperator::apply doesn't. commit d9f42220f98462440829ee6eb318744de6f78fa7 Author: tpietzsch Date: Thu Feb 13 10:18:26 2025 +0100 clean up commit 8b097b5e7dda6db10992bef2daa32d8e87c84c77 Author: tpietzsch Date: Mon Feb 3 17:46:20 2025 +0100 Add ReadData method toByteBuffer() commit 4fd4723318f78195d10689ff35d9180aa1ce6fd9 Author: tpietzsch Date: Mon Feb 3 17:53:31 2025 +0100 fix javadoc error commit 24dc39398fdcb218f3f68d25b9ca1cfc302092d9 Author: Caleb Hulbert Date: Wed Feb 5 10:11:33 2025 -0500 test: don't override with zarr (oops) commit 231aacdffd6e9335734cca7fe8cbfed688297b0b Author: Caleb Hulbert Date: Wed Feb 5 10:10:42 2025 -0500 test: easier to local debug commit 5b9c4a371ff83921f7f3ef48977cac00cf4dbd49 Author: Caleb Hulbert Date: Wed Feb 5 10:10:22 2025 -0500 fix: readblock with split data commit f854ac64f3c53e1a767e3b87dc306d1fa397cf1f Author: Caleb Hulbert Date: Tue Feb 4 16:53:01 2025 -0500 chore: rebase cleanup commit 4c0e0fe8171c9a8285202a6dff528f5302f69667 Author: Caleb Hulbert Date: Mon Feb 3 14:20:32 2025 -0500 fix: revert some changes that broke split data for ShardTest commit 7970281a6de4237f88cfca05d15ac193dca76b0e Author: Caleb Hulbert Date: Fri Jan 24 16:28:53 2025 -0500 refactor: more SplitData implementation commit 765a7b428502a992c32a98e38ad28d94f5390be4 Author: Caleb Hulbert Date: Fri Jan 24 15:27:27 2025 -0500 feat(wip): initial SplitableData implementation commit 0d1eb525c8065f77fa73280f25629e2ff73a0d25 Author: Caleb Hulbert Date: Tue Feb 4 17:10:40 2025 -0500 refactor: more from refactorShard branch commit 3bf8fe830747aa4f96e53ae50de374b25649133d Author: Caleb Hulbert Date: Tue Feb 4 16:02:20 2025 -0500 refactor: large codecs/shards implementation refactor commit 3b54467a253f2ec255ea74248bbc10fc5cdb1570 Author: tpietzsch Date: Mon Feb 3 13:42:06 2025 +0100 Remove DataBlock de/serialize() methods Maybe we'll rename readData()/writeData() later, the point is to remove the ByteBuffer methods from the API commit decdafdada750047452e65027da9fa2ccb1219bc Author: tpietzsch Date: Mon Feb 3 12:32:09 2025 +0100 Remove old Compression methods. Everything goes through ReadData commit 5ff62a98e7f93d31306715ee675e5067e862ab88 Author: tpietzsch Date: Sun Feb 2 21:36:12 2025 +0100 javacod commit 89a6bfd805be826718a898f7fbdad3269c789cbd Author: tpietzsch Date: Sun Feb 2 20:55:50 2025 +0100 Clean up commit a43bf40a8378526bfa7bad3921cc5d5cdd1ca1cd Author: tpietzsch Date: Sun Feb 2 20:31:13 2025 +0100 Remove deprecated BlockWriter/BlockReader interfaces commit 80fa6a87827381ec59f007701eb3e26715557cff Author: tpietzsch Date: Sun Feb 2 20:21:20 2025 +0100 Hide ReadData implementations, use static ReadData factory methods instead commit 47578e3ddd2c70566f20dfec4b910093daea7183 Author: tpietzsch Date: Sat Feb 1 22:20:51 2025 +0100 Add decode(ReadData readData) variant that knows the length of the decoded data commit 4b6ee4ff12962a78d22b7fc4a073b498c1808d94 Author: tpietzsch Date: Sat Feb 1 22:08:46 2025 +0100 WIP ReadData.decode(Codec) commit 4c348baae207f7b91ab3357ad52f9801bed3b69b Author: tpietzsch Date: Fri Jan 31 22:38:22 2025 +0100 Lean more on ReadData * Move ReadData etc to separate classes * Add ReadData.writeTo(OutputStream) * Add EncodedReadData that wraps a ReadData and an OutputStreamEncoder * Compression (BytesCodec) can encode ReadData. (This might happen immediately or later when the ReadData is written to OutputStream). commit fbee9db8091ad4fdf08bdcfa387509feb864234b Author: tpietzsch Date: Thu Jan 30 08:44:33 2025 +0100 Add explicit commons-io dependency commit 5c43f5bc1897729f1a74fbb327fb230beb112457 Author: tpietzsch Date: Thu Jan 30 08:44:01 2025 +0100 switch from byte[] to ByteBuffer for DataBlock.de/serialize() commit 705ffb329e1644198948bfde75d95d1ae070d5ad Author: tpietzsch Date: Wed Jan 29 13:58:09 2025 -0500 fix javadoc error commit 3df2b0b4f6c650a82da3aba78898abb1452752ec Author: tpietzsch Date: Mon Jan 27 21:23:59 2025 -0500 Clean up commit 190a8ce38805dff0a084395be693355afd35a28f Author: tpietzsch Date: Mon Jan 27 21:19:11 2025 -0500 Clean up commit 8732a0b5a31189f11f47736120fffeb93785e1fe Author: tpietzsch Date: Mon Jan 27 21:10:05 2025 -0500 Add ByteOrder to DataBlock.writeData() commit e47ba4c50d1614b33e9eab39fb6f63875be9a903 Author: tpietzsch Date: Mon Jan 27 16:10:03 2025 -0500 WIP revise StringDataBlock revert to holding serialized and actual data. otherwise we can't be compatible with existing N5 datasets. commit 8cffeed489cd75c8d1bd5a6f59eb1b5d4c2df07d Author: tpietzsch Date: Mon Jan 27 15:21:48 2025 -0500 WIP revise StringDataBlock (doesn't work yet) commit 701d4218cdeb1d30f0474466dc4460d3a191ebbf Author: tpietzsch Date: Mon Jan 27 14:55:50 2025 -0500 WIP use Splittable.ReadData in DefaultBlockReader commit 4fca725c2a81ec04a51e0778663e8e3e1f028d62 Author: tpietzsch Date: Sun Jan 26 19:52:16 2025 -0500 WIP DataBlock.writeData(OutputStream) commit 069f3e2755c1adad561b61223452a5143d85482f Author: tpietzsch Date: Sat Jan 25 10:59:29 2025 -0500 WIP speed up DataBlock.readData(InputStream), and clean up commit 8811f1d570817fbd2b1e35ae81b0e065bbece606 Author: tpietzsch Date: Sat Jan 25 10:58:37 2025 -0500 WIP clean up Compression interface commit a18d137e179674e22d542d038bde571389d64b92 Author: tpietzsch Date: Fri Jan 24 21:22:58 2025 -0500 WIP SplittableReadData commit 22ece34ecb5cdd98dc2f4fb5ca0fb2c1796e49e2 Author: tpietzsch Date: Fri Jan 24 20:23:17 2025 -0500 WIP DataBlock readData variants commit 953715bcf55ef376dcc8995ce9b9bdd57937324d Author: tpietzsch Date: Fri Jan 24 20:22:33 2025 -0500 WIP: SplittableReadData commit b94ba8b10b7c8024380987043ac5c6d1780d01ae Author: John Bogovic Date: Tue Sep 3 16:48:17 2024 -0400 feat: DataBlock methods to read/write directly from DataInput/Output commit eba1fd750322e1d8b0072e6b2a5297dfbe9cb00b Author: tpietzsch Date: Fri Jan 24 10:11:00 2025 -0500 wip commit 63f51ad6c08461fa0912896e18c3871493671688 Author: tpietzsch Date: Wed Jan 22 18:29:53 2025 -0500 cleanup commit e8dde834dcd6234069df70951aeda5f64c19b8d2 Author: tpietzsch Date: Wed Jan 22 18:25:57 2025 -0500 bugfix commit 17f197612afb58b1fc89aefbab63f4bd4308f2f5 Author: tpietzsch Date: Wed Jan 22 17:17:15 2025 -0500 WIP encode/decode implementations commit ad30b381c74ddefa9fcfd33206ca1d658961c97c Author: tpietzsch Date: Wed Jan 22 15:19:15 2025 -0500 WIP Compression / Codec API commit 08940ee6668863a906f72a8a403262e547bde912 Author: tpietzsch Date: Wed Jan 22 15:17:14 2025 -0500 Add byte[] DataType.createSerializeArray(numElements) commit 27b6057b51bba7b1ee1830931d669e1e030d13cc Author: tpietzsch Date: Wed Jan 22 10:07:41 2025 -0500 Fix benchmark image URL commit 9151207e2604339650e09c82e2c10e7a17327244 Author: tpietzsch Date: Wed Jan 22 10:07:03 2025 -0500 WIP: remove @Override annotations for ByteBuffer methods to be able to turn those methods on/off in the DataBlock interface without causing compile errors in the implementations. commit 3401fcf7cb78874ff37149342a35b93e8cb5fe59 Author: tpietzsch Date: Wed Jan 22 09:40:30 2025 -0500 remove redundant modifiers commit 9518d00b8a54bf1bc014f96137d991901c908be3 Author: tpietzsch Date: Wed Jan 22 18:11:47 2025 -0500 Add N5ReadBenchmark commit ce50fa9a746179e4f741ef0c62939586f44fcc82 Author: John Bogovic Date: Fri Jan 24 13:12:19 2025 -0500 test: add a test for nested sharding codecs * but ignore it for now commit a74d3437d0b5520076e4e2629de4dc50a11db787 Author: John Bogovic Date: Tue Jan 21 10:08:02 2025 -0500 feat: add ShardIndex.isEmpty * use it to return early for getBlocks commit 9224215e811bf9f0f637291d86d68ace92a1353a Author: John Bogovic Date: Tue Jan 21 09:31:28 2025 -0500 wip/feat: N5Reader.readShard commit f67943dcbb804227ca2d1d2ae441fa03dc7e1063 Author: John Bogovic Date: Fri Jan 17 13:59:16 2025 -0500 refactor: InMemoryShard, read/writeBlocks * Using Position class * using ShardParameters.groupBlocks helper method commit 709730868111ffcdc584efaffe5379836c5d49a0 Author: John Bogovic Date: Fri Jan 17 13:57:17 2025 -0500 wip: rm unused flatIndex in Shard * methods in GridIterator replaces this commit 0e353e42fc2c6175561ead61258197580fd875c0 Author: John Bogovic Date: Fri Jan 17 13:54:18 2025 -0500 feat: ShardParameters methods * shardsPerImage, blocksPerImage * grouping DataBlocks by shard postion * stream of block positions ordered by shard commit 43bc1e04eaa6ff8253439032322c18718da6267f Author: John Bogovic Date: Fri Jan 17 13:51:04 2025 -0500 feat: add positionToIndex static methods in GridIterator commit 1a44168191b161def0f6a73d262608b9dab97c0f Author: John Bogovic Date: Fri Jan 17 13:48:40 2025 -0500 feat: add Position * so that we can index by position * primitive long arrays are not great as keys for maps commit 367cadbe712473cb901ae984857216eb454bb73f Author: John Bogovic Date: Wed Jan 15 16:29:23 2025 -0500 fix: ShardIndex.getOffsetIndex commit aca03d43e959c8fa627a9478cf6f7becaaf8540c Author: John Bogovic Date: Tue Jan 14 09:02:53 2025 -0500 demo: BlockIterators commit 6fac5f236102c050222e3cecd47dd8e6c4334bd1 Author: John Bogovic Date: Mon Jan 13 16:12:18 2025 -0500 wip: minor change to writeBlocks, implement readBlocks * where readBlocks batches when possible commit 52aaa2ab1c4fa38ba625f72da74eae542d382907 Author: John Bogovic Date: Mon Jan 13 15:02:22 2025 -0500 feat: InMemoryShard add new write methods commit 1c92d14a2149e1084947c234e264d31095d86809 Author: John Bogovic Date: Mon Jan 13 13:14:02 2025 -0500 feat: add getBlocks(int[] blockIndexes) commit cd54053d42c4f12b0fa897182a508da1e38431ca Author: John Bogovic Date: Mon Jan 13 13:09:45 2025 -0500 fix: EMPTY_INDEX_NBYTES now in ShardIndex commit a791244b42d162a8ffd04ed4b8ca19286071e5f0 Author: John Bogovic Date: Mon Jan 13 11:34:39 2025 -0500 feat: Codec add composition helpers commit 7dde5bb0633d17b1c32af7ef9585742345eeec8f Author: John Bogovic Date: Fri Jan 10 11:36:29 2025 -0500 chore: rm unused method commit dd962e0f61d28108f1cf3a7e61eaf29480493944 Merge: 0affc94 539959f Author: John Bogovic Date: Fri Jan 10 11:32:54 2025 -0500 Merge branch 'wip/codecsShards' of github.com:saalfeldlab/n5 into wip/codecsShards commit 0affc94d5302be443e85cc08127684e364c96195 Author: John Bogovic Date: Fri Jan 10 11:31:54 2025 -0500 refactor: EMPTY_INDEX_NBYTES to ShardIndex commit 539959f23b90bd87924055023abdd8e3d71badb6 Author: John Bogovic Date: Fri Jan 10 11:31:54 2025 -0500 refactor: EMPTY_INDEX_NBYTES to ShardIndex commit edb61a5c5b2257a6209d1ce514b3f695e2ed9db4 Author: John Bogovic Date: Fri Jan 10 11:20:33 2025 -0500 fix/test: clone gridPosition commit 9a59de2f52abd7280d452fb2859a758b6e2408b3 Author: John Bogovic Date: Fri Jan 10 11:18:46 2025 -0500 chore: rm unused ShardReader/Writer classes commit 707addf6e9127d91d339176fe4ce7d250b7ac9c3 Author: Caleb Hulbert Date: Fri Jan 10 09:51:09 2025 -0500 feat: DataBlockIterator skips missing blocks in Shard commit 55682ee04d271de70f5d0803f753f6f5f1833584 Author: John Bogovic Date: Thu Jan 9 17:01:10 2025 -0500 fix: GridIterator iteration order commit b6a5b4f307b3e624f6d350b5c11f8eeceb3b7ffa Author: John Bogovic Date: Thu Jan 9 17:00:53 2025 -0500 perf: VirtualShard smart override of getBlocks commit 84493829977de0cd7786aef8f8b339028e7a0e81 Author: John Bogovic Date: Thu Jan 9 17:00:40 2025 -0500 feat: add Shard.getNumBlocks commit 334e46c64dc60053fddc6a420ab176b49069c329 Author: John Bogovic Date: Thu Jan 9 16:57:48 2025 -0500 feat: ShardIndex get properties by block index commit b17cb1fbbb9a037e5cdaac33b4bd708e010def85 Author: John Bogovic Date: Wed Jan 8 21:45:05 2025 -0500 feat/test: add block position iterator for shard * add ShardProperties test commit 06477551d7b7f49b9d394874ddd5aebcfc921fe9 Author: John Bogovic Date: Wed Jan 8 16:55:15 2025 -0500 feat: toward direct reading of InMemoryShard commit 6e3cbe524e9b4af81afe29850c012ba46a5011db Author: John Bogovic Date: Wed Jan 8 16:54:46 2025 -0500 test: ShardIndexTest commit 116185992a65e05063d662f83c9d8d6a78c941d9 Merge: 52751f5 d4dcbe8 Author: Caleb Hulbert Date: Wed Jan 8 16:11:41 2025 -0500 feat: merge wip ShardParameters interface commit 52751f5a22e4ccbe6b77a114d282f67999278fcf Author: Caleb Hulbert Date: Wed Jan 8 15:39:34 2025 -0500 fix: Shard as Iterator> commit b2b3d2ba2e5dff263556b857fe0ef5514bb58fa0 Author: Caleb Hulbert Date: Wed Jan 8 11:59:36 2025 -0500 refactor: some signatures commit d4dcbe8ae72f23374a9b79de848a82b3ab01226f Author: Caleb Hulbert Date: Wed Jan 8 11:57:25 2025 -0500 refactor: remove generic from ShardParameter commit 52f762eeb9bbef3f57f95f1545c363cd22ca0d90 Author: Caleb Hulbert Date: Wed Jan 8 10:05:48 2025 -0500 feat: writeBlocks respects existing blocks in a given shard if not overwriting those blocks explicitly commit da9b9eddb19549b45881736e1c5af1f530350a93 Author: John Bogovic Date: Tue Jan 7 09:54:27 2025 -0500 feat: add getByteOrder method for ArrayCodecs commit 458295cd7e1bc7b783d3d91bf057e8aca3f6a9b7 Author: John Bogovic Date: Tue Jan 7 09:54:08 2025 -0500 refactor: createIndex now a default method in ShardParameters commit b0092d315567f9e477203376f599e04a4b3e2dcc Author: John Bogovic Date: Tue Jan 7 09:53:31 2025 -0500 fix: null compression should result in empty byteCodecs list commit eb9fbc1b4203245599e8e532843d0b5c432960be Author: John Bogovic Date: Mon Jan 6 16:29:41 2025 -0500 feat/refactor: add BlockParameters and ShardParameters interfaces * will make zarr implementation less repetative commit 5badbb7110e14ffcff999d577e1f62058f5dbfa3 Author: Caleb Hulbert Date: Fri Jan 3 16:28:15 2025 -0500 feat(test): wip shard writeBlocks refactor(test): use temp writer via N5FSTest commit c3c3ceb20dfa409ff6aae340d321c787caa52571 Author: Caleb Hulbert Date: Fri Jan 3 16:27:37 2025 -0500 fix: index offset calculation commit 20a9677dbfd0f7d5f5bcce13aedc9c5fbf56a1c3 Author: Caleb Hulbert Date: Fri Jan 3 16:25:50 2025 -0500 feat: writeBlocks aggregate shard commit 367b987c0406da4af4d27cea1633a3f6b26062fe Author: Caleb Hulbert Date: Fri Jan 3 13:53:11 2025 -0500 refactor: remove `DatasetAttributes#getShardedAttributes()` unnecessary as you can do the same with an instance check commit cc5b8c531c2ff1225ea657c005abbaf18569eea2 Author: John Bogovic Date: Fri Jan 3 16:33:39 2025 -0500 feat: ShardedDatasetAttributes validate shard/block size on construction commit f98b11a80b9ee44d45f359cd411cc72b46f4bb52 Author: John Bogovic Date: Fri Jan 3 11:39:04 2025 -0500 test: BytesTest operates on n5 container commit 0e045cd05deac296b331e22144da044031e21e6c Author: John Bogovic Date: Fri Jan 3 10:55:38 2025 -0500 feat: serialize shardSize in DatasetAttributes for n5 commit 1ced5708260110a7699031772c09dcce8e26e556 Author: Caleb Hulbert Date: Thu Jan 2 17:03:53 2025 -0500 feat: writeShardEndStream commit b62dc00db0805ad4be3297eac94eb1fbbb7ef3ad Author: John Bogovic Date: Thu Jan 2 16:31:59 2025 -0500 fix/test: add writeShardTest * fix ShardIndexBuilder commit 1c9e6018ae9de65e40fed328d03519f75802e087 Author: Caleb Hulbert Date: Thu Jan 2 15:58:57 2025 -0500 fix(test): failing on github actions commit f294898599c242da35f1c1d949eb7d9b6e67124b Merge: bc5d103 bdb8cde Author: Caleb Hulbert Date: Thu Jan 2 15:52:04 2025 -0500 Merge branch 'master' into wip/codecsShards commit bc5d103662c0d993e78bb589db34ae94c866d400 Author: John Bogovic Date: Thu Jan 2 15:51:06 2025 -0500 fix: be quiet commit d4142c5fe1df9091342d8e5e14a32fb9c69669ae Author: John Bogovic Date: Thu Jan 2 15:43:09 2025 -0500 chore: stop using deprecated BoundedInputStream constructor commit 9183d11f72971130cf526d9b61c03b40393f6bfb Author: John Bogovic Date: Thu Jan 2 15:42:50 2025 -0500 feat(wip): toward an implementation of writeShard commit 275aaa61d4a45f1c1d7978de645ad9a369f4b14a Author: Caleb Hulbert Date: Thu Jan 2 15:34:05 2025 -0500 feat(test): parameterize ShardDemo read/write test; add new test commit 2c753a1a0b191fc1221693055eb7d65b130a0589 Author: Caleb Hulbert Date: Thu Jan 2 15:01:10 2025 -0500 Revert "fix: BlockWriter should not close stream" This reverts commit bec29965b77a6981c42848c8eb578ada7883f904. commit bec29965b77a6981c42848c8eb578ada7883f904 Author: John Bogovic Date: Mon Dec 23 11:44:26 2024 -0500 fix: BlockWriter should not close stream * rather, should be closed where it is opened * this change enables the stream to be re-used by multiple block writers, e.g. during sharding commit 1b672ded40616286244bd772ae6d49a19cc9bc9b Author: John Bogovic Date: Mon Dec 23 09:04:27 2024 -0500 fix: gzip make uzeZlib parameter optional * e.g. not specified in zarrs written by tensorstore commit edbdef6cda0a781e601d975ce72442b3583426f8 Author: John Bogovic Date: Mon Dec 23 08:34:56 2024 -0500 fix: make removeAttribute methods' behavior more consistent * this change allows re-use of this method by zarr v3 commit 4f6b49bb9624c804f97c77f7cfcba2f3a1b158b0 Merge: 362c74d f46aa52 Author: John Bogovic Date: Fri Dec 20 10:28:05 2024 -0500 Merge branch 'wip/codecsShards' of github.com:saalfeldlab/n5 into wip/codecsShards commit 362c74d21e944f26eff70418c9b710a43b4888f7 Author: John Bogovic Date: Fri Dec 20 10:27:51 2024 -0500 perf: initialize cache only if using it commit f46aa524c230863194c3423235c31a4cc90f5f3b Author: John Bogovic Date: Wed Dec 18 13:20:21 2024 -0500 chore: bump pom-scijva to 40.0.0 commit b270ece22224cbe326f0ea1a7f0140a05b4b4d9f Author: John Bogovic Date: Wed Dec 18 13:17:36 2024 -0500 perf: override writeData * (Short/Float/Double)DataBlock commit 3229fdd6c1bcae0984b6f702b5145c5115dc5ad8 Author: John Bogovic Date: Mon Dec 16 16:07:51 2024 -0500 fix: ShardedDatasetAttributes.getBlockPosition commit 19b8618251284654752286f68221c7ec88105ba5 Author: John Bogovic Date: Mon Dec 16 16:07:25 2024 -0500 doc: getShardSize commit fccdb9abe4149f1a5175305a006e90e9c1aa7609 Author: John Bogovic Date: Tue Nov 19 11:39:59 2024 -0500 wip: dummy impl of writeShard commit 865c861a6ff268e143d46a5f61d8d0213b311f31 Author: John Bogovic Date: Tue Nov 19 11:17:21 2024 -0500 wip: n5 exception and InMemoryShard commit a8df678ff3e3e048bbc1f378ee1eff655b2a38ec Author: John Bogovic Date: Fri Sep 20 11:36:55 2024 -0400 feat: add getShardAttributes method to DatasetAttributes * and minor clean up commit eb6de85b51c70b96f02c35218de8d3795262c8ed Author: John Bogovic Date: Fri Sep 20 11:35:55 2024 -0400 fix/wip: sharding codec block sizes needs reversing in zarr commit 4489116ce5e5a003d2108b37436df565b4a4bb54 Author: John Bogovic Date: Fri Sep 20 11:34:50 2024 -0400 fix: getBlockPositionInShard commit 66351079e151cb42c44c1610663d0d70a2db195f Author: John Bogovic Date: Thu Sep 19 15:44:10 2024 -0400 style: ShardingCodec commit f667bb53e91038953a62863e1caec453abedd4e8 Author: John Bogovic Date: Thu Sep 19 15:39:06 2024 -0400 fix: ShardingCodec indexLocation should default to END commit 48695aaba6110879cf95b6be4425610a81d4169a Merge: b80fef0 9fac328 Author: John Bogovic Date: Tue Sep 17 14:42:49 2024 -0400 Merge pull request #128 from cmhulbert/wip/codecsShards feat: wip writing through shards commit 9fac328eb1f7fa88f5c3eccdf49ec2cc58680528 Author: Caleb Hulbert Date: Tue Sep 17 14:41:50 2024 -0400 feat: wip support for ShardIndex location and bytesorder This is very preliminary. It works in a way, but is not as performant or reasonable as it likely should be. Consider this an initial implementation proof of concept for implementing the rest of sharding, especially working towards multiple read/write block aggregations commit 7a1bbcb33c358a1c78f52f0867b92b2fe9202380 Author: Caleb Hulbert Date: Tue Sep 17 14:39:58 2024 -0400 feat: rethrow NoSuchFile as NoSuchKey commit ae1f347c9d0713aadebea000912af2792a90b8aa Author: Caleb Hulbert Date: Mon Sep 16 15:05:24 2024 -0400 feat: wip shard/codec support Basic working examples with file based shards writing new shards, and reading existing shards (with index at the end) commit b80fef09617c76347bd8d28dedabe57c847b3b11 Author: John Bogovic Date: Wed Sep 4 15:08:50 2024 -0400 fix: BytesTest N5BytesCodec has name "n5bytes" commit 7ef4297a1d32e7082b12ad497e15c1f1530b5b67 Author: John Bogovic Date: Wed Sep 4 14:40:46 2024 -0400 style: import order commit 87456a42e11072cdd62d65d6c7ede21e7fb452f2 Author: John Bogovic Date: Wed Sep 4 13:37:37 2024 -0400 wip: DatasetAttributes allow empty codecs commit 784a4a2ca6228d8293d970fffb56da2642345971 Author: John Bogovic Date: Wed Sep 4 13:37:18 2024 -0400 wip: toward supporting endianness commit 5f8fa624e29df82867b774644e583932cbe1bf5a Author: John Bogovic Date: Wed Sep 4 11:48:35 2024 -0400 test/fix: BytesTests now works with refactor commit c24605d807cfafde2b8a3ca99c238cf70848184a Author: John Bogovic Date: Wed Sep 4 10:47:11 2024 -0400 fix(wip): DefaultBlockReader/Writer can get ArrayToBytesCodec directly commit 2a7327a306aaa4906ba4eba7496b124dfe6502b6 Author: John Bogovic Date: Wed Sep 4 10:38:48 2024 -0400 wip: undo ArrayToBytes codec changes commit 786e4d5ea38f7a739f3dc827bfb8e22d6c5639c9 Author: John Bogovic Date: Wed Sep 4 09:17:14 2024 -0400 chore(pom): depend on guava commit dbd7b158930734cb85af2ecf5bbad40eddd60b62 Author: John Bogovic Date: Tue Sep 3 20:13:59 2024 -0400 refactor: create N5BytesCodec, BytesCodec is simple zarr approach commit 764d05ad63305447dc0f757a46f4175400af03c5 Author: John Bogovic Date: Tue Sep 3 16:48:17 2024 -0400 feat: DataBlock methods to read/write directly from DataInput/Output commit 71802905336096f9db27cfe3556993479a5986fd Author: John Bogovic Date: Mon Aug 26 14:57:35 2024 -0400 pref: normalGetDatasetAttributes should call createDatasetAttributes * so zarr only needs to override createDatasetAttributes commit 2e18288dbf647e93e39b110fefaf9c685a0b5a16 Author: John Bogovic Date: Mon Aug 26 14:56:26 2024 -0400 fix: NameConfigAdapter avoid NPE * configuration may be null when all parameters are optional commit 17ef0fef07ad0e25992ba5b088c179e52a2efeb0 Author: John Bogovic Date: Mon Aug 26 11:45:19 2024 -0400 fix: LockedFileChannel truncation logic commit e8cfefd382a007a5ce7865cb4106e2c7f227fc00 Author: John Bogovic Date: Mon Aug 26 11:18:46 2024 -0400 fix: LockedFileChannel locking commit 9155ec5e4faa710568424c98d774771c387ffa64 Author: John Bogovic Date: Mon Aug 26 10:52:31 2024 -0400 test: fix FixedScaleOffsetTests commit f38fef7ff26edf8ead04ef9e117e39f8f5b60324 Author: Caleb Hulbert Date: Fri Aug 23 10:02:50 2024 -0400 fix: isDataset caching commit 5ad00fd58d55a3c03e53a5bdf3c2eed30d2c14c9 Author: Caleb Hulbert Date: Fri Aug 16 16:58:21 2024 -0400 feat: WIP initial read/write blocks through composed codecs implemention analogous to zarr array->bytes/ bytes->bytes codecs. commit 314b1996df72b528fcc7798ddaecd9e9c78ebb1b Author: John Bogovic Date: Wed Aug 14 13:08:05 2024 -0400 perf: BlockReader have read call static method commit 21ab72c34b63a8ddf66ef162b829521d4d7e1412 Author: John Bogovic Date: Wed Aug 14 13:07:47 2024 -0400 test: minor updates commit 664007b7cb5e9da951aa58901bd6a06a73a63fa5 Author: Caleb Hulbert Date: Wed Aug 14 12:15:40 2024 -0400 feat: more shard and codec work commit 6b1e9a188d71e87abd7a925e4fe358bd7d2717f0 Author: John Bogovic Date: Mon Aug 12 16:38:52 2024 -0400 test: deserialization behavior commit 344aa462d0855019a5475eddcf46a288112d9c0a Author: John Bogovic Date: Mon Aug 12 13:39:59 2024 -0400 test: codec array with a compressor commit c7fb316082ab1ee3bc794ba8bb700b03fec469c0 Author: John Bogovic Date: Mon Aug 12 13:18:34 2024 -0400 test: start CodecSerialization test * make AsType, FixedScaleOffset, Identity codecs serializable commit 1d9c0c2414afb309ae1def918b36c761d136b61d Author: John Bogovic Date: Mon Aug 12 11:37:49 2024 -0400 refactor: rename former 'chunkGrid' variables commit d636b83133476e037bd9b466d4f6a0cc41196cc6 Author: Caleb Hulbert Date: Mon Aug 12 10:24:33 2024 -0400 feat: annotations for extensible serialization for codecs commit 35a4d354ee54d1c22849f92c49fad50269b069e9 Author: Caleb Hulbert Date: Fri Aug 9 15:21:45 2024 -0400 test: remove outdated wip config parsing commit 553c85e3dfe2e3a662a48d30e97f2830ef777c96 Author: John Bogovic Date: Fri Aug 9 15:25:19 2024 -0400 wip: BytesCodec update commit 0cb5a0f5fd99a7c0b8cc19e3140be4612d38bd23 Author: John Bogovic Date: Thu Aug 8 16:10:47 2024 -0400 refactor: Compression interface extends Codec * no need for getCompressionAsCodec * use getType for codecs commit b01b6becfbeab6ff9479e4538c843631bb106c8b Author: John Bogovic Date: Tue Aug 6 16:57:36 2024 -0400 wip add constant N5_DATASET_ATTRIBUTES commit bf4f63e79aaa827fa5d4fab066ee06f5cebb588b Author: John Bogovic Date: Mon Aug 5 13:27:26 2024 -0400 wip: move getAttributesKey to GsonN5Reader commit 786ec1fb5545319167d860465282a7b5f313a8c5 Author: John Bogovic Date: Mon Aug 5 12:50:36 2024 -0400 feat: add getAttributesKey * this may make more sense in a more basic interface commit 1931530a758e87350fc29698eee9ce304bf383aa Author: John Bogovic Date: Fri Aug 2 16:23:58 2024 -0400 fix: return KVA return for types for ranged lockForReading/Writing commit 1990424e982c776f9a35d684009f63387061fe04 Author: John Bogovic Date: Fri Aug 2 16:23:22 2024 -0400 wip/feat: add VirtualShard.createIndex * edit behavior of getIndex commit bdadb582ec849dbc65497bb465dc19d28b690a0c Author: Caleb Hulbert Date: Fri Aug 2 16:17:23 2024 -0400 wip: zone serialization and sharding? commit 083cb548e5c68af48d63f7ac3f2b370360f267fb Author: Caleb Hulbert Date: Thu Aug 1 14:49:43 2024 -0400 fix: partial write defaults with 0 test,build: include n5-universe for tests commit 0b4d73c33d38e9a5853ba64e893a14d7493d2289 Author: John Bogovic Date: Wed Jul 24 17:31:24 2024 -0400 wip: toward block writing through shard commit c5ee84dc59ee86150cbdee6a929c786e391b4340 Author: John Bogovic Date: Wed Jul 24 17:26:44 2024 -0400 fix: AbstractShard getBlockSize commit be870132f371e9cb2feb9603f03c461d49f29644 Author: John Bogovic Date: Wed Jul 24 17:25:28 2024 -0400 feat: make partial writes possible for key value access commit 6874c39981c6341751cbc3f1a6828e798306f395 Author: John Bogovic Date: Wed Jul 24 13:27:08 2024 -0400 test: reading a zarr shard demo commit 77c096d96ccead5050bcae516e07fb12470bd67f Author: John Bogovic Date: Wed Jul 24 12:41:56 2024 -0400 feat/wip: add VirtualShard commit 8aabcf43cad0a553f773349ab775c06c706e5c7c Author: John Bogovic Date: Wed Jul 24 12:04:20 2024 -0400 wip/feat: ShardIndex progress, VirtualShard progress commit ce0b25dcf87d3eae0ffd9f0d441d6d470b827d93 Author: John Bogovic Date: Wed Jul 24 12:00:55 2024 -0400 feat/wip: add size partial read lockForReading methods commit 352427e09285da36f2f7b45999b894143e9946da Author: Caleb Hulbert Date: Tue Jul 23 15:49:51 2024 -0400 wip: more shard/codec work commit a2131902c62774cb22be894e034cfc212fa71f87 Author: John Bogovic Date: Tue Jul 23 13:45:38 2024 -0400 feat(wip): update core n5 api with sharding commit 6fbb729271104f2a1ef27c2827019ac1f9b58670 Author: John Bogovic Date: Tue Jul 23 13:45:00 2024 -0400 feat(wip): add prelim Shard classes commit 72cd669af0ecfc8e122b3b7b7d8c9f77ef6b1363 Author: John Bogovic Date: Tue Jul 23 13:44:16 2024 -0400 feat: add ShardingCodec commit 1cd307c42879dbda6a7ce7b8d73443dee776eeec Author: John Bogovic Date: Tue Jul 23 13:43:55 2024 -0400 feat: add BytesCodec commit 5a821593f31b1b82377834c56ea5ac8888c88992 Author: John Bogovic Date: Tue Jul 23 13:35:14 2024 -0400 refactor: codec getId to getName * to match zarr v3 commit e756f8e18b18013f1cc442963bf42548779cd73e Author: John Bogovic Date: Fri Jul 19 10:24:56 2024 -0400 wip: add ChecksumException commit c54bdb98b9e8270b86c7bcf3fc644bca79c30edf Author: John Bogovic Date: Fri Jul 19 10:09:41 2024 -0400 wip: use CheckedInput/Output streams from java.util.zip * move some functionality to ChecksumCodec commit adb84bb7c3dcd282a1e5dae2cc06ce89db0144d3 Author: John Bogovic Date: Thu Jul 18 17:54:21 2024 -0400 feat: add ChecksumCodec * DeterministicSizeCodec and Crc32cChecksumCodec commit 76e1ec2a34dd231ab110f5a890476382ee37d16a Merge: 34bf14f 14d3b69 Author: John Bogovic Date: Wed Jul 17 15:10:41 2024 -0400 Merge remote-tracking branch 'origin/codecs' into dev/shards # Conflicts: # src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java commit 14d3b696ae9c399f4ff0cab45c1dcd77ab55a1e7 Author: John Bogovic Date: Tue May 28 15:23:22 2024 -0400 feat(wip): reading and writing blocks uses codecs commit c4f96f24aaa14aac8881ae62b7acafffba6b1fb8 Author: John Bogovic Date: Tue May 21 17:12:39 2024 -0400 feat: wip toward codecs --- pom.xml | 36 +- .../saalfeldlab/n5/AbstractDataBlock.java | 31 +- .../saalfeldlab/n5/ByteArrayDataBlock.java | 25 - .../saalfeldlab/n5/Bzip2Compression.java | 25 - .../n5/CachedGsonKeyValueN5Reader.java | 12 +- .../n5/CachedGsonKeyValueN5Writer.java | 14 +- .../janelia/saalfeldlab/n5/Compression.java | 34 +- .../saalfeldlab/n5/CompressionAdapter.java | 27 +- .../org/janelia/saalfeldlab/n5/DataBlock.java | 27 +- .../org/janelia/saalfeldlab/n5/DataType.java | 25 - .../saalfeldlab/n5/DatasetAttributes.java | 427 ++++++++++++++++-- .../saalfeldlab/n5/DoubleArrayDataBlock.java | 25 - .../n5/FileSystemKeyValueAccess.java | 25 - .../saalfeldlab/n5/FloatArrayDataBlock.java | 25 - .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 53 +-- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 69 ++- .../janelia/saalfeldlab/n5/GsonN5Reader.java | 38 +- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 36 -- .../saalfeldlab/n5/HttpKeyValueAccess.java | 9 +- .../saalfeldlab/n5/IntArrayDataBlock.java | 25 - .../saalfeldlab/n5/KeyValueAccess.java | 25 - .../janelia/saalfeldlab/n5/LockedChannel.java | 34 +- .../saalfeldlab/n5/LongArrayDataBlock.java | 25 - .../saalfeldlab/n5/Lz4Compression.java | 5 +- .../janelia/saalfeldlab/n5/N5FSReader.java | 25 - .../janelia/saalfeldlab/n5/N5FSWriter.java | 25 - .../saalfeldlab/n5/N5KeyValueReader.java | 23 +- .../org/janelia/saalfeldlab/n5/N5Reader.java | 60 +-- .../org/janelia/saalfeldlab/n5/N5URI.java | 120 +++-- .../org/janelia/saalfeldlab/n5/N5Writer.java | 64 ++- .../saalfeldlab/n5/NameConfigAdapter.java | 5 + .../saalfeldlab/n5/RawCompression.java | 36 +- .../saalfeldlab/n5/ReflectionUtils.java | 25 - .../saalfeldlab/n5/ShortArrayDataBlock.java | 25 - .../saalfeldlab/n5/StringDataBlock.java | 82 ++-- .../saalfeldlab/n5/cache/N5JsonCache.java | 2 + .../saalfeldlab/n5/codec/BlockCodec.java | 23 + .../saalfeldlab/n5/codec/BlockCodecInfo.java | 33 +- .../saalfeldlab/n5/codec/CodecInfo.java | 3 +- ...oncatenatedDeterministicSizeDataCodec.java | 21 + .../saalfeldlab/n5/codec/DataCodec.java | 21 +- .../saalfeldlab/n5/codec/DataCodecInfo.java | 2 + .../n5/codec/DeterministicSizeCodecInfo.java | 13 + .../n5/codec/DeterministicSizeDataCodec.java | 38 ++ .../saalfeldlab/n5/codec/IdentityCodec.java | 41 ++ .../n5/codec/IndexCodecAdapter.java | 45 ++ .../n5/codec/N5BlockCodecInfo.java | 26 +- .../saalfeldlab/n5/codec/N5BlockCodecs.java | 36 +- .../n5/codec/RawBlockCodecInfo.java | 47 +- .../saalfeldlab/n5/codec/RawBlockCodecs.java | 11 + .../n5/codec/checksum/ChecksumCodec.java | 109 +++++ .../n5/codec/checksum/ChecksumException.java | 12 + .../codec/checksum/Crc32cChecksumCodec.java | 52 +++ .../n5/serialization/NameConfig.java | 14 + .../saalfeldlab/n5/shard/DatasetAccess.java | 29 ++ .../n5/shard/DefaultDatasetAccess.java | 384 ++++++++++++++++ .../n5/shard/DefaultShardCodecInfo.java | 136 ++++++ .../janelia/saalfeldlab/n5/shard/Nesting.java | 347 ++++++++++++++ .../n5/shard/PositionValueAccess.java | 100 ++++ .../saalfeldlab/n5/shard/RawShard.java | 52 +++ .../saalfeldlab/n5/shard/RawShardCodec.java | 83 ++++ .../n5/shard/RawShardDataBlock.java | 42 ++ .../saalfeldlab/n5/shard/ShardCodecInfo.java | 44 ++ .../saalfeldlab/n5/shard/ShardIndex.java | 238 ++++++++++ .../saalfeldlab/n5/util/FinalPosition.java | 38 ++ .../saalfeldlab/n5/util/GridIterator.java | 178 ++++++++ .../janelia/saalfeldlab/n5/util/Position.java | 66 +++ .../saalfeldlab/n5/AbstractN5Test.java | 290 +++++++++--- .../saalfeldlab/n5/DatasetAttributesTest.java | 247 ++++++++++ .../janelia/saalfeldlab/n5/N5Benchmark.java | 25 - .../saalfeldlab/n5/N5CachedFSTest.java | 4 +- .../n5/backward/CompatibilityTest.java | 141 ++++++ .../n5/backward/CreateSampleData.java | 69 +++ .../n5/benchmarks/ReadDataBenchmarks.java | 136 ++++++ .../ReadDataBenchmarksKvaReadData.java | 43 ++ .../saalfeldlab/n5/cache/N5CacheTest.java | 2 +- .../saalfeldlab/n5/codec/BlockCodecTests.java | 291 ++++++++++++ .../saalfeldlab/n5/codec/BytesCodecTests.java | 208 +++++++++ .../n5/compression/CompressionTypesTest.java | 3 - .../saalfeldlab/n5/demo/BlockIterators.java | 93 ++++ .../n5/http/HttpReaderFsWriter.java | 41 +- .../n5/kva/AbstractKeyValueAccessTest.java | 3 - .../n5/kva/FileSystemKeyValueAccessTest.java | 3 - .../n5/kva/HttpKeyValueAccessTest.java | 3 - .../n5/serialization/CodecSerialization.java | 92 ++++ .../saalfeldlab/n5/shard/NestedGridTest.java | 88 ++++ .../saalfeldlab/n5/shard/RawShardTest.java | 142 ++++++ .../saalfeldlab/n5/shard/ShardIndexTest.java | 105 +++++ .../saalfeldlab/n5/shard/ShardTest.java | 427 ++++++++++++++++++ .../n5/shard/TestPositionValueAccess.java | 57 +++ .../backward/data-1.5.0.n5/attributes.json | 1 + .../resources/backward/data-1.5.0.n5/raw/0/0 | Bin 0 -> 32 bytes .../resources/backward/data-1.5.0.n5/raw/0/1 | Bin 0 -> 17 bytes .../resources/backward/data-1.5.0.n5/raw/1/0 | Bin 0 -> 20 bytes .../resources/backward/data-1.5.0.n5/raw/1/1 | Bin 0 -> 14 bytes .../data-1.5.0.n5/raw/attributes.json | 1 + .../backward/data-2.5.1.n5/attributes.json | 1 + .../resources/backward/data-2.5.1.n5/raw/0/0 | Bin 0 -> 32 bytes .../resources/backward/data-2.5.1.n5/raw/0/1 | Bin 0 -> 17 bytes .../resources/backward/data-2.5.1.n5/raw/1/0 | Bin 0 -> 20 bytes .../resources/backward/data-2.5.1.n5/raw/1/1 | Bin 0 -> 14 bytes .../data-2.5.1.n5/raw/attributes.json | 1 + .../backward/data-3.1.3.n5/attributes.json | 1 + .../resources/backward/data-3.1.3.n5/raw/0/0 | Bin 0 -> 32 bytes .../resources/backward/data-3.1.3.n5/raw/0/1 | Bin 0 -> 17 bytes .../resources/backward/data-3.1.3.n5/raw/1/0 | Bin 0 -> 20 bytes .../resources/backward/data-3.1.3.n5/raw/1/1 | Bin 0 -> 14 bytes .../data-3.1.3.n5/raw/attributes.json | 1 + .../urlAttributes.n5/a/aa/aaa/attributes.json | 28 -- .../url/urlAttributes.n5/a/aa/attributes.json | 28 -- .../url/urlAttributes.n5/a/attributes.json | 28 -- .../url/urlAttributes.n5/attributes.json | 30 +- .../url/urlAttributes.n5/objs/attributes.json | 28 -- 113 files changed, 5401 insertions(+), 1113 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDeterministicSizeDataCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/IndexCodecAdapter.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumException.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/util/Position.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java create mode 100644 src/test/resources/backward/data-1.5.0.n5/attributes.json create mode 100644 src/test/resources/backward/data-1.5.0.n5/raw/0/0 create mode 100644 src/test/resources/backward/data-1.5.0.n5/raw/0/1 create mode 100644 src/test/resources/backward/data-1.5.0.n5/raw/1/0 create mode 100644 src/test/resources/backward/data-1.5.0.n5/raw/1/1 create mode 100644 src/test/resources/backward/data-1.5.0.n5/raw/attributes.json create mode 100644 src/test/resources/backward/data-2.5.1.n5/attributes.json create mode 100644 src/test/resources/backward/data-2.5.1.n5/raw/0/0 create mode 100644 src/test/resources/backward/data-2.5.1.n5/raw/0/1 create mode 100644 src/test/resources/backward/data-2.5.1.n5/raw/1/0 create mode 100644 src/test/resources/backward/data-2.5.1.n5/raw/1/1 create mode 100644 src/test/resources/backward/data-2.5.1.n5/raw/attributes.json create mode 100644 src/test/resources/backward/data-3.1.3.n5/attributes.json create mode 100644 src/test/resources/backward/data-3.1.3.n5/raw/0/0 create mode 100644 src/test/resources/backward/data-3.1.3.n5/raw/0/1 create mode 100644 src/test/resources/backward/data-3.1.3.n5/raw/1/0 create mode 100644 src/test/resources/backward/data-3.1.3.n5/raw/1/1 create mode 100644 src/test/resources/backward/data-3.1.3.n5/raw/attributes.json diff --git a/pom.xml b/pom.xml index 89858106c..2636e9052 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.scijava pom-scijava - 40.0.0 + 43.0.0 @@ -161,6 +161,18 @@ com.google.code.gson gson + + org.scijava + scijava-common + + + org.apache.commons + commons-compress + + + commons-io + commons-io + @@ -168,6 +180,17 @@ junit test + + org.janelia.saalfeldlab + n5-universe + + + org.janelia.saalfeldlab + n5 + + + test + net.imagej ij @@ -194,13 +217,16 @@ ${commons-collections4.version} test + - org.scijava - scijava-common + org.openjdk.jmh + jmh-core + test - org.apache.commons - commons-compress + org.openjdk.jmh + jmh-generator-annprocess + test diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index 60a9b789f..d69648d49 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.util.function.ToIntFunction; @@ -65,9 +40,9 @@ */ public abstract class AbstractDataBlock implements DataBlock { - private final int[] size; - private final long[] gridPosition; - private final T data; + protected final int[] size; + protected final long[] gridPosition; + protected final T data; private final ToIntFunction numElements; public AbstractDataBlock( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index 85771d125..da2658113 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; public class ByteArrayDataBlock extends AbstractDataBlock { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 8ccddd5a8..5fa871088 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index 6f7e7c0a8..3f0db47bf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -73,7 +73,7 @@ default DatasetAttributes getDatasetAttributes(final String pathName) { return null; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPath, N5KeyValueReader.ATTRIBUTES_JSON); + attributes = getCache().getAttributes(normalPath, getAttributesKey()); } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPath); } @@ -99,7 +99,7 @@ default T getAttribute( final JsonElement attributes; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + attributes = getCache().getAttributes(normalPathName, getAttributesKey()); } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName); } @@ -120,7 +120,7 @@ default T getAttribute( final String normalizedAttributePath = N5URI.normalizeAttributePath(key); JsonElement attributes; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + attributes = getCache().getAttributes(normalPathName, getAttributesKey()); } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName); } @@ -136,7 +136,7 @@ default boolean exists(final String pathName) { final String normalPathName = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) - return getCache().isGroup(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + return getCache().isGroup(normalPathName, getAttributesKey()); else { return existsFromContainer(normalPathName, null); } @@ -180,7 +180,7 @@ default boolean datasetExists(final String pathName) throws N5IOException { final String normalPathName = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) { - return getCache().isDataset(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + return getCache().isDataset(normalPathName, getAttributesKey()); } return isDatasetFromContainer(normalPathName); } @@ -212,7 +212,7 @@ default JsonElement getAttributes(final String pathName) throws N5IOException { /* If cached, return the cache */ if (cacheMeta()) { - return getCache().getAttributes(groupPath, N5KeyValueReader.ATTRIBUTES_JSON); + return getCache().getAttributes(groupPath, getAttributesKey()); } else { return GsonKeyValueN5Reader.super.getAttributes(groupPath); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index 5e6c85d15..3801e671b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -28,10 +28,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Arrays; - import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import com.google.gson.Gson; @@ -62,9 +58,9 @@ default void createGroup(final String path) throws N5Exception { // else if exists is true (then a dataset is present) so throw an exception to avoid // overwriting / invalidating existing data if (cacheMeta()) { - if (getCache().isGroup(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) + if (getCache().isGroup(normalPath, getAttributesKey())) return; - else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { + else if (getCache().exists(normalPath, getAttributesKey())) { throw new N5Exception("Can't make a group on existing path."); } } @@ -88,8 +84,8 @@ else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { for (final String child : pathParts) { final String childPath = parent.isEmpty() ? child : parent + "/" + child; - getCache().initializeNonemptyCache(childPath, N5KeyValueReader.ATTRIBUTES_JSON); - getCache().updateCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON); + getCache().initializeNonemptyCache(childPath, getAttributesKey()); + getCache().updateCacheInfo(childPath, getAttributesKey()); // only add if the parent exists and has children cached already if (parent != null && !child.isEmpty()) @@ -130,7 +126,7 @@ default void writeAndCacheAttributes( nullRespectingAttributes = getGson().toJsonTree(attributes); } /* Update the cache, and write to the writer */ - getCache().updateCacheInfo(normalGroupPath, N5KeyValueReader.ATTRIBUTES_JSON, nullRespectingAttributes); + getCache().updateCacheInfo(normalGroupPath, getAttributesKey(), nullRespectingAttributes); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index 1cd5e913f..8c059c8a0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.Serializable; @@ -61,8 +36,8 @@ import java.lang.annotation.Target; import org.janelia.saalfeldlab.n5.codec.DataCodec; -import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.scijava.annotations.Indexable; /** @@ -74,11 +49,12 @@ * serialization. *

      * See also: an alternative method for serializing general {@link CodecInfo}s is - * with the {@link NameConfigAdapter}. + * with the {@link NameConfigAdapter}. This interface remains for legacy + * (de)serialization. * * @author Stephan Saalfeld */ -public interface Compression extends Serializable, DataCodecInfo, DataCodec { +public interface Compression extends Serializable, DataCodec, DataCodecInfo { /** * Annotation for runtime discovery of compression schemes. @@ -102,7 +78,6 @@ public interface Compression extends Serializable, DataCodecInfo, DataCodec { @Target(ElementType.FIELD) @interface CompressionParameter {} - @Override default String getType() { final CompressionType compressionType = getClass().getAnnotation(CompressionType.class); @@ -116,4 +91,5 @@ default String getType() { default DataCodec create() { return this; } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java index 3cd399a48..022f4b78a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.lang.reflect.Constructor; @@ -124,7 +99,7 @@ public static synchronized void update(final boolean override) { newInstance.compressionConstructors.put(type, constructor); newInstance.compressionParameters.put(type, parameters); - } catch (final ClassNotFoundException | NoSuchMethodException | ClassCastException + } catch (final NoClassDefFoundError | ClassNotFoundException | NoSuchMethodException | ClassCastException | UnsatisfiedLinkError e) { System.err.println("Compression '" + item.className() + "' could not be registered"); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 506096e08..b2e2cf1d6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; /** @@ -75,7 +50,7 @@ public interface DataBlock { int[] getSize(); /** - * Returns the position of this data block on the block grid. + * Returns the position of this data block on the block grid relative to dataset. *

      * The dimensionality of the grid position is expected to be equal to the * dimensionality of the dataset. Consistency is not enforced. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 3ef9a410d..08bebf1e6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.lang.reflect.Type; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 1127c658a..b3e1890c1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -28,14 +28,31 @@ */ package org.janelia.saalfeldlab.n5; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +import org.janelia.saalfeldlab.n5.codec.BlockCodec; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.shard.DatasetAccess; +import org.janelia.saalfeldlab.n5.shard.DefaultDatasetAccess; +import org.janelia.saalfeldlab.n5.shard.ShardCodecInfo; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; + import java.io.Serializable; +import java.lang.reflect.Type; import java.util.Arrays; import java.util.HashMap; +import java.util.stream.Collectors; -import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; + /** * Mandatory dataset attributes: @@ -44,11 +61,10 @@ *

    • long[] : dimensions
      • *
    • int[] : blockSize
      • *
    • {@link DataType} : dataType
      • - *
    • {@link Compression} : compression
      • + *
    • {@link CodecInfo}... : encode/decode routines
      • * * * @author Stephan Saalfeld - * */ public class DatasetAttributes implements Serializable { @@ -56,46 +72,137 @@ public class DatasetAttributes implements Serializable { public static final String DIMENSIONS_KEY = "dimensions"; public static final String BLOCK_SIZE_KEY = "blockSize"; + public static final String SHARD_SIZE_KEY = "shardSize"; public static final String DATA_TYPE_KEY = "dataType"; public static final String COMPRESSION_KEY = "compression"; + public static final String CODEC_KEY = "codecs"; + + public static final String[] N5_DATASET_ATTRIBUTES = new String[]{ + DIMENSIONS_KEY, BLOCK_SIZE_KEY, DATA_TYPE_KEY, COMPRESSION_KEY, CODEC_KEY + }; /* version 0 */ protected static final String compressionTypeKey = "compressionType"; private final long[] dimensions; + + // number of samples per block per dimension private final int[] blockSize; + + // TODO add a getter? + // the shard size + private final int[] outerBlockSize; + private final DataType dataType; private final BlockCodecInfo blockCodecInfo; private final DataCodecInfo[] dataCodecInfos; - private final BlockCodec blockCodec; + private transient final DatasetAccess access; public DatasetAttributes( final long[] dimensions, - final int[] blockSize, + final int[] outerBlockSize, final DataType dataType, final BlockCodecInfo blockCodecInfo, final DataCodecInfo... dataCodecInfos) { this.dimensions = dimensions; - this.blockSize = blockSize; this.dataType = dataType; + this.outerBlockSize = outerBlockSize; this.blockCodecInfo = blockCodecInfo == null ? defaultBlockCodecInfo() : blockCodecInfo; - this.dataCodecInfos = Arrays.stream(dataCodecInfos).filter(it -> !(it instanceof RawCompression)).toArray(DataCodecInfo[]::new); - blockCodec = this.blockCodecInfo.create(this, this.dataCodecInfos); + + if (dataCodecInfos == null) + this.dataCodecInfos = new DataCodecInfo[0]; + else + this.dataCodecInfos = Arrays.stream(dataCodecInfos) + .filter(it -> it != null && !(it instanceof RawCompression)) + .toArray(DataCodecInfo[]::new); + + access = createDatasetAccess(); + blockSize = access.getGrid().getBlockSize(0); } + /** + * Constructs a DatasetAttributes instance with specified dimensions, block size, data type, + * and single compressor with default codec. + * + * @param dimensions the dimensions of the dataset + * @param blockSize the size of the blocks in the dataset + * @param dataType the data type of the dataset + * @param dataCodecInfos the codecs used encode/decode the data + */ public DatasetAttributes( final long[] dimensions, final int[] blockSize, final DataType dataType, - final DataCodecInfo compression) { + final DataCodecInfo... dataCodecInfos) { + + this(dimensions, blockSize, dataType, null, dataCodecInfos); + } + + /** + * Constructs a DatasetAttributes instance with specified dimensions, block size, data type, and default codecs + * + * @param dimensions the dimensions of the dataset + * @param blockSize the size of the blocks in the dataset + * @param dataType the data type of the dataset + */ + public DatasetAttributes( + final long[] dimensions, + final int[] blockSize, + final DataType dataType) { + + this(dimensions, blockSize, dataType, new DataCodecInfo[0]); + } + + protected DatasetAccess createDatasetAccess() { + + final int m = nestingDepth(blockCodecInfo); + + // There are m codecs: 1 DataBlock codecs, and m-1 shard codecs. + // The inner-most codec (the DataBlock codec) is at index 0. + final int[][] blockSizes = new int[m][]; + + // NestedGrid validates block sizes, so instantiate it before creating the blockCodecs + // blockCodecInfo.create below could fail unexpecedly with invalid + // blockSizes so validate first + blockSizes[m - 1] = outerBlockSize; + BlockCodecInfo tmpInfo = blockCodecInfo; + for (int l = m - 1; l > 0; --l) { + final ShardCodecInfo info = (ShardCodecInfo)tmpInfo; + blockSizes[l - 1] = info.getInnerBlockSize(); + tmpInfo = info.getInnerBlockCodecInfo(); + } + + BlockCodecInfo currentBlockCodecInfo = blockCodecInfo; + DataCodecInfo[] currentDataCodecInfos = dataCodecInfos; + + final NestedGrid grid = new NestedGrid(blockSizes); + final BlockCodec[] blockCodecs = new BlockCodec[m]; + for (int l = m - 1; l >= 0; --l) { + blockCodecs[l] = currentBlockCodecInfo.create(dataType, blockSizes[l], currentDataCodecInfos); + if (l > 0) { + final ShardCodecInfo info = (ShardCodecInfo)currentBlockCodecInfo; + currentBlockCodecInfo = info.getInnerBlockCodecInfo(); + currentDataCodecInfos = info.getInnerDataCodecInfos(); + } + } - this(dimensions, blockSize, dataType, null, compression); + return new DefaultDatasetAccess<>(grid, blockCodecs); } + private static int nestingDepth(BlockCodecInfo info) { + + if (info instanceof ShardCodecInfo) { + return 1 + nestingDepth(((ShardCodecInfo)info).getInnerBlockCodecInfo()); + } else { + return 1; + } + } + + protected BlockCodecInfo defaultBlockCodecInfo() { return new N5BlockCodecInfo(); @@ -111,11 +218,175 @@ public int getNumDimensions() { return dimensions.length; } +// public int[] getShardSize() { +// +// return shardSize; +// } + public int[] getBlockSize() { return blockSize; } +// /** +// * Returns the number of blocks per dimension for each shard. +// * +// * @return the blocks per shard +// */ +// public int[] getBlocksPerShard() { +// +// final int[] shardSize = getShardSize(); +// final int nd = getNumDimensions(); +// final int[] blocksPerShard = new int[nd]; +// final int[] blockSize = getBlockSize(); +// for (int i = 0; i < nd; i++) +// blocksPerShard[i] = shardSize[i] / blockSize[i]; +// +// return blocksPerShard; +// } + +// /** +// * Returns the number of shards per dimension for this dataset. +// * +// * @return shards per dataset +// */ +// public long[] shardsPerDataset() { +// +// return IntStream.range(0, getNumDimensions()) +// .mapToLong(i -> (long)Math.ceil((double)getDimensions()[i] / getShardSize()[i])) +// .toArray(); +// } +// +// /** +// * Returns the total number of blocks in each shard. +// * +// * @return number of blocks in a shard +// */ +// public long getNumBlocksPerShard() { +// +// return Arrays.stream(getBlocksPerShard()).reduce(1, (x, y) -> x * y); +// } +// +// /** +// * Given a block's position relative to the dataset, returns the position of +// * the shard containing that block. +// * +// * @param blockGridPosition position of a block relative to the dataset +// * @return the position of the containing shard in the shard grid +// */ +// public long[] getShardPositionForBlock(final long... blockGridPosition) { +// +// final int[] blocksPerShard = getBlocksPerShard(); +// final long[] shardGridPosition = new long[blockGridPosition.length]; +// for (int i = 0; i < shardGridPosition.length; i++) { +// shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / blocksPerShard[i]); +// } +// +// return shardGridPosition; +// } +// +// /** +// * Given a {@code datasetRelativeBlockPosition} returns the position +// * relative the the shard at position {@code shardPosition}. +// * +// * @param shardPosition position of the shard +// * @param datasetRelativeBlockPosition position of the block relative to the dataset +// * @return position of the block relative to the shard +// */ +// public int[] getShardRelativeBlockPosition(final long[] shardPosition, final long[] datasetRelativeBlockPosition) { +// +// final long[] shardPos = getShardPositionForBlock(datasetRelativeBlockPosition); +// if (!Arrays.equals(shardPosition, shardPos)) +// return null; +// +// final int[] shardSize = getBlocksPerShard(); +// final int[] shardRelativeBlockPosition = new int[shardSize.length]; +// for (int i = 0; i < shardSize.length; i++) { +// shardRelativeBlockPosition[i] = (int)(datasetRelativeBlockPosition[i] % shardSize[i]); +// } +// return shardRelativeBlockPosition; +// } +// +// /** +// * Given a {@code shardRelativeBlockPosition} relative to the shard at +// * position {@code shardPosition}, returns the block' position relative the +// * dataset. +// * +// * @param shardPosition position of the shard +// * @param shardRelativeBlockPosition position of the block relative to the shard +// * @return position of the block relative to the dataset +// */ +// public long[] getBlockPositionFromShardPosition(final long[] shardPosition, final int[] shardRelativeBlockPosition) { +// +// final int[] shardBlockSize = getBlocksPerShard(); +// final long[] datasetRelativeBlockPosition = new long[getNumDimensions()]; +// for (int i = 0; i < getNumDimensions(); i++) { +// datasetRelativeBlockPosition[i] = (shardPosition[i] * shardBlockSize[i]) + (shardRelativeBlockPosition[i]); +// } +// +// return datasetRelativeBlockPosition; +// } +// +// /** +// * Returns the number of shards per dimension for the dataset. +// * +// * @return the size of the shard grid of a dataset +// */ +// public int[] getShardBlockGridSize() { +// +// final int nd = getNumDimensions(); +// final int[] shardBlockGridSize = new int[nd]; +// final int[] blockSize = getBlockSize(); +// for (int i = 0; i < nd; i++) +// shardBlockGridSize[i] = (int)(Math.ceil((double)getDimensions()[i] / blockSize[i])); +// +// return shardBlockGridSize; +// } +// +// public Map> groupBlockPositions(final List blockPositions) { +// +// final TreeMap> map = new TreeMap<>(); +// for (final long[] blockPos : blockPositions) { +// Position shardPos = Position.wrap(getShardPositionForBlock(blockPos)); +// if (!map.containsKey(shardPos)) { +// map.put(shardPos, new ArrayList<>()); +// } +// map.get(shardPos).add(blockPos); +// } +// +// return map; +// } +// +// public Map>> groupBlocks(final List> blocks) { +// +// // figure out how to re-use groupBlockPositions here? +// final TreeMap>> map = new TreeMap<>(); +// for (final DataBlock block : blocks) { +// Position shardPos = Position.wrap(getShardPositionForBlock(block.getGridPosition())); +// if (!map.containsKey(shardPos)) { +// map.put(shardPos, new ArrayList<>()); +// } +// map.get(shardPos).add(block); +// } +// +// return map; +// } + + public boolean isSharded() { + + return blockCodecInfo instanceof ShardCodecInfo; + } + + /** + * Only used for deserialization for N5 backwards compatibility. + * {@link Compression} is no longer a special case. Prefer to reference {@link #getDataCodecInfos()} + * Will return {@link RawCompression} if no compression is otherwise provided, for legacy compatibility. + *

      + * Deprecated in favor of {@link #getDataCodecInfos()}. + * + * @return compression CodecInfo, if one was present, or else RawCompression + */ + @Deprecated public Compression getCompression() { return Arrays.stream(dataCodecInfos) @@ -131,19 +402,40 @@ public DataType getDataType() { } /** - * Get the {@link BlockCodecInfo} for this dataset. + * Get the {@link DatasetAccess} for this dataset. + * + * @return the {@code DatasetAccess} for this dataset + */ + DatasetAccess getDatasetAccess() { + + return (DatasetAccess)access; + } + + /** + * Returns the {@code NestedGrid} for this dataset, from which block and + * shard sizes are accessible. * - * @return the {@code BlockCodecInfo} for this dataset + * @return the NestedGrid */ + public NestedGrid getNestedBlockGrid() { + + return getDatasetAccess().getGrid(); + } + + public BlockCodecInfo getBlockCodecInfo() { return blockCodecInfo; } - @SuppressWarnings("unchecked") - BlockCodec getBlockCodec() { + public DataCodecInfo[] getDataCodecInfos() { - return (BlockCodec) blockCodec; + return dataCodecInfos; + } + + public String relativeBlockPath(long... position) { + + return Arrays.stream(position).mapToObj(Long::toString).collect(Collectors.joining("/")); } public HashMap asMap() { @@ -156,37 +448,94 @@ public HashMap asMap() { return map; } - static DatasetAttributes from( - final long[] dimensions, - final DataType dataType, - int[] blockSize, - Compression compression, - final String compressionVersion0Name) { + private static DatasetAttributesAdapter adapter = null; + + public static DatasetAttributesAdapter getJsonAdapter() { + + if (adapter == null) { + adapter = new DatasetAttributesAdapter(); + } + return adapter; + } - if (blockSize == null) - blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); + public static class DatasetAttributesAdapter implements JsonSerializer, JsonDeserializer { + + @Override public DatasetAttributes deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { + + if (json == null || !json.isJsonObject()) + return null; + final JsonObject obj = json.getAsJsonObject(); + final boolean validKeySet = obj.has(DIMENSIONS_KEY) + && obj.has(BLOCK_SIZE_KEY) + && obj.has(DATA_TYPE_KEY) + && (obj.has(CODEC_KEY) || obj.has(COMPRESSION_KEY) || obj.has(compressionTypeKey)); + + if (!validKeySet) + return null; + + final long[] dimensions = context.deserialize(obj.get(DIMENSIONS_KEY), long[].class); + final int[] blockSize = context.deserialize(obj.get(BLOCK_SIZE_KEY), int[].class); + + final DataType dataType = context.deserialize(obj.get(DATA_TYPE_KEY), DataType.class); + + final BlockCodecInfo blockCodecInfo; + final DataCodecInfo[] dataCodecs; + if (obj.has(CODEC_KEY)) { + final CodecInfo[] codecs = context.deserialize(obj.get(CODEC_KEY), CodecInfo[].class); + blockCodecInfo = (BlockCodecInfo)codecs[0]; + dataCodecs = new DataCodecInfo[codecs.length - 1]; + for (int i = 1; i < codecs.length; i++) { + dataCodecs[i - 1] = (DataCodecInfo)codecs[i]; + } + } else if (obj.has(COMPRESSION_KEY)) { + final Compression compression = CompressionAdapter.getJsonAdapter().deserialize(obj.get(COMPRESSION_KEY), Compression.class, context); + dataCodecs = new DataCodecInfo[]{compression}; + blockCodecInfo = new N5BlockCodecInfo(); + } else if (obj.has(compressionTypeKey)) { + final Compression compression = getCompressionVersion0(obj.get(compressionTypeKey).getAsString()); + dataCodecs = new DataCodecInfo[]{compression}; + blockCodecInfo = new N5BlockCodecInfo(); + } else { + return null; + } + + return new DatasetAttributes(dimensions, blockSize, dataType, blockCodecInfo, dataCodecs); + } + + //FIXME + // this implements multi-codec serialization for N5. We probably don't want this now + @Override public JsonElement serialize(DatasetAttributes src, Type typeOfSrc, JsonSerializationContext context) { + + final JsonObject obj = new JsonObject(); + obj.add(DIMENSIONS_KEY, context.serialize(src.dimensions)); + obj.add(BLOCK_SIZE_KEY, context.serialize(src.blockSize)); + obj.add(DATA_TYPE_KEY, context.serialize(src.dataType)); + + final DataCodecInfo[] codecs = src.dataCodecInfos; + // length > 1 is actually invalid, but this is checked on construction + if (codecs.length == 0) + obj.add(COMPRESSION_KEY, context.serialize(new RawCompression())); + else + obj.add(COMPRESSION_KEY, context.serialize(codecs[0])); + + return obj; + } + + private static Compression getCompressionVersion0(final String compressionVersion0Name) { - /* version 0 */ - if (compression == null) { switch (compressionVersion0Name) { case "raw": - compression = new RawCompression(); - break; + return new RawCompression(); case "gzip": - compression = new GzipCompression(); - break; + return new GzipCompression(); case "bzip2": - compression = new Bzip2Compression(); - break; + return new Bzip2Compression(); case "lz4": - compression = new Lz4Compression(); - break; + return new Lz4Compression(); case "xz": - compression = new XzCompression(); - break; + return new XzCompression(); } + return null; } - - return new DatasetAttributes(dimensions, blockSize, dataType, compression); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index 91bdfe84f..6346556e0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; public class DoubleArrayDataBlock extends AbstractDataBlock { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index d6a9803b8..e6ae8439b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index 52c11b401..6fb46a6bd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; public class FloatArrayDataBlock extends AbstractDataBlock { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index e30c7f88e..8ede4da3a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -32,9 +32,10 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.UncheckedIOException; +import java.util.List; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -91,55 +92,35 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { } @Override - default DataBlock readBlock( + default DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { - final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), gridPosition); - try { - final ReadData blockData = getKeyValueAccess().createReadData(path); - return datasetAttributes.getBlockCodec().decode(blockData, gridPosition); + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), + datasetAttributes); + return datasetAttributes. getDatasetAccess().readBlock(posKva, gridPosition); + } catch (N5Exception.N5NoSuchKeyException e) { return null; } } @Override - default String[] list(final String pathName) throws N5Exception { + default List> readBlocks( + final String pathName, + final DatasetAttributes datasetAttributes, + final List blockPositions) throws N5Exception { - return getKeyValueAccess().listDirectories(absoluteGroupPath(pathName)); + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), datasetAttributes); + return datasetAttributes. getDatasetAccess().readBlocks(posKva, blockPositions); } - /** - * Constructs the path for a data block in a dataset at a given grid - * position. - *

      - * The returned path is - * - * 

      -	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
-	 *
      - *

      - * This is the file into which the data block will be stored. - * - * @param normalPath - * normalized dataset path - * @param gridPosition to the target data block - * @return the absolute path to the data block ad gridPosition - */ - default String absoluteDataBlockPath( - final String normalPath, - final long... gridPosition) { - - final String[] components = new String[gridPosition.length + 1]; - components[0] = normalPath; - int i = 0; - for (final long p : gridPosition) - components[++i] = Long.toString(p); + @Override + default String[] list(final String pathName) throws N5Exception { - return getKeyValueAccess().compose(getURI(), components); + return getKeyValueAccess().listDirectories(absoluteGroupPath(pathName)); } /** @@ -165,6 +146,6 @@ default String absoluteGroupPath(final String normalGroupPath) { */ default String absoluteAttributesPath(final String normalPath) { - return getKeyValueAccess().compose(getURI(), normalPath, N5KeyValueReader.ATTRIBUTES_JSON); + return getKeyValueAccess().compose(getURI(), normalPath, getAttributesKey()); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 97b9a745e..d0a9c0e66 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -26,35 +26,9 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

      - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

      - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

      - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.IOException; -import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.Arrays; import java.util.List; @@ -62,6 +36,7 @@ import com.google.gson.JsonSyntaxException; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -218,7 +193,7 @@ default T removeAttribute(final String pathName, final String key, final Cla throw new N5Exception.N5ClassCastException(e); } if (obj != null) { - writeAttributes(normalPath, attributes); + setAttributes(normalPath, attributes); } return obj; } @@ -235,23 +210,36 @@ default boolean removeAttributes(final String pathName, final List attri return removed; } + @Override + default void writeBlocks( + final String datasetPath, + final DatasetAttributes datasetAttributes, + final DataBlock... dataBlocks) throws N5Exception { + + try { + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), datasetAttributes); + datasetAttributes.getDatasetAccess().writeBlocks(posKva, Arrays.asList(dataBlocks)); + } catch (final UncheckedIOException e) { + throw new N5IOException( + "Failed to write blocks into dataset " + datasetPath, e); + } + } + @Override default void writeBlock( final String path, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception { - final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition()); - try ( - final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath); - final OutputStream out = lock.newOutputStream() - ) { - datasetAttributes.getBlockCodec().encode(dataBlock).writeTo(out); - } catch (final IOException | UncheckedIOException e) { + try { + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); + datasetAttributes. getDatasetAccess().writeBlock(posKva, dataBlock); + } catch (final UncheckedIOException e) { throw new N5IOException( "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, e); } + } @Override @@ -271,12 +259,9 @@ default boolean deleteBlock( final String path, final long... gridPosition) throws N5Exception { - final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), gridPosition); - if (getKeyValueAccess().isFile(blockPath)) - getKeyValueAccess().delete(blockPath); - - - /* an IOException should have occurred if anything had failed midway */ - return true; + final String normalPath = N5URI.normalizeGroupPath(path); + final DatasetAttributes datasetAttributes = getDatasetAttributes(normalPath); + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); + return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition); } -} +} \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index a0aab927a..91b3747d3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -31,6 +31,9 @@ import java.lang.reflect.Type; import java.util.Map; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonParseException; + import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonSyntaxException; @@ -43,6 +46,14 @@ public interface GsonN5Reader extends N5Reader { Gson getGson(); + /** + * Get the key for the {@link KeyValueAccess}, that is used for storing attributes. + * The N5 format uses "attributes.json". + * + * @return the attributes key + */ + String getAttributesKey(); + @Override default Map> listAttributes(final String pathName) throws N5Exception { @@ -59,30 +70,15 @@ default DatasetAttributes getDatasetAttributes(final String pathName) throws N5E default DatasetAttributes createDatasetAttributes(final JsonElement attributes) { - try { - final long[] dimensions = GsonUtils.readAttribute(attributes, DatasetAttributes.DIMENSIONS_KEY, long[].class, getGson()); - if (dimensions == null) { - return null; - } - - final DataType dataType = GsonUtils.readAttribute(attributes, DatasetAttributes.DATA_TYPE_KEY, DataType.class, getGson()); - if (dataType == null) { - return null; - } + final JsonDeserializationContext context = new JsonDeserializationContext() { - final int[] blockSize = GsonUtils.readAttribute(attributes, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, getGson()); - final Compression compression = GsonUtils.readAttribute(attributes, DatasetAttributes.COMPRESSION_KEY, Compression.class, getGson()); + @Override public T deserialize(JsonElement json, Type typeOfT) throws JsonParseException { - /* version 0 */ - final String compressionVersion0Name = compression == null - ? GsonUtils.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, getGson()) - : null; + return getGson().fromJson(json, typeOfT); + } + }; - return DatasetAttributes.from(dimensions, dataType, blockSize, compression, compressionVersion0Name); - } catch (JsonSyntaxException | NumberFormatException | ClassCastException e) { - /* We cannot create a dataset, so return null. */ - return null; - } + return DatasetAttributes.getJsonAdapter().deserialize(attributes, DatasetAttributes.class, context); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 6970ae2d4..3d3db1aeb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - *

      - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

      - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

      - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.IOException; @@ -63,7 +38,6 @@ import java.util.regex.Matcher; import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -71,7 +45,6 @@ import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; import org.janelia.saalfeldlab.n5.N5Exception.N5JsonParseException; -import org.janelia.saalfeldlab.n5.codec.CodecInfo; /** * Utility class for working with JSON. @@ -80,15 +53,6 @@ */ public interface GsonUtils { - static Gson registerGson(final GsonBuilder gsonBuilder) { - - gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); - gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); - gsonBuilder.disableHtmlEscaping(); - return gsonBuilder.create(); - } - /** * Reads the attributes json from a given {@link Reader}. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index e7bc39355..2d50cb9ad 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -305,7 +305,7 @@ private static N5Exception validExistsResponse(int code, String responseMsg, Str if (code >= 200 && code < (allowRedirect ? 400 : 300)) return null; final RuntimeException cause = new RuntimeException(message + "( "+ responseMsg + ")(" + code + ")"); - if (code == 404) + if (code == 404 | code == 410) return new N5Exception.N5NoSuchKeyException(message, cause); return new N5Exception(message, cause); @@ -384,6 +384,7 @@ public InputStream newInputStream() throws N5IOException { } return conn.getInputStream(); } catch (FileNotFoundException e) { + /*default HttpURLConnection throws FileNotFoundException on 404 or 410 */ throw new N5Exception.N5NoSuchKeyException("Could not open stream for " + uri, e); } catch (IOException e) { throw new N5IOException("Could not open stream for " + uri, e); @@ -397,7 +398,7 @@ private String rangeString() { } @Override - public Reader newReader() throws N5IOException { + public Reader newReader() { final InputStreamReader reader = new InputStreamReader(newInputStream(), StandardCharsets.UTF_8); synchronized (resources) { @@ -407,13 +408,13 @@ public Reader newReader() throws N5IOException { } @Override - public OutputStream newOutputStream() throws N5IOException { + public OutputStream newOutputStream() { throw new NonWritableChannelException(); } @Override - public Writer newWriter() throws N5IOException { + public Writer newWriter() { throw new NonWritableChannelException(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 4bea296c2..915c9d65c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; public class IntArrayDataBlock extends AbstractDataBlock { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 5ed6eec8a..832e5368a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java index 7184ea10d..f956937e4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java @@ -26,37 +26,11 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import java.io.Closeable; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Reader; @@ -77,7 +51,7 @@ public interface LockedChannel extends Closeable { * @throws N5IOException * if the reader could not be created */ - public Reader newReader() throws N5IOException; + Reader newReader() throws N5IOException; /** * Create a new {@link InputStream}. @@ -86,7 +60,7 @@ public interface LockedChannel extends Closeable { * @throws N5IOException * if an input stream could not be created */ - public InputStream newInputStream() throws N5IOException; + InputStream newInputStream() throws N5IOException; /** * Create a new UTF-8 {@link Writer}. @@ -95,7 +69,7 @@ public interface LockedChannel extends Closeable { * @throws N5IOException * if a writer could not be created */ - public Writer newWriter() throws N5IOException; + Writer newWriter() throws N5IOException; /** * Create a new {@link OutputStream}. @@ -104,5 +78,5 @@ public interface LockedChannel extends Closeable { * @throws N5IOException * if an output stream could not be created */ - public OutputStream newOutputStream() throws N5IOException; + OutputStream newOutputStream() throws N5IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index bad39ec6d..be0d8883d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; public class LongArrayDataBlock extends AbstractDataBlock { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index 1039713b5..ea01879df 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -31,12 +31,9 @@ import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; -import java.io.IOException; - @CompressionType("lz4") @NameConfig.Name("lz4") public class Lz4Compression implements Compression { @@ -67,7 +64,7 @@ public boolean equals(final Object other) { } @Override - public ReadData decode(final ReadData readData) throws N5IOException { + public ReadData decode(final ReadData readData) { return ReadData.from(new LZ4BlockInputStream(readData.inputStream())); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 8d332098b..4397b4544 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.nio.file.FileSystems; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 589b566db..d856ebb22 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.nio.file.FileSystems; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index e86b712ae..de60ebd5e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -127,9 +127,13 @@ protected N5KeyValueReader( throws N5Exception { this.keyValueAccess = keyValueAccess; - this.gson = GsonUtils.registerGson(gsonBuilder); + this.gson = registerGson(gsonBuilder).create(); this.cacheMeta = cacheMeta; - this.cache = newCache(); + + if (this.cacheMeta) + this.cache = newCache(); + else + this.cache = null; try { uri = keyValueAccess.uri(basePath); @@ -159,6 +163,21 @@ private boolean inferExistence(String path) { return attributes != null || exists(path); } + protected GsonBuilder registerGson(final GsonBuilder gsonBuilder) { + + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(DatasetAttributes.class, DatasetAttributes.getJsonAdapter()); + gsonBuilder.disableHtmlEscaping(); + return gsonBuilder; + } + + @Override + public String getAttributesKey() { + + return ATTRIBUTES_JSON; + } + @Override public Gson getGson() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 367ac2954..e7004cdf3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.ByteArrayInputStream; @@ -252,8 +227,6 @@ default Version getVersion() throws N5Exception { * * @return the base path URI */ - // TODO: should this throw URISyntaxException or can we assume that this is - // never possible if we were able to instantiate this N5Reader? URI getURI(); /** @@ -321,11 +294,39 @@ T getAttribute( * @throws N5Exception * the exception */ - DataBlock readBlock( + DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception; + /** + * Reads multiple {@link DataBlock}s. + *

      + * Implementations may optimize / batch read operations when possible, e.g. + * in the case that the datasets are sharded. + * + * @param pathName + * dataset path + * @param datasetAttributes + * the dataset attributes + * @param gridPositions + * a list of grid positions + * @return a list of data blocks + * @throws N5Exception + * the exception + */ + default List> readBlocks( + final String pathName, + final DatasetAttributes datasetAttributes, + final List gridPositions) throws N5Exception { + + final ArrayList> blocks = new ArrayList<>(); + for( final long[] p : gridPositions ) + blocks.add(readBlock(pathName, datasetAttributes, p)); + + return blocks; + } + /** * Load a {@link DataBlock} as a {@link Serializable}. The offset is given * in @@ -345,13 +346,12 @@ DataBlock readBlock( * @throws ClassNotFoundException * the class not found exception */ - @SuppressWarnings("unchecked") default T readSerializedBlock( final String dataset, final DatasetAttributes attributes, final long... gridPosition) throws N5Exception, ClassNotFoundException { - final DataBlock block = (DataBlock) readBlock(dataset, attributes, gridPosition); + final DataBlock block = readBlock(dataset, attributes, gridPosition); if (block == null) return null; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java index 1eac58a68..c086acc05 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java @@ -454,10 +454,86 @@ public static String normalizeGroupPath(final String path) { return normalizePath(path.startsWith("/") || path.startsWith("\\") ? path.substring(1) : path); } + private enum N5UriPattern { + + /** + * matches any `/` or `[N]` where `N` is non-negative + */ + MULTI_PART_ATTRIBUTE(Pattern.compile(".*((?\\[[0-9]+])")), + /** + * matches `A[N]` where A is some non-empty preceding path + */ + ATTRIBUTE_ARRAY_EXCEPT_START(Pattern.compile("((?\\[[0-9]+]))")), + /** + * The following Pattern has 4 possible matches. + * It is intended to be used to remove matching portions iteratively until no further matches are found: + *

      + * The first 3 matches can remove redundant separators of the form: + *

        + *
      • (?<=/)/+ : `a///b` -> `a/b`
        • + *
      • (?<=(/|^))(\./)+ : `a/./b` -> `a/b`
        • + *
      • ((/|(?<=/))\.)$ : `a/b/` -> `a/b`
        • + *
      + * The next match avoids removing `/` when it is NOT redundant (e.g. only character, or escaped): + *
        + *
      • (? `/ , `/a/b/\\/` -> `/a/b/\\/`
        • + *
      + * The last match resolves relative paths: + *
        + *
      • 5. ((?<=^/)|^|(?<=(/|^))[^/]+(? + *
          + *
        • `a/../b` -> `b`
          • + *
        • `/a/../b` -> `/b`
          • + *
        • `../a/../b` -> `b`
          • + *
        • `/../a/../b` -> `/b`
          • + *
        • `/../a/../../b` -> `/b`
          • + *
        + *
      + *

      + */ + RELATIVE_ATTRIBUTE_PARTS(Pattern.compile( "((?<=/)/+|(?<=(/|^))(\\./)+|((/|(?<=/))\\.)$|(? - * Attribute paths have a few of special characters: + * Attribute paths have a few special characters: *

        *
      • "." which represents the current element
        • *
      • ".." which represent the previous elemnt
        • @@ -503,50 +579,20 @@ public static String normalizeAttributePath(final String attributePath) { * Short circuit if there are no non-escaped `/` or array indices (e.g. * [N] where N is a non-negative integer) */ - if (!attributePath.matches(".*((? `[10]/b` */ - final String attrPathPlusFirstIndexSeparator = attributePath.replaceAll("^(?\\[[0-9]+])", "${array}/"); + final String attrPathPlusFirstIndexSeparator = N5UriPattern.appendSlashAfterArrayStart(attributePath); + /* * Add separator before and after arrays not at the beginning `a[10]b` * -> `a/[10]/b` */ - final String attrPathPlusIndexSeparators = attrPathPlusFirstIndexSeparator - .replaceAll("((?\\[[0-9]+]))", "/${array}/"); + final String attrPathPlusIndexSeparators = N5UriPattern.addSlashAroundArrayExceptStart(attrPathPlusFirstIndexSeparator); - /* - * The following has 4 possible matches, in each case it removes the - * match: - * The first 3 remove redundant separators of the form: - * 1.`a///b` -> `a/b` : (?<=/)/+ - * 2.`a/./b` -> `a/b` : (?<=(/|^))(\./)+ - * 3.`a/b/` -> `a/b` : ((/|(?<=/))\.)$ - * The next avoids removing `/` when it is NOT redundant (e.g. only character, or escaped): - * 4. `/` -> `/ , `/a/b/\\/` -> `/a/b/\\/` : (? `b` - * - `/a/../b` -> `/b` - * - `../a/../b` -> `b` - * - `/../a/../b` -> `/b` - * - `/../a/../../b` -> `/b` - * - * This is run iteratively, since earlier removals may cause later - * removals to be valid, - * as well as the need to match once per relative `../` pattern. - */ - final Pattern relativePathPattern = Pattern.compile( - "((?<=/)/+|(?<=(/|^))(\\./)+|((/|(?<=/))\\.)$|(? void writeBlock( final DataBlock dataBlock) throws N5Exception; /** - * Deletes the block at {@code gridPosition} + * Write multiple data blocks, useful for request aggregation. * * @param datasetPath dataset path - * @param gridPosition position of block to be deleted + * @param datasetAttributes the dataset attributes + * @param dataBlocks the data block + * @param the data block data type * @throws N5Exception the exception + */ + default void writeBlocks( + final String datasetPath, + final DatasetAttributes datasetAttributes, + final DataBlock... dataBlocks) throws N5Exception { + + // default method is naive + for (DataBlock block : dataBlocks) + writeBlock(datasetPath, datasetAttributes, block); + } + + /** + * Deletes the block at {@code gridPosition}. * - * @return {@code true} if the block at {@code gridPosition} is "empty" - * after - * deletion. The meaning of "empty" is implementation dependent. For - * example "empty" means that no file exists on the file system for - * the - * deleted block in case of the file system implementation. + * @param datasetPath dataset path + * @param gridPosition position of block to be deleted + * @throws N5Exception if the block exists but could not be deleted * + * @return {@code true} if the block at {@code gridPosition} existed and was deleted. */ boolean deleteBlock( final String datasetPath, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java index 7fd05a262..2c7fa9b39 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java @@ -87,6 +87,11 @@ public static synchronized void update(final NameConfigAdapter adapter) { Class clazz; try { clazz = (Class)Class.forName(item.className()); + + final NameConfig.Serialize serialize = clazz.getAnnotation(NameConfig.Serialize.class); + if (serialize != null && !serialize.value()) + continue; + final String name = clazz.getAnnotation(NameConfig.Name.class).value(); final String type = prefix + "." + name; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index bb4dfe51b..6d3ebc86e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -26,43 +26,22 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import org.janelia.saalfeldlab.n5.Compression.CompressionType; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeDataCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("raw") -public class RawCompression implements Compression { +@NameConfig.Name("raw") +public class RawCompression implements Compression, DeterministicSizeDataCodec { private static final long serialVersionUID = 7526445806847086477L; @Override public boolean equals(final Object other) { + return other != null && other.getClass() == RawCompression.class; } @@ -75,4 +54,9 @@ public ReadData encode(final ReadData readData) { public ReadData decode(final ReadData readData) { return readData; } + + @Override + public long encodedSize(final long size) { + return size; + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java index eeff6fdc3..3a7fed420 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ReflectionUtils.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.lang.reflect.Field; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index a181e589c..e247f76a6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; public class ShortArrayDataBlock extends AbstractDataBlock { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java index 5b811b5e4..fe846ebd6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/StringDataBlock.java @@ -1,31 +1,3 @@ -/*- - * #%L - * Not HDF5 - * %% - * Copyright (C) 2017 - 2025 Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. @@ -53,10 +25,58 @@ */ package org.janelia.saalfeldlab.n5; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + public class StringDataBlock extends AbstractDataBlock { - public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { + protected static final Charset ENCODING = StandardCharsets.UTF_8; + protected static final String NULLCHAR = "\0"; + protected byte[] serializedData = null; + protected String[] actualData = null; + + public StringDataBlock(final int[] size, final long[] gridPosition, final String[] data) { + super(size, gridPosition, new String[0], a -> a.length); + actualData = data; + } + + public StringDataBlock(final int[] size, final long[] gridPosition, final byte[] data) { + super(size, gridPosition, new String[0], a -> a.length); + serializedData = data; + } + + public void readData(final ByteBuffer buffer) { + + if (buffer.hasArray()) { + if (buffer.array() != serializedData) + buffer.get(serializedData); + actualData = deserialize(buffer.array()); + } else + actualData = ENCODING.decode(buffer).toString().split(NULLCHAR); + } + + protected byte[] serialize(String[] strings) { + final String flattenedArray = String.join(NULLCHAR, strings) + NULLCHAR; + return flattenedArray.getBytes(ENCODING); + } + + protected String[] deserialize(byte[] rawBytes) { + final String rawChars = new String(rawBytes, ENCODING); + return rawChars.split(NULLCHAR); + } + + @Override + public int getNumElements() { + if (serializedData == null) + serializedData = serialize(actualData); + return serializedData.length; + } - super(size, gridPosition, data, a -> a.length); - } + @Override + public String[] getData() { + if (actualData == null) + actualData = deserialize(serializedData); + return actualData; + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 770cece13..2a60743a0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -28,6 +28,7 @@ */ package org.janelia.saalfeldlab.n5.cache; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashSet; @@ -183,6 +184,7 @@ public String[] list(final String normalPathKey) { for (final String child : cacheInfo.children) { children[i++] = child; } + Arrays.sort(children); return children; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java index 8eb889661..269daa107 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodec.java @@ -30,6 +30,7 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.RawCompression; import org.janelia.saalfeldlab.n5.readdata.ReadData; /** @@ -43,4 +44,26 @@ public interface BlockCodec { ReadData encode(DataBlock dataBlock) throws N5IOException; DataBlock decode(ReadData readData, long[] gridPosition) throws N5IOException; + + /** + * Given the {@code blockSize} of a {@code DataBlock} return the size of + * the encoded block in bytes. + *

        + * A {@code UnsupportedOperationException} is thrown, if this {@code + * BlockCodec} cannot determine encoded size independent of block content. + * For example, if the block type contains var-length elements or if the + * serializer uses a non-deterministic {@code DataCodec}. + * + * @param blockSize + * size of the block to be encoded + * + * @return size of the encoded block in bytes + * + * @throws UnsupportedOperationException + * if this {@code DataBlockSerializer} cannot determine encoded size independent of block content + */ + default long encodedSize(int[] blockSize) throws UnsupportedOperationException { + + throw new UnsupportedOperationException(); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java index ef5f3825a..bd18f1759 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BlockCodecInfo.java @@ -1,7 +1,7 @@ package org.janelia.saalfeldlab.n5.codec; -import java.util.Arrays; import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -11,15 +11,32 @@ * {@code BlockCodec}s encode {@link DataBlock}s into {@link ReadData} and * decode {@link ReadData} into {@link DataBlock}s. */ -public interface BlockCodecInfo extends CodecInfo { +public interface BlockCodecInfo extends CodecInfo, DeterministicSizeCodecInfo { - BlockCodec create(final DatasetAttributes attributes, final DataCodec... codecs); + default long[] getKeyPositionForBlock(final DatasetAttributes attributes, final DataBlock datablock) { - default BlockCodec create(final DatasetAttributes attributes, final DataCodecInfo... codecInfos) { - final DataCodec[] codecs = new DataCodec[codecInfos.length]; - Arrays.setAll(codecs, i -> codecInfos[i].create()); - return create(attributes, codecs); + return datablock.getGridPosition(); + } + + default long[] getKeyPositionForBlock(final DatasetAttributes attributes, final long... blockPosition) { + + return blockPosition; + } + + @Override default long encodedSize(long size) { + + return size; } - // TODO: Should we have both create() signatures? + @Override default long decodedSize(long size) { + + return size; + } + + BlockCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs); + + default BlockCodec create(final DatasetAttributes attributes, final DataCodecInfo... codecInfos) { + + return create(attributes.getDataType(), attributes.getBlockSize(), codecInfos); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java index 95b934643..a39032dab 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/CodecInfo.java @@ -1,11 +1,10 @@ package org.janelia.saalfeldlab.n5.codec; import java.io.Serializable; -import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.serialization.NameConfig; /** - * {@code Codec}s can encode and decode {@link ReadData} objects. + * {@code CodecInfo}s are an untyped semantic layer for {@link BlockCodec}s and {@link DataCodec}s. *

        * Modeled after Codecs in * Zarr. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDeterministicSizeDataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDeterministicSizeDataCodec.java new file mode 100644 index 000000000..99febb924 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedDeterministicSizeDataCodec.java @@ -0,0 +1,21 @@ +package org.janelia.saalfeldlab.n5.codec; + +class ConcatenatedDeterministicSizeDataCodec extends ConcatenatedDataCodec implements DeterministicSizeDataCodec { + + private final DeterministicSizeDataCodec[] codecs; + + ConcatenatedDeterministicSizeDataCodec(final DeterministicSizeDataCodec[] codecs) { + + super(codecs); + this.codecs = codecs; + } + + @Override + public long encodedSize(long size) { + + for (DeterministicSizeDataCodec codec : codecs) { + size = codec.encodedSize(size); + } + return size; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java index be40d2e59..50a11cbbf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.codec; +import java.util.Arrays; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -45,6 +46,9 @@ public interface DataCodec { /** * Create a {@code DataCodec} that sequentially applies {@code codecs} in * the given order for encoding, and in reverse order for decoding. + *

        + * If all {@code codecs} implement {@code DeterministicSizeDataCodec}, the + * returned {@code DataCodec} will also be a {@code DeterministicSizeDataCodec}. * * @param codecs * a list of DataCodecs @@ -58,6 +62,21 @@ static DataCodec concatenate(final DataCodec... codecs) { if (codecs.length == 1) return codecs[0]; - return new ConcatenatedDataCodec(codecs); + if (Arrays.stream(codecs).allMatch(DeterministicSizeDataCodec.class::isInstance)) + return new ConcatenatedDeterministicSizeDataCodec(Arrays.copyOf(codecs, codecs.length, DeterministicSizeDataCodec[].class)); + else + return new ConcatenatedDataCodec(codecs); + } + + static DataCodec create(final DataCodecInfo... codecInfos) { + + if (codecInfos == null) + throw new NullPointerException(); + + final DataCodec[] codecs = new DataCodec[codecInfos.length]; + Arrays.setAll(codecs, i -> codecInfos[i].create()); + + return DataCodec.concatenate(codecs); } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java index ca3ab8c89..44d84662d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java @@ -1,11 +1,13 @@ package org.janelia.saalfeldlab.n5.codec; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; /** * {@code DataCodec}s transform one {@link ReadData} into another, * for example, compressing it. */ +@NameConfig.Prefix("data-codec") public interface DataCodecInfo extends CodecInfo { DataCodec create(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java new file mode 100644 index 000000000..9fd9f0178 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java @@ -0,0 +1,13 @@ +package org.janelia.saalfeldlab.n5.codec; + +/** + * A {@link CodecInfo} that can deterministically determine the size of encoded data from the size of the raw data and vice versa from the data length alone (i.e. encoding is data + * independent). + */ +public interface DeterministicSizeCodecInfo extends CodecInfo { + + public abstract long encodedSize(long size); + + public abstract long decodedSize(long size); + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java new file mode 100644 index 000000000..5c533b34a --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeDataCodec.java @@ -0,0 +1,38 @@ +package org.janelia.saalfeldlab.n5.codec; + +/** + * A {@link DataCodec} that can deterministically determine the size of encoded + * data from the size of the raw data (i.e. encoding is data independent). + */ +public interface DeterministicSizeDataCodec extends DataCodec { + + /** + * Given {@code size} bytes of raw data, how many bytes will the encoded + * data have. + * + * @param size in bytes + * @return encoded size in bytes + */ + long encodedSize(long size); + + /** + * Create a {@code DeterministicSizeDataCodec} that sequentially applies + * {@code codecs} in the given order for encoding, and in reverse order for + * decoding. + * + * @param codecs + * a list of DeterministicSizeDataCodec + * @return the concatenated DeterministicSizeDataCodec + */ + static DeterministicSizeDataCodec concatenate(final DeterministicSizeDataCodec... codecs) { + + if (codecs == null) + throw new NullPointerException(); + + if (codecs.length == 1) + return codecs[0]; + + return new ConcatenatedDeterministicSizeDataCodec(codecs); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java new file mode 100644 index 000000000..046b99b67 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IdentityCodec.java @@ -0,0 +1,41 @@ +package org.janelia.saalfeldlab.n5.codec; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + +@NameConfig.Name(IdentityCodec.TYPE) +public class IdentityCodec implements DeterministicSizeDataCodec, DataCodecInfo { + + private static final long serialVersionUID = 8354269325800855621L; + + public static final String TYPE = "id"; + + @Override + public String getType() { + + return TYPE; + } + + @Override + public ReadData decode(ReadData readData) { + + return readData; + } + + @Override + public ReadData encode(ReadData readData) { + + return readData; + } + + @Override public DataCodec create() { + + return this; + } + + @Override + public long encodedSize(long size) { + + return size; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/IndexCodecAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/IndexCodecAdapter.java new file mode 100644 index 000000000..94bc66bbb --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/IndexCodecAdapter.java @@ -0,0 +1,45 @@ +package org.janelia.saalfeldlab.n5.codec; + +public class IndexCodecAdapter { + + private final BlockCodecInfo blockCodecInfo; + private final DeterministicSizeCodecInfo[] dataCodecs; + + public IndexCodecAdapter(final BlockCodecInfo blockCodecInfo, final DeterministicSizeCodecInfo... dataCodecs) { + + this.blockCodecInfo = blockCodecInfo; + this.dataCodecs = dataCodecs; + } + + public BlockCodecInfo getBlockCodecInfo() { + + return blockCodecInfo; + } + + public DataCodecInfo[] getDataCodecs() { + + final DataCodecInfo[] dataCodecs = new DataCodecInfo[this.dataCodecs.length]; + System.arraycopy(this.dataCodecs, 0, dataCodecs, 0, this.dataCodecs.length); + return dataCodecs; + } + + public long encodedSize(long initialSize) { + long totalNumBytes = initialSize; + for (DeterministicSizeCodecInfo codec : dataCodecs) { + totalNumBytes = codec.encodedSize(totalNumBytes); + } + return totalNumBytes; + } + + public static IndexCodecAdapter create(final CodecInfo... codecs) { + if (codecs == null || codecs.length == 0) + return new IndexCodecAdapter(new RawBlockCodecInfo()); + + if (codecs[0] instanceof BlockCodecInfo) + return new IndexCodecAdapter((BlockCodecInfo)codecs[0]); + + final DeterministicSizeCodecInfo[] indexCodecs = new DeterministicSizeCodecInfo[codecs.length]; + System.arraycopy(codecs, 0, indexCodecs, 0, codecs.length); + return new IndexCodecAdapter(new RawBlockCodecInfo(), indexCodecs); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java index 70c3555b3..b8946a2f4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecInfo.java @@ -1,6 +1,8 @@ package org.janelia.saalfeldlab.n5.codec; +import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @NameConfig.Name(value = N5BlockCodecInfo.TYPE) @@ -10,6 +12,25 @@ public class N5BlockCodecInfo implements BlockCodecInfo { public static final String TYPE = "n5bytes"; + private transient DatasetAttributes attributes; + + @Override public long[] getKeyPositionForBlock(DatasetAttributes attributes, DataBlock datablock) { + + return datablock.getGridPosition(); + } + + @Override public long[] getKeyPositionForBlock(DatasetAttributes attributes, long... blockPosition) { + + return blockPosition; + } + + @Override public long encodedSize(long size) { + + final int[] blockSize = attributes.getBlockSize(); + int headerSize = new N5BlockCodecs.BlockHeader(blockSize, DataBlock.getNumElements(blockSize)).getSize(); + return headerSize + size; + } + @Override public String getType() { @@ -17,8 +38,7 @@ public String getType() { } @Override - public BlockCodec create(final DatasetAttributes attributes, final DataCodec... dataCodecs) { - return N5BlockCodecs.create(attributes.getDataType(), DataCodec.concatenate(dataCodecs)); + public BlockCodec create(final DataType dataType, final int[] blockSize, final DataCodecInfo... codecInfos) { + return N5BlockCodecs.create(dataType, DataCodec.create(codecInfos)); } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java index 03e058c6f..300190d6d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5BlockCodecs.java @@ -51,6 +51,7 @@ import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.MODE_DEFAULT; import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.MODE_OBJECT; import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.MODE_VARLENGTH; +import static org.janelia.saalfeldlab.n5.codec.N5BlockCodecs.BlockHeader.headerSizeInBytes; public class N5BlockCodecs { @@ -120,9 +121,9 @@ private interface BlockCodecFactory { abstract static class N5AbstractBlockCodec implements BlockCodec { - private final FlatArrayCodec dataCodec; + final FlatArrayCodec dataCodec; private final DataBlockFactory dataBlockFactory; - private final DataCodec codec; + final DataCodec codec; N5AbstractBlockCodec(FlatArrayCodec dataCodec, DataBlockFactory dataBlockFactory, DataCodec codec) { this.dataCodec = dataCodec; @@ -189,6 +190,18 @@ protected BlockHeader decodeBlockHeader(final InputStream in) throws N5IOExcepti return BlockHeader.readFrom(in, MODE_DEFAULT, MODE_VARLENGTH); } + + @Override + public long encodedSize(final int[] blockSize) throws UnsupportedOperationException { + if (codec instanceof DeterministicSizeDataCodec) { + final int bytesPerElement = dataCodec.bytesPerElement(); + final int numElements = DataBlock.getNumElements(blockSize); + final int headerSize = headerSizeInBytes(MODE_DEFAULT, blockSize.length); + return headerSize + ((DeterministicSizeDataCodec) codec).encodedSize((long) numElements * bytesPerElement); + } else { + throw new UnsupportedOperationException(); + } + } } /** @@ -315,6 +328,25 @@ private static void writeBlockSize(final int[] blockSize, final DataOutputStream } } + static int headerSizeInBytes(final short mode, final int numDimensions) { + switch (mode) { + case MODE_DEFAULT: + return 2 + // 1 short for mode + 2 + // 1 short for blockSize.length + 4 * numDimensions; // 1 int for each blockSize element + case MODE_VARLENGTH: + return 2 +// 1 short for mode + 2 + // 1 short for blockSize.length + 4 * numDimensions + // 1 int for each blockSize dimension + 4; // 1 int for numElements + case MODE_OBJECT: + return 2 + // 1 short for mode + 4; // 1 int for numElements + default: + throw new N5Exception("unexpected mode: " + mode); + } + } + void writeTo(final OutputStream out) throws N5IOException { try { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java index ed93433f0..a9cec7003 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecInfo.java @@ -1,8 +1,15 @@ package org.janelia.saalfeldlab.n5.codec; import java.nio.ByteOrder; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.serialization.NameConfig; @@ -11,7 +18,7 @@ public class RawBlockCodecInfo implements BlockCodecInfo { private static final long serialVersionUID = 3282569607795127005L; - public static final String TYPE = "rawbytes"; + public static final String TYPE = "bytes"; @NameConfig.Parameter(value = "endian", optional = true) private final ByteOrder byteOrder; @@ -37,12 +44,12 @@ public ByteOrder getByteOrder() { } @Override - public BlockCodec create(final DatasetAttributes attributes, final DataCodec... dataCodecs) { - ensureValidByteOrder(attributes.getDataType(), getByteOrder()); - return RawBlockCodecs.create(attributes.getDataType(), byteOrder, attributes.getBlockSize(), DataCodec.concatenate(dataCodecs)); + public BlockCodec create(final DataType dataType, final int[] blockSize, final DataCodecInfo... codecInfos) { + ensureValidByteOrder(dataType, getByteOrder()); + return RawBlockCodecs.create(dataType, byteOrder, blockSize, DataCodec.create(codecInfos)); } - private static void ensureValidByteOrder(final DataType dataType, final ByteOrder byteOrder) { + public static void ensureValidByteOrder(final DataType dataType, final ByteOrder byteOrder) { switch (dataType) { case INT8: @@ -55,4 +62,32 @@ private static void ensureValidByteOrder(final DataType dataType, final ByteOrde if (byteOrder == null) throw new IllegalArgumentException("DataType (" + dataType + ") requires ByteOrder, but was null"); } + + public static ByteOrderAdapter byteOrderAdapter = new ByteOrderAdapter(); + + public static class ByteOrderAdapter implements JsonDeserializer, JsonSerializer { + + @Override + public JsonElement serialize(ByteOrder src, java.lang.reflect.Type typeOfSrc, + JsonSerializationContext context) { + + if (src.equals(ByteOrder.LITTLE_ENDIAN)) + return new JsonPrimitive("little"); + else + return new JsonPrimitive("big"); + } + + @Override + public ByteOrder deserialize(JsonElement json, java.lang.reflect.Type typeOfT, + JsonDeserializationContext context) throws JsonParseException { + + if (json.getAsString().equals("little")) + return ByteOrder.LITTLE_ENDIAN; + if (json.getAsString().equals("big")) + return ByteOrder.BIG_ENDIAN; + + return null; + } + + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java index 1fc606500..4a1ddb4ca 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBlockCodecs.java @@ -115,5 +115,16 @@ public DataBlock decode(ReadData readData, long[] gridPosition) { final T data = dataCodec.decode(decodeData, numElements); return dataBlockFactory.createDataBlock(blockSize, gridPosition, data); } + + @Override + public long encodedSize(final int[] blockSize) throws UnsupportedOperationException { + if (codec instanceof DeterministicSizeDataCodec) { + final int bytesPerElement = dataCodec.bytesPerElement(); + final int numElements = DataBlock.getNumElements(blockSize); + return ((DeterministicSizeDataCodec) codec).encodedSize((long) numElements * bytesPerElement); + } else { + throw new UnsupportedOperationException(); + } + } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java new file mode 100644 index 000000000..68b12cd0c --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumCodec.java @@ -0,0 +1,109 @@ +package org.janelia.saalfeldlab.n5.codec.checksum; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.zip.CheckedInputStream; +import java.util.zip.CheckedOutputStream; +import java.util.zip.Checksum; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodec; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +/** + * A {@link CodecInfo} that appends a checksum to data when encoding and can validate against that checksum when decoding. + */ +public abstract class ChecksumCodec implements DataCodec, DataCodecInfo, DeterministicSizeCodecInfo { + + private static final long serialVersionUID = 3141427377277375077L; + + private int numChecksumBytes; + + private Checksum checksum; + + public ChecksumCodec(Checksum checksum, int numChecksumBytes) { + + this.checksum = checksum; + this.numChecksumBytes = numChecksumBytes; + } + + public Checksum getChecksum() { + + return checksum; + } + + public int numChecksumBytes() { + + return numChecksumBytes; + } + + private CheckedOutputStream createStream(OutputStream out) { + return new CheckedOutputStream(out, getChecksum()) { + + private boolean closed = false; + @Override public void close() throws IOException { + + if (!closed) { + writeChecksum(out); + closed = true; + out.close(); + } + } + }; + } + + @Override public ReadData encode(ReadData readData) { + + return readData.encode(this::createStream); + + } + + @Override public ReadData decode(ReadData readData) throws N5IOException { + + + return ReadData.from(new CheckedInputStream(readData.inputStream(), getChecksum())); + } + + @Override + public long encodedSize(final long size) { + + return size + numChecksumBytes(); + } + + @Override + public long decodedSize(final long size) { + + return size - numChecksumBytes(); + } + + protected boolean valid(InputStream in) throws IOException { + + return readChecksum(in) == getChecksum().getValue(); + } + + protected long readChecksum(InputStream in) throws IOException { + + final byte[] checksum = new byte[numChecksumBytes()]; + in.read(checksum); + return ByteBuffer.wrap(checksum).getLong(); + } + + /** + * Return the value of the checksum as a {@link ByteBuffer} to be serialized. + * + * @return a ByteBuffer representing the checksum value + */ + public abstract ByteBuffer getChecksumValue(); + + public void writeChecksum(OutputStream out) throws IOException { + + out.write(getChecksumValue().array()); + } + + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumException.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumException.java new file mode 100644 index 000000000..034343c42 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/ChecksumException.java @@ -0,0 +1,12 @@ +package org.janelia.saalfeldlab.n5.codec.checksum; + +public class ChecksumException extends Exception { + + private static final long serialVersionUID = 905130066386622561L; + + public ChecksumException(final String message) { + + super(message); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java new file mode 100644 index 000000000..93c701ab1 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/checksum/Crc32cChecksumCodec.java @@ -0,0 +1,52 @@ +package org.janelia.saalfeldlab.n5.codec.checksum; + +import org.apache.commons.lang3.NotImplementedException; +import org.janelia.saalfeldlab.n5.codec.DataCodec; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + +import java.nio.ByteBuffer; +import java.util.zip.CRC32; + +@NameConfig.Name(Crc32cChecksumCodec.TYPE) +public class Crc32cChecksumCodec extends ChecksumCodec { + + private static final long serialVersionUID = 7424151868725442500L; + + public static final String TYPE = "crc32c"; + + public Crc32cChecksumCodec() { + + super(new CRC32(), 4); + } + + @Override + public long encodedSize(final long size) { + + return size + numChecksumBytes(); + } + + @Override + public long decodedSize(final long size) { + + return size - numChecksumBytes(); + } + + @Override + public ByteBuffer getChecksumValue() { + + final ByteBuffer buf = ByteBuffer.allocate(numChecksumBytes()); + buf.putInt((int)getChecksum().getValue()); + return buf; + } + + @Override + public String getType() { + + return TYPE; + } + + @Override public DataCodec create() { + + return this; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java b/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java index f19776906..184837b55 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java @@ -63,6 +63,20 @@ public interface NameConfig extends Serializable { String value(); } + /** + * Controls whether a class should be serializable as a {@code NameConfig}. + *

        + * This annotation allows explicitly enabling or disabling serialization for a class. + *

        + * By default, classes are serialized. + */ + @Retention(RetentionPolicy.RUNTIME) + @Inherited + @Target(ElementType.TYPE) + @Indexable @interface Serialize { + boolean value() default true; + } + /** * Marks a field as a parameter to be serialized. *

        diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java new file mode 100644 index 000000000..541c1cd8d --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -0,0 +1,29 @@ +package org.janelia.saalfeldlab.n5.shard; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; + +import java.util.List; + +/** + * Wrap an instantiated DataBlock/shard codec hierarchy to implement (single and + * batch) DataBlock read/write methods. + * + * @param + * type of the data contained in the DataBlock + */ +public interface DatasetAccess { + + DataBlock readBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; + + void writeBlock(PositionValueAccess kva, DataBlock dataBlock) throws N5IOException; + + boolean deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; + + List> readBlocks(PositionValueAccess kva, List positions); + + void writeBlocks(PositionValueAccess kva, List> blocks); + + NestedGrid getGrid(); +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java new file mode 100644 index 000000000..e14e835a1 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -0,0 +1,384 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.TreeMap; +import java.util.stream.Collectors; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; + +public class DefaultDatasetAccess implements DatasetAccess { + + private final NestedGrid grid; + private final BlockCodec[] codecs; + + public DefaultDatasetAccess(final NestedGrid grid, final BlockCodec[] codecs) { + this.grid = grid; + this.codecs = codecs; + } + + public NestedGrid getGrid() { + return grid; + } + + @Override + public DataBlock readBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { + final NestedPosition position = new NestedPosition(grid, gridPosition); + return readBlockRecursive(kva.get(position.key()), position, grid.numLevels() - 1); + } + + private DataBlock readBlockRecursive( + final ReadData readData, + final NestedPosition position, + final int level) { + if (readData == null) { + return null; + } else if (level == 0) { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[0]; + return codec.decode(readData, position.absolute(0)); + } else { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[level]; + final RawShard shard = codec.decode(readData, position.absolute(level)).getData(); + return readBlockRecursive(shard.getElementData(position.relative(level - 1)), position, level - 1); + } + } + + @Override + public List> readBlocks(PositionValueAccess pva, List positions) { + + if (grid.numLevels() == 1) { + return positions.stream().map(it -> readBlock(pva, it)).collect(Collectors.toList()); + } + + final List blockPositions = positions.stream().map(it -> new NestedPosition(grid, it)).collect(Collectors.toList()); + + final int outermostLevel = grid.numLevels() - 1; + final Collection> blocksPerOutermostShard = groupInnerPositions(grid, blockPositions, outermostLevel); + + final ArrayList> blocks = new ArrayList<>(blockPositions.size()); + for (List blocksInSingleShard : blocksPerOutermostShard) { + if (blocksInSingleShard.isEmpty()) + continue; + + final NestedPosition firstBlock = blocksInSingleShard.get(0); + final ReadData readData = pva.get(firstBlock.key()); + final List> shardBlocks; + try { + shardBlocks = readShardRecursive(readData, blocksInSingleShard, outermostLevel); + } catch (N5NoSuchKeyException e) { + continue; + } + + blocks.addAll(shardBlocks); + } + + //TODO Caleb: No guarantee of order; If we want that, we need to sort the result. + return blocks; + } + + /** + * Bulk Read operation on a shard. `positions` MUST all be in the same shard. + * That is, for each `position` in `positions`, `position.absolute(level)` must be the same. + * + * @param readData for the corresponding shard + * @param positions of blocks within the shard to be read + * @param level of the shard + * @return list of blocks read from a single shard + */ + private List> readShardRecursive( + final ReadData readData, + final List positions, + final int level + ) { + // Cannot have a shard at level 0 + if (readData == null || level == 0) { + return null; + } + + if (positions.isEmpty()) { + return Collections.emptyList(); + } + + final NestedPosition firstBlock = positions.get(0); + final long[] shardPosition = firstBlock.absolute(level); + + final BlockCodec codec = (BlockCodec) codecs[level]; + final RawShard shard = codec.decode(readData, shardPosition).getData(); + + final ArrayList> blocks = new ArrayList<>(positions.size()); + if (level == 1) { + final int innerMostLevel = 0; + //Base case; read the blocks + for (NestedPosition blockPosition : positions) { + final long[] elementPos = blockPosition.relative(innerMostLevel); + final ReadData elementData = shard.getElementData(elementPos); + final DataBlock block = readBlockRecursive(elementData, blockPosition, innerMostLevel); + blocks.add(block); + } + } else { + // group the blocks by shard for next level, and call again for each nested shard + final Collection> nextLevelShards = groupInnerPositions(grid, positions, level - 1); + for (List innerPositions : nextLevelShards) { + final List> innerBlocks = readShardRecursive(readData, innerPositions, level - 1); + blocks.addAll(innerBlocks); + } + } + + return blocks; + } + + @Override + public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException { + final NestedPosition position = new NestedPosition(grid, dataBlock.getGridPosition()); + final long[] key = position.key(); + + final ReadData existingData = getExistingReadData(pva, key); + final ReadData modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1); + pva.put(key, modifiedData); + } + + private ReadData writeBlockRecursive( + final ReadData existingReadData, + final DataBlock dataBlock, + final NestedPosition position, + final int level) { + if (level == 0) { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[0]; + return codec.encode(dataBlock); + } else { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[level]; + final long[] gridPos = position.absolute(level); + final RawShard shard = existingReadData == null ? + new RawShard(grid.relativeBlockSize(level)) : + codec.decode(existingReadData, gridPos).getData(); + final long[] elementPos = position.relative(level - 1); + final ReadData existingElementData = (level == 1) + ? null // if level == 1, we don't need to extract the nested (DataBlock) ReadData because it will be overridden anyway + : shard.getElementData(elementPos); + final ReadData modifiedElementData = writeBlockRecursive(existingElementData, dataBlock, position, level - 1); + shard.setElementData(modifiedElementData, elementPos); + return codec.encode(new RawShardDataBlock(gridPos, shard)); + } + } + + @Override public void writeBlocks(PositionValueAccess pva, List> dataBlocks) { + + if (grid.numLevels() == 1) { + dataBlocks.forEach(it -> writeBlock(pva, it)); + return; + } + + final List> nestedPositionDataBlocks = dataBlocks.stream() + .map(it -> new NestedPositionDataBlock<>(grid, it)) + .collect(Collectors.toList()); + + final int outermostLevel = grid.numLevels() - 1; + final Collection>> dataBlocksPerShard = groupInnerPositions(grid, nestedPositionDataBlocks, outermostLevel); + + for (List> dataBlocksForShard : dataBlocksPerShard) { + if (dataBlocksForShard.isEmpty()) + continue; + + final NestedPositionDataBlock firstDataBlock = dataBlocksForShard.get(0); + final long[] shardKey = firstDataBlock.key(); + + //TODO Caleb: When writing all blocks in a shard, we don't need to read existing data. + // Also, could only materialize the index and skip reading if we are overwriting all existing blocks. + final ReadData existingReadData = getExistingReadData(pva, shardKey); + final ReadData writeShardReadData = writeShardRecursive(existingReadData, dataBlocksForShard, outermostLevel); + + pva.put(shardKey, writeShardReadData); + } + } + + private ReadData writeShardRecursive( + final ReadData existingShard, + final List> dataBlocks, + final int level) { + + // cannot have shard level 0, or nothing to write + if (level == 0 || dataBlocks.isEmpty()) { + return null; + } + + final BlockCodec codec = (BlockCodec)codecs[level]; + final NestedPositionDataBlock firstBlock = dataBlocks.get(0); + final long[] shardPosition = firstBlock.absolute(level); + + final RawShard shard; + if (existingShard == null) + shard = new RawShard(grid.relativeBlockSize(level)); + else + shard = codec.decode(existingShard, shardPosition).getData(); + + if (level == 1) { + final int innerMostLevel = 0; + // Base case, write the blocks + for (NestedPositionDataBlock nestedPosDataBlock : dataBlocks) { + final DataBlock dataBlock = nestedPosDataBlock.getDataBlock(); + + final long[] blockRelativePos = nestedPosDataBlock.relative(innerMostLevel); + final ReadData blockReadData = shard.getElementData(blockRelativePos); + final ReadData modifiedShardBlock = writeBlockRecursive(blockReadData, dataBlock, nestedPosDataBlock, innerMostLevel); + shard.setElementData(modifiedShardBlock, blockRelativePos); + } + } else { + final Collection>> dataBlocksForInnerShards = groupInnerPositions(grid, dataBlocks, level - 1); + for (List> innerShardDataBlocks : dataBlocksForInnerShards) { + if (innerShardDataBlocks.isEmpty()) + continue; + + final ReadData innerShardReadData = writeShardRecursive(existingShard, innerShardDataBlocks, level - 1); + + final NestedPositionDataBlock firstInnerNestedPosBlock = innerShardDataBlocks.get(0); + final long[] relPosInShard = firstInnerNestedPosBlock.relative(level - 1); + shard.setElementData(innerShardReadData, relPosInShard); + } + } + return codec.encode(new RawShardDataBlock(shardPosition, shard)); + } + + @Override + public boolean deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { + final NestedPosition position = new NestedPosition(grid, gridPosition); + final long[] key = position.key(); + if (grid.numLevels() == 1) { + // for non-sharded dataset, don't bother getting the value, just remove the key. + try { + return kva.remove(key); + } catch (final Exception e) { + throw new N5Exception("The shard at " + Arrays.toString(key) + " could not be deleted.", e); + } + } else { + final ReadData existingData = kva.get(key); + final ReadData modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); + if (existingData != null && modifiedData == null) { + return kva.remove(key); + } else if (modifiedData != existingData) { + kva.put(key, modifiedData); + return true; + } else { + return false; + } + } + } + + private ReadData deleteBlockRecursive( + final ReadData existingReadData, + final NestedPosition position, + final int level) { + if (level == 0 || existingReadData == null) { + return null; + } else { + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[level]; + final long[] gridPos = position.absolute(level); + final RawShard shard = codec.decode(existingReadData, gridPos).getData(); + final long[] elementPos = position.relative(level - 1); + final ReadData existingElementData = shard.getElementData(elementPos); + if (existingElementData == null) { + // The DataBlock (or the whole nested shard containing it) does not exist. + // This shard remains unchanged. + return existingReadData; + } else { + final ReadData modifiedElementData = deleteBlockRecursive(existingElementData, position, level - 1); + if (modifiedElementData == existingElementData) { + // The nested shard was not modified. + // This shard remains unchanged. + return existingReadData; + } + shard.setElementData(modifiedElementData, elementPos); + if (modifiedElementData == null) { + // The DataBlock or nested shard was removed. + // Check whether this shard becomes empty. + if (shard.index().allElementsNull()) { + // This shard is empty and should be removed. + return null; + } + } + return codec.encode(new RawShardDataBlock(gridPos, shard)); + } + } + } + + private static ReadData getExistingReadData(final PositionValueAccess pva, final long[] key) { + // need to read the shard anyway, and currently (Sept 24 2025) + // have no way to tell if they key exist from what is in this method except to attempt + // to materialize and catch the N5NoSuchKeyException + try { + ReadData existingData = pva.get(key); + if (existingData != null) + existingData.materialize(); + return existingData; + } catch (N5NoSuchKeyException e) { + return null; + } + } + + /** + * Sort a list of {@link NestedPosition}s by their parent level {@link NestedPosition}. + * nestedPositions are grouped at `outerLevel`. + * If {@code NestedPosition.level()} level is already equivalent to {@code NestedGrid.numLevels() - 1}, nestedPositions is returned. + * + * @param grid to sort the blocky by + * @param nestedPositions to group per shard. + * @param outerLevel of the outerLevel shard position to group by. must be in range {@code [1, NestedGrid.numLevels() - 1]} + * @return map of outerLevel shard positions to inner level block positions + */ + private static Collection> groupInnerPositions(final NestedGrid grid, final List nestedPositions, final int outerLevel) { + + if (outerLevel < 1 || outerLevel >= grid.numLevels()) + throw new IllegalArgumentException("outerLevel must be in range [1, grid.numLevels() - 1]"); + + if (nestedPositions.isEmpty()) + return Collections.emptyList(); + + final TreeMap> blocksPerShard = new TreeMap<>(); + for (T nestedPosition : nestedPositions) { + final NestedPosition outerNestedPosition; + if (nestedPosition.level() == outerLevel) + outerNestedPosition = nestedPosition; + else + outerNestedPosition = new NestedPosition(grid, nestedPosition.absolute(0), outerLevel); + + + final List blocks = blocksPerShard.computeIfAbsent(outerNestedPosition, it -> new ArrayList<>()); + blocks.add(nestedPosition); + } + return blocksPerShard.values(); + } + + /** + * NestedPosition wrapper for a DataBlock. Useful for grouping DataBlock by nested shard position. + * + * @param type of the datablock + */ + private static class NestedPositionDataBlock extends NestedPosition { + + private final DataBlock dataBlock; + + private NestedPositionDataBlock(NestedGrid grid, DataBlock dataBlock) { + + super(grid, dataBlock.getGridPosition(), 0); + this.dataBlock = dataBlock; + } + + private DataBlock getDataBlock() { + + return dataBlock; + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java new file mode 100644 index 000000000..7b7cc0473 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultShardCodecInfo.java @@ -0,0 +1,136 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.Arrays; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.serialization.N5Annotations; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; + +/** + * Default (and probably only) implementation of {@link ShardCodecInfo}. + */ +@NameConfig.Name(value = "sharding_indexed") +public class DefaultShardCodecInfo implements ShardCodecInfo { + + @Override + public String getType() { + return "sharding_indexed"; + } + + @N5Annotations.ReverseArray + @NameConfig.Parameter(value = "chunk_shape") + private final int[] innerBlockSize; + + @NameConfig.Parameter(value = "index_location") + private final IndexLocation indexLocation; + + @NameConfig.Parameter + private CodecInfo[] codecs; + + @NameConfig.Parameter(value = "index_codecs") + private CodecInfo[] indexCodecs; + + private transient final BlockCodecInfo innerBlockCodecInfo; + + private transient final DataCodecInfo[] innerDataCodecInfos; + + private transient final BlockCodecInfo indexBlockCodecInfo; + + private transient final DataCodecInfo[] indexDataCodecInfos; + + DefaultShardCodecInfo() { + // for serialization + this(null, null, null, null, null, null); + } + + public DefaultShardCodecInfo( + final int[] innerBlockSize, + final BlockCodecInfo innerBlockCodecInfo, + final DataCodecInfo[] innerDataCodecInfos, + final BlockCodecInfo indexBlockCodecInfo, + final DataCodecInfo[] indexDataCodecInfos, + final IndexLocation indexLocation) { + + this.innerBlockSize = innerBlockSize; + this.innerBlockCodecInfo = innerBlockCodecInfo; + this.innerDataCodecInfos = innerDataCodecInfos; + this.indexBlockCodecInfo = indexBlockCodecInfo; + this.indexDataCodecInfos = indexDataCodecInfos; + this.indexLocation = indexLocation; + + codecs = concatenateCodecs(innerBlockCodecInfo, innerDataCodecInfos); + indexCodecs = concatenateCodecs(indexBlockCodecInfo, indexDataCodecInfos); + } + + @Override + public int[] getInnerBlockSize() { + return innerBlockSize; + } + + @Override + public BlockCodecInfo getInnerBlockCodecInfo() { + return innerBlockCodecInfo; + } + + @Override + public DataCodecInfo[] getInnerDataCodecInfos() { + return innerDataCodecInfos; + } + + @Override + public BlockCodecInfo getIndexBlockCodecInfo() { + return indexBlockCodecInfo; + } + + @Override + public DataCodecInfo[] getIndexDataCodecInfos() { + return indexDataCodecInfos; + } + + @Override + public IndexLocation getIndexLocation() { + return indexLocation; + } + + public CodecInfo[] getCodecs() { + return codecs; + } + + public CodecInfo[] getIndexCodecs() { + return indexCodecs; + } + + @Override + public RawShardCodec create(final int[] blockSize, final DataCodecInfo... codecs) { + + // Number of elements (DataBlocks, nested shards) in each dimension per shard. + final int[] size = new int[blockSize.length]; + // blockSize argument is number of pixels in the shard + // innerBlockSize is number of pixels in each shard element (nested shard or DataBlock) + Arrays.setAll(size, d -> blockSize[d] / innerBlockSize[d]); + + final BlockCodec indexCodec = indexBlockCodecInfo.create( + DataType.UINT64, + ShardIndex.blockSizeFromIndexSize(size), + indexDataCodecInfos); + + return new RawShardCodec(size, indexLocation, indexCodec); + } + + private static CodecInfo[] concatenateCodecs(BlockCodecInfo blkInfo, DataCodecInfo[] dataInfos) { + + if (dataInfos == null) { + return new CodecInfo[]{blkInfo}; + } + + final CodecInfo[] allCodecs = new CodecInfo[dataInfos.length + 1]; + allCodecs[0] = blkInfo; + System.arraycopy(dataInfos, 0, allCodecs, 1, dataInfos.length); + + return allCodecs; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java new file mode 100644 index 000000000..892900ae4 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java @@ -0,0 +1,347 @@ +package org.janelia.saalfeldlab.n5.shard; + + +import java.util.ArrayList; + +// TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// +// [ ] NestedGrid +// [ ] validation in constructor +// [ ] test for that validation +// [ ] javadoc +// +// [ ] NestedPosition interface +// [+] LazyNestedPosition class +// [+] fields: NestedGrid, long[] position, int level +// [+] construct with source level 0 +// [+] minimal abs/rel access methods +// [+] toString() +// [-] extract NestedPosition interface +// ==> postpone until necessary +// [ ] equals / hashcode +// [ ] should we have prefix()? suffix()? head()? tail()? +// [+] Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...). +// For nested = {X,Y,Z} compare by Z, then Y, then X. +// For X = {x,y,z} compare by z, then y, then x. (flattening order) +// +// TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +import java.util.Arrays; +import java.util.List; + +import org.janelia.saalfeldlab.n5.util.GridIterator; + +public class Nesting { + + public static void main(String[] args) { + final int[][] blockSizes = {{1,1,1}, {3,2,2}, {6,4,4}, {24,24,24}}; + final NestedGrid grid = new NestedGrid(blockSizes); + final NestedPosition pos = new NestedPosition(grid, new long[] {38, 7, 129}); + System.out.println("pos = " + pos); + System.out.println("key = " + Arrays.toString(pos.key())); + } + + public static class NestedPosition implements Comparable { + + private final NestedGrid grid; + private final long[] position; + private final int level; + + public NestedPosition(final NestedGrid grid, final long[] position, final int level) { + this.grid = grid; + this.position = position; + this.level = level; + } + + public NestedPosition(final NestedGrid grid, final long[] position) { + this(grid, position, 0); + } + + /** + * Get the nesting level of this position. + *

        + * Positions with {@code level=0} refer to DataBlocks, positions with + * {@code level=1} refer to first-level shards (containing DataBlocks), + * and so on. + * + * @return nesting level + */ + public int level() { + return level; + } + + public int numDimensions() { + return grid.numDimensions(); + } + + /** + * Get the relative grid position at {@code level}, that is, relative + * offset within containing the {@code (level+1)} element. + * + * @param level + * requested nesting level + * + * @return relative grid position + */ + public long[] relative(final int level) { + return grid.relativePosition(position, level); + } + + /** + * Get the absolute grid position at {@code level}. + * + * @param level + * requested nesting level + * + * @return absolute grid position + */ + public long[] absolute(final int level) { + return grid.absolutePosition(position, level); + } + + public long[] key() { + return relative(grid.numLevels() - 1); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append('{'); + for ( int l = level; l < grid.numLevels(); ++l ) { + if ( l > level ) { + sb.append(" / "); + } + sb.append(Arrays.toString(relative(l))); + } + sb.append(" (level ").append(level).append(")}"); + return sb.toString(); + } + + @Override public int compareTo(NestedPosition o) { + + final int dimensionInequality = Integer.compare(numDimensions(), o.numDimensions()); + if (dimensionInequality != 0) + return dimensionInequality; + + final int levelInequality = Integer.compare(level, o.level); + if (levelInequality != 0) + return levelInequality; + + final long[] otherAbsPos = o.absolute(level); + final long[] absPos = absolute(level); + + for (int i = absPos.length - 1; i >= 0; --i) { + final long diff = absPos[i] - otherAbsPos[i]; + if (diff != 0) + return (int)diff; + } + + return 0; + } + + // TODO: equals() and hashCode() + // TODO: should we have prefix()? suffix()? head()? tail()? + } + + /** + * A nested grid of blocks used to coordinate the relationships of shards and the blocks / chunks they contain. + */ + public static class NestedGrid { + + private final int numLevels; + + private final int numDimensions; + + // relativeToBase[i][d] is block size at level i relative to level 0 + private final int[][] relativeToBase; + + // relativeToAdjacent[i][d] is block size at level i relative to level i-1 + private final int[][] relativeToAdjacent; + + private final int[][] blockSizes; + + /** + * {@code blockSizes[l][d]} is the block size at level {@code l} in dimension {@code d}. + * Level 0 contains the smallest blocks. blockSizes[l+1][d] must be a multiple of blockSizes[l][d]. + * + * @param blockSizes + * block sizes for all levels and dimensions. + */ + public NestedGrid(int[][] blockSizes) { + + if (blockSizes == null) + throw new IllegalArgumentException("blockSizes is null"); + + if (blockSizes[0] == null) + throw new IllegalArgumentException("blockSizes[0] is null"); + + this.blockSizes = blockSizes; + + numLevels = blockSizes.length; + numDimensions = blockSizes[0].length; + relativeToBase = new int[numLevels][numDimensions]; + relativeToAdjacent = new int[numLevels][numDimensions]; + for (int l = 0; l < numLevels; ++l) { + final int k = Math.max(0, l - 1); + + if (blockSizes[l] == null) + throw new IllegalArgumentException("blockSizes[" + l + "] null"); + + if (blockSizes[l].length != numDimensions) + throw new IllegalArgumentException( + String.format("Block size at level %d has a different length (%d vs %d)", l, numDimensions, blockSizes[l].length)); + + for (int d = 0; d < numDimensions; ++d) { + + if (blockSizes[l][d] <= 0 ) { + throw new IllegalArgumentException( + String.format("Block sizes at level %d (%d) is negative for dimension %d.", + l, blockSizes[l][d], d)); + } + + if (blockSizes[l][d] < blockSizes[k][d]) { + throw new IllegalArgumentException( + String.format("Block sizes at level %d (%d) is smaller than previous level (%d) " + + " for dimension %d.", + l, blockSizes[l][d], blockSizes[k][d], d)); + } + + if (blockSizes[l][d] % blockSizes[k][d] != 0) { + throw new IllegalArgumentException( + String.format("Block sizes at level %d (%d) not a multiple of previous level (%d) " + + " for dimension %d.", + l, blockSizes[l][d], blockSizes[k][d], d)); + } + + relativeToBase[l][d] = blockSizes[l][d] / blockSizes[0][d]; + relativeToAdjacent[l][d] = blockSizes[l][d] / blockSizes[k][d]; + } + } + } + + public int numLevels() { + return numLevels; + } + + public int numDimensions() { + return numDimensions; + } + + public int[] getBlockSize(int level) { + return blockSizes[level]; + } + + public void absolutePosition( + final long[] sourcePos, + final int sourceLevel, + final long[] targetPos, + final int targetLevel) { + final int[] sk = relativeToBase[sourceLevel]; + final int[] si = relativeToBase[targetLevel]; + for (int d = 0; d < numDimensions; ++d) { + targetPos[d] = sourcePos[d] * sk[d] / si[d]; + } + } + + public void relativePosition( + final long[] sourcePos, + final int sourceLevel, + final long[] targetPos, + final int targetLevel) { + absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel); + if (targetLevel < numLevels - 1) { + final int[] rj = relativeToAdjacent[targetLevel + 1]; + for (int d = 0; d < numDimensions; ++d) { + targetPos[d] %= rj[d]; + } + } + } + + /** + * The absolute position of * source position for the given source level at the target level. + * + * @param sourcePos the source position j + * @param sourceLevel the source level + * @param targetLevel the target level + * @return absolute position at the target level + */ + public long[] absolutePosition( + final long[] sourcePos, + final int sourceLevel, + final int targetLevel) { + final long[] targetPos = new long[numDimensions]; + absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel); + return targetPos; + } + + /** + * The absolute position of the level 0 source position at + * the target level. + * + * @param sourcePos the source position j + * @param targetLevel the target level + * @return absolute position at the target level + */ + public long[] absolutePosition( + final long[] sourcePos, + final int targetLevel) { + return absolutePosition(sourcePos, 0, targetLevel); + } + + public long[] relativePosition( + final long[] sourcePos, + final int sourceLevel, + final int targetLevel) { + final long[] targetPos = new long[numDimensions]; + relativePosition(sourcePos, sourceLevel, targetPos, targetLevel); + return targetPos; + } + + public long[] relativePosition( + final long[] sourcePos, + final int targetLevel) { + return relativePosition(sourcePos, 0, targetLevel); + } + + public int[] relativeBlockSize(final int level) { + return relativeToAdjacent[level]; + } + + public int[] absoluteBlockSize(final int level) { + return relativeToBase[level]; + } + + /** + * Given a block position at a particular level, returns a list of + * positions of all sub-blocks at a particular subLevel. + *

        + * Can be used to get a list of chunk positions for a shard with a + * particular position. + * + * @param position + * a position + * @param level + * the nesting level of the given position + * @param subLevel + * the nesting sub-level of positions to return + * @return the sub-block positions + */ + public List positionInSubGrid(long[] position, int level, int subLevel) { + + final long[] subPosition = new long[numDimensions()]; + absolutePosition(position, level, subPosition, subLevel); + + final int[] numElementsInSubGrid = absoluteBlockSize(numLevels() - 1); + final GridIterator git = new GridIterator(GridIterator.int2long(numElementsInSubGrid), subPosition); + + // TODO return NestedPositions instead? + final ArrayList positions = new ArrayList<>(); + while (git.hasNext()) + positions.add(git.next().clone()); + + return positions; + } + + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java new file mode 100644 index 000000000..f39819067 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java @@ -0,0 +1,100 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; + +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +/** + * Idea is to wrap a KeyValueAccess and a dataset URI to be able to get/put values (ReadData) by {@code long[]} key + */ +public interface PositionValueAccess { + + /** + * @return ReadData for the given key or {@code null} if the key doesn't exist + */ + ReadData get(long[] key) throws N5Exception.N5IOException; + + void put(long[] key, ReadData data) throws N5Exception.N5IOException; + + boolean remove(long[] key) throws N5Exception.N5IOException; + + public static PositionValueAccess fromKva( + final KeyValueAccess kva, + final URI uri, + final String normalPath, + final DatasetAttributes attributes) { + + return new KvaPositionValueAccess(kva, uri, normalPath, attributes); + } + + class KvaPositionValueAccess implements PositionValueAccess { + + private final KeyValueAccess kva; + private final URI uri; + private final String normalPath; + private final DatasetAttributes attributes; + + KvaPositionValueAccess(final KeyValueAccess kva, + final URI uri, + final String normalPath, + final DatasetAttributes attributes) { + + this.kva = kva; + this.uri = uri; + this.normalPath = normalPath; + this.attributes = attributes; + } + + /** + * Constructs the path for a shard or data block at a given + * grid position. + *
        + * If the gridPosition passed in refers to shard position in a sharded + * dataset, this will return the path to the shard key. + * + * @param normalPath + * normalized dataset path + * @param gridPosition + * to the target data block + * @return the absolute path to the data block ad gridPosition + */ + protected String absolutePath( final long... gridPosition) { + return kva.compose(uri, normalPath, attributes.relativeBlockPath(gridPosition)); + } + + @Override + public ReadData get(long[] key) throws N5IOException { + return kva.createReadData(absolutePath(key)); + } + + @Override + public void put(long[] key, ReadData data) throws N5IOException { + + try ( final LockedChannel ch = kva.lockForWriting(absolutePath(key)); + final OutputStream outputStream = ch.newOutputStream();) { + data.writeTo(outputStream); + } catch (IOException e) { + throw new N5IOException(e); + } + } + + @Override + public boolean remove(long[] gridPosition) throws N5IOException { + + final String key = absolutePath(gridPosition); + if (!kva.isFile(key)) + return false; + + kva.delete(key); + return true; + } + + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java new file mode 100644 index 000000000..690b3d180 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java @@ -0,0 +1,52 @@ +package org.janelia.saalfeldlab.n5.shard; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.Segment; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.NDArray; + +public class RawShard { + + private final SegmentedReadData sourceData; + + private final NDArray index; + + RawShard(final int[] size) { + sourceData = null; + index = new NDArray<>(size, Segment[]::new); + } + + RawShard(final SegmentedReadData sourceData, final NDArray index) { + this.sourceData = sourceData; + this.index = index; + } + + RawShard(final ShardIndex.SegmentIndexAndData segmentIndexAndData) { + this(segmentIndexAndData.data(), segmentIndexAndData.index()); + } + + /** + * The ReadData from which the shard was constructed, or {@code null} + * for a new empty shard. + */ + public SegmentedReadData sourceData() { + return sourceData; + } + + /** + * Maps grid position of shard elements to {@link Segment}s. + */ + public NDArray index() { + return index; + } + + public ReadData getElementData(final long[] pos) { + final Segment segment = index.get(pos); + return segment == null ? null : segment.source().slice(segment); + } + + public void setElementData(final ReadData data, final long[] pos) { + final Segment segment = data == null ? null : SegmentedReadData.wrap(data).segments().get(0); + index.set(segment, pos); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java new file mode 100644 index 000000000..8ba859336 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java @@ -0,0 +1,83 @@ +package org.janelia.saalfeldlab.n5.shard; + +import static org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation.START; + +import java.util.ArrayList; +import java.util.List; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.codec.BlockCodec; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.Segment; +import org.janelia.saalfeldlab.n5.readdata.Range; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.NDArray; + +public class RawShardCodec implements BlockCodec { + + /** + * Number of elements (DataBlocks, nested shards) in each dimension per shard. + */ + private final int[] size; + private final IndexLocation indexLocation; + private final BlockCodec indexCodec; + private final long indexBlockSizeInBytes; + + RawShardCodec(final int[] size, final IndexLocation indexLocation, final BlockCodec indexCodec) { + + this.size = size; + this.indexLocation = indexLocation; + this.indexCodec = indexCodec; + indexBlockSizeInBytes = indexCodec.encodedSize(ShardIndex.blockSizeFromIndexSize(size)); + } + + @Override + public ReadData encode(final DataBlock shard) throws N5Exception.N5IOException { + + // concatenate slices for all non-null segments in shard.getData().index() + final NDArray index = shard.getData().index(); + final List readDatas = new ArrayList<>(); + // TODO: Any clever ReadData grouping, slice merging, etc. should go here + // This basic implementation just slices ReadData for all non-null + // elements and concatenates in flat index order. + for (Segment segment : index.data) { + if (segment != null) { + readDatas.add(segment.source().slice(segment)); + } + } + final SegmentedReadData data = SegmentedReadData.concatenate(readDatas); + + final ReadData.OutputStreamWriter writer; + if (indexLocation == START) { + data.materialize(); + final NDArray locations = ShardIndex.locations(index, data); + final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, indexBlockSizeInBytes); + final ReadData indexReadData = indexCodec.encode(indexDataBlock); + writer = out -> { + indexReadData.writeTo(out); + data.writeTo(out); + }; + } else { // indexLocation == END + writer = out -> { + data.writeTo(out); + final NDArray locations = ShardIndex.locations(index, data); + final DataBlock indexDataBlock = ShardIndex.toDataBlock(locations, 0); + final ReadData indexReadData = indexCodec.encode(indexDataBlock); + indexReadData.writeTo(out); + }; + } + return ReadData.from(writer); + } + + @Override + public DataBlock decode(final ReadData readData, final long[] gridPosition) throws N5Exception.N5IOException { + + final long indexOffset = (indexLocation == START) ? 0 : (readData.requireLength() - indexBlockSizeInBytes); + final ReadData indexReadData = readData.slice(indexOffset, indexBlockSizeInBytes); + final DataBlock indexDataBlock = indexCodec.decode(indexReadData, new long[size.length]); + final NDArray locations = ShardIndex.fromDataBlock(indexDataBlock); + final ShardIndex.SegmentIndexAndData segments = ShardIndex.segments(locations, readData); + return new RawShardDataBlock(gridPosition, new RawShard(segments)); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java new file mode 100644 index 000000000..e36017ffd --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java @@ -0,0 +1,42 @@ +package org.janelia.saalfeldlab.n5.shard; + +import org.janelia.saalfeldlab.n5.DataBlock; + +/** + * Wrap a RawShard as a DataBlock. + * This basically just adds a gridPosition for the shard. + */ +public class RawShardDataBlock implements DataBlock { + + private final long[] gridPosition; + + private final RawShard shard; + + RawShardDataBlock(final long[] gridPosition, final RawShard shard) { + this.gridPosition = gridPosition; + this.shard = shard; + } + + // TODO: should this be the number of elements in the Shard (number of + // sub-shards / datablock) along each dimension, or the number of + // pixels alon each dimension? + @Override + public int[] getSize() { + return shard.index().size(); + } + + @Override + public long[] getGridPosition() { + return gridPosition; + } + + @Override + public int getNumElements() { + return shard.index().numElements(); + } + + @Override + public RawShard getData() { + return shard; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java new file mode 100644 index 000000000..0f2d2150f --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java @@ -0,0 +1,44 @@ +package org.janelia.saalfeldlab.n5.shard; + +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; + +public interface ShardCodecInfo extends BlockCodecInfo { + + /** + * Chunk size of each shard element (either nested shard or DataBlock) + */ + int[] getInnerBlockSize(); + + /** + * BlockCodec for shard elements (either nested shard or DataBlock) + */ + BlockCodecInfo getInnerBlockCodecInfo(); + + /** + * DataCodecs for inner BlockCodec + */ + DataCodecInfo[] getInnerDataCodecInfos(); + + /** + * BlockCodec for shard index + */ + BlockCodecInfo getIndexBlockCodecInfo(); + + /** + * Deterministic-size DataCodecs for index BlockCodec + */ + DataCodecInfo[] getIndexDataCodecInfos(); + + IndexLocation getIndexLocation(); + + @SuppressWarnings("unchecked") + @Override + default RawShardCodec create(DataType dataType, int[] blockSize, DataCodecInfo... codecs) { + return create(blockSize, codecs); + } + + RawShardCodec create(int[] blockSize, DataCodecInfo... codecs); +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java new file mode 100644 index 000000000..d5d25eda7 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardIndex.java @@ -0,0 +1,238 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.function.IntFunction; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.Segment; +import org.janelia.saalfeldlab.n5.readdata.Range; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; +import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData.SegmentsAndData; + +import com.google.gson.annotations.SerializedName; + +public class ShardIndex { + + private ShardIndex() { + // utility class. should not be instantiated. + } + + public enum IndexLocation { + @SerializedName("start") START, + @SerializedName("end") END + } + + /** + * Access flat {@code T[]} array as n-dimensional array. + * + * @param + * element type + */ + public static class NDArray { + + final int[] size; + private final int[] stride; + final T[] data; + + NDArray(final int[] size, final IntFunction createArray) { + this.size = size; + stride = getStrides(size); + data = createArray.apply(getNumElements(size)); + } + + NDArray(final int[] size, final T[] data) { + this.size = size; + stride = getStrides(size); + this.data = data; + } + + T get(long... position) { + return data[index(position)]; + } + + void set(T value, long... position) { + data[index(position)] = value; + } + + private int index(long... position) { + int index = 0; + for (int i = 0; i < stride.length; i++) { + index += stride[i] * position[i]; + } + return index; + } + + public int[] size() { + return size; + } + + public int numElements() { + return data.length; + } + + public boolean allElementsNull() { + for (T t : data) { + if (t != null) { + return false; + } + } + return true; + } + } + + static int getNumElements(final int[] size) { + int numElements = 1; + for (int s : size) { + numElements *= s; + } + return numElements; + } + + static int[] getStrides(final int[] size) { + final int n = size.length; + final int[] stride = new int[n]; + stride[0] = 1; + for (int i = 1; i < n; i++) { + stride[i] = stride[i - 1] * size[i - 1]; + } + return stride; + } + + /** + * Special value indicating an empty block entry in the index. + * Used for both offset and length when a block doesn't exist. + */ + static final long EMPTY_INDEX_NBYTES = 0xFFFFFFFFFFFFFFFFL; + + /** + * Size of first dimension of the {@code DataBlock} representation of the shard index. + */ + private static final int LONGS_PER_BLOCK = 2; + + static NDArray fromDataBlock( final DataBlock block ) { + + final long[] blockData = block.getData(); + final int[] size = indexSizeFromBlockSize(block.getSize()); + final int n = getNumElements(size); + final Range[] locations = new Range[n]; + + for (int i = 0; i < n; i++) { + long offset = blockData[i * LONGS_PER_BLOCK]; + long length = blockData[i * LONGS_PER_BLOCK + 1]; + if (offset != EMPTY_INDEX_NBYTES && length != EMPTY_INDEX_NBYTES) { + locations[i] = Range.at(offset, length); + } + } + return new NDArray<>(size, locations); + } + + static DataBlock toDataBlock( final NDArray locations, final long offset ) { + + final Range[] data = locations.data; + + final int[] blockSize = blockSizeFromIndexSize(locations.size); + final long[] blockData = new long[data.length * 2]; + + for (int i = 0; i < data.length; ++i) { + if (data[i] != null) { + blockData[i * LONGS_PER_BLOCK] = data[i].offset() + offset; + blockData[i * LONGS_PER_BLOCK + 1] = data[i].length(); + } else { + blockData[i * LONGS_PER_BLOCK] = EMPTY_INDEX_NBYTES; + blockData[i * LONGS_PER_BLOCK + 1] = EMPTY_INDEX_NBYTES; + } + } + return new LongArrayDataBlock(blockSize, new long[blockSize.length], blockData); + } + + /** + * Prepends a value to an array. + * + * @param value the value to prepend + * @param array the original array + * @return a new array with the value prepended + */ + private static int[] prepend(final int value, final int[] array) { + + final int[] indexBlockSize = new int[array.length + 1]; + indexBlockSize[0] = value; + System.arraycopy(array, 0, indexBlockSize, 1, array.length); + return indexBlockSize; + } + + /** + * Prepends {@code LONGS_PER_BLOCK} to the {@code indexSize} array. + */ + static int[] blockSizeFromIndexSize(final int[] indexSize) { + return prepend(LONGS_PER_BLOCK, indexSize); + } + + /** + * Strips first element (should be {@code LONGS_PER_BLOCK} from the {@code blockSize} array. + */ + static int[] indexSizeFromBlockSize(final int[] blockSize) { + assert blockSize[ 0 ] == LONGS_PER_BLOCK; + return Arrays.copyOfRange(blockSize, 1, blockSize.length); + } + + /** + * Retrieves the {@code SegmentLocation} of each non-null {@code Segment} in + * {@code segments}. Returns a {@code NDArray} with entries + * corresponding tho the {@code segments} entries. + */ + static NDArray locations(final NDArray segments, final SegmentedReadData readData) { + + final Segment[] data = segments.data; + final Range[] locations = new Range[data.length]; + for (int i = 0; i < data.length; ++i) { + final Segment segment = data[i]; + if ( segment != null ) { + locations[i] = readData.location(segment); + } + } + return new NDArray<>(segments.size, locations); + } + + interface SegmentIndexAndData { + NDArray index(); + SegmentedReadData data(); + } + + /** + * Puts a {@code Segment} at each non-null {@code SegmentLocation} in {@code + * locations} on the given {@code readData}. Returns both the {@code + * SegmentedReadData} with these segments and a {@code NDArray} + * with segment entries corresponding to the {@code locations} entries. + */ + static SegmentIndexAndData segments(final NDArray locations, final ReadData readData) { + + final Range[] locationsData = locations.data; + final Segment[] segmentsData = new Segment[locationsData.length]; + + final List presentLocations = new ArrayList<>(); + for (int i = 0; i < locationsData.length; i++) { + if (locationsData[i] != null) { + presentLocations.add(locationsData[i]); + } + } + + final SegmentsAndData segmentsAndData = SegmentedReadData.wrap(readData, presentLocations); + final Iterator presentSegments = segmentsAndData.segments().iterator(); + for (int i = 0; i < locationsData.length; i++) { + if (locationsData[i] != null) { + segmentsData[i] = presentSegments.next(); + } + } + + final NDArray index = new NDArray<>(locations.size, segmentsData); + final SegmentedReadData data = segmentsAndData.data(); + return new SegmentIndexAndData() { + @Override public NDArray index() {return index;} + @Override public SegmentedReadData data() {return data;} + }; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java b/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java new file mode 100644 index 000000000..1b7076d54 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java @@ -0,0 +1,38 @@ +package org.janelia.saalfeldlab.n5.util; + +/* + * An immutable {@Position}. + */ +public class FinalPosition implements Position { + + public final long[] position; + + public FinalPosition(long[] position) { + this.position = position; + } + + public FinalPosition(Position p) { + this.position = p.get().clone(); + } + + @Override + public long[] get() { + return position; + } + + @Override + public long get(int i) { + return position[i]; + } + + @Override + public String toString() { + return Position.toString(this); + } + + @Override + public boolean equals(Object obj) { + return Position.equals(this, obj); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java new file mode 100644 index 000000000..43258efaf --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java @@ -0,0 +1,178 @@ +package org.janelia.saalfeldlab.n5.util; + +import java.util.Iterator; + +/** + * Essentially imglib2's IntervalIterator, but N5 does not depend on imglib2. + */ +public class GridIterator implements Iterator { + + final protected long[] dimensions; + + final protected long[] steps; + + final protected long[] position; + + final protected int[] intPosition; + + final protected long[] min; + + final protected int lastIndex; + + protected int index = -1; + + public GridIterator(final long[] dimensions, final long[] min) { + + final int n = dimensions.length; + this.dimensions = new long[n]; + this.position = new long[n]; + this.intPosition = new int[n]; + this.min = min; + steps = new long[n]; + + final int m = n - 1; + long k = steps[0] = 1; + for (int d = 0; d < m; ) { + final long dimd = dimensions[d]; + this.dimensions[d] = dimd; + k *= dimd; + steps[++d] = k; + } + final long dimm = dimensions[m]; + this.dimensions[m] = dimm; + lastIndex = (int)(k * dimm - 1); + } + + public GridIterator(final long[] dimensions) { + + this(dimensions, new long[dimensions.length]); + } + + public GridIterator(final int[] dimensions) { + + this(int2long(dimensions)); + } + + public void fwd() { + + ++index; + } + + public void reset() { + + index = -1; + } + + @Override + public boolean hasNext() { + + return index < lastIndex; + } + + @Override + public long[] next() { + + fwd(); + indexToPosition(index, dimensions, min, position); + return position; + } + + public int[] nextInt() { + + next(); + long2int(position, intPosition); + return intPosition; + } + + public int getIndex() { + + return index; + } + + public static void indexToPosition(long index, final long[] dimensions, final long[] offset, + final long[] position) { + + for (int dim = 0; dim < dimensions.length; dim++) { + position[dim] = (index % dimensions[dim]) + offset[dim]; + index /= dimensions[dim]; + } + } + + public static void indexToPosition(long index, final int[] dimensions, final long[] offset, + final long[] position) { + + for (int dim = 0; dim < dimensions.length; dim++) { + position[dim] = (index % dimensions[dim]) + offset[dim]; + index /= dimensions[dim]; + } + } + + final static public long positionToIndex(final long[] dimensions, final long[] position) { + long idx = 0; + int cumulativeSize = 1; + for (int i = 0; i < position.length; i++) { + idx += position[i] * cumulativeSize; + cumulativeSize *= dimensions[i]; + } + return idx; + } + + final static public long positionToIndex(final long[] dimensions, final int[] position) { + long idx = 0; + int cumulativeSize = 1; + for (int i = 0; i < position.length; i++) { + idx += position[i] * cumulativeSize; + cumulativeSize *= dimensions[i]; + } + return idx; + } + + final static public long positionToIndex(final int[] dimensions, final long[] position) { + long idx = 0; + int cumulativeSize = 1; + for (int i = 0; i < position.length; i++) { + idx += position[i] * cumulativeSize; + cumulativeSize *= dimensions[i]; + } + return idx; + } + + final static public long positionToIndex(final int[] dimensions, final int[] position) { + long idx = 0; + int cumulativeSize = 1; + for (int i = 0; i < position.length; i++) { + idx += position[i] * cumulativeSize; + cumulativeSize *= dimensions[i]; + } + return idx; + } + + public static int[] long2int(final long[] src, final int[] tgt) { + + for (int d = 0; d < tgt.length; ++d) + tgt[d] = (int)src[d]; + + return tgt; + } + + public static int[] long2int(final long[] a) { + + final int[] i = new int[a.length]; + + for (int d = 0; d < a.length; ++d) + i[d] = (int)a[d]; + + return i; + } + + public static long[] int2long(final int[] i) { + + final long[] l = new long[i.length]; + + for (int d = 0; d < l.length; ++d) + l[d] = i[d]; + + return l; + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java new file mode 100644 index 000000000..2403835b9 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java @@ -0,0 +1,66 @@ +package org.janelia.saalfeldlab.n5.util; + +import java.util.Arrays; + +/* + * A wrapper around a primitive long array that is lexicographically {@link Comparable} + * and for which we can test equality. + */ +public interface Position extends Comparable { + + long[] get(); + + long get(int i); + + default int numDimensions() { + return get().length; + } + + @Override + default int compareTo(Position other) { + + // use Arrays.compare when we update to Java 9+ + final int N = numDimensions() > other.numDimensions() ? numDimensions() : other.numDimensions(); + for (int i = 0; i < N; i++) { + final long diff = get(i) - other.get(i); + if (diff != 0) + return (int) diff; + } + return 0; + } + + static boolean equals(final Position a, final Object b) { + + if (a == null && b == null) + return true; + + if (b == null) + return false; + + if (!(b instanceof Position)) + return false; + + final Position other = (Position) b; + if (other.numDimensions() != a.numDimensions()) + return false; + + for (int i = 0; i < a.numDimensions(); i++) + if (other.get(i) != a.get(i)) + return false; + + return true; + } + + static String toString(Position p) { + return "Position: " + Arrays.toString(p.get()); + } + + static Position wrap(final long[] p) { + return new FinalPosition(p); + } + + static Position wrap(final int[] p) { + return new FinalPosition(GridIterator.int2long(p)); + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index f6b332654..60025e453 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

        - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

        - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

        - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import static org.junit.Assert.assertArrayEquals; @@ -60,6 +35,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.IOException; import java.net.URI; @@ -82,7 +58,9 @@ import org.janelia.saalfeldlab.n5.N5Reader.Version; import org.janelia.saalfeldlab.n5.url.UriAttributeTest; import org.junit.After; +import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import com.google.gson.GsonBuilder; @@ -107,8 +85,8 @@ public abstract class AbstractN5Test { static protected final String groupName = "/test/group"; static protected final String[] subGroupNames = new String[]{"a", "b", "c"}; static protected final String datasetName = "/test/group/dataset"; - static protected final long[] dimensions = new long[]{100, 200, 300}; - static protected final int[] blockSize = new int[]{44, 33, 22}; + static protected final long[] dimensions = new long[]{6, 15, 35}; + static protected final int[] blockSize = new int[]{3, 5, 7}; static protected final int blockNumElements = blockSize[0] * blockSize[1] * blockSize[2]; static protected byte[] byteBlock; @@ -132,7 +110,7 @@ public static URI createTempUri(String prefix, String suffix, URI base) { return N5URI.getAsUri(name); } - protected final N5Writer createTempN5Writer() { + public N5Writer createTempN5Writer() { try { return createTempN5Writer(tempN5Location()); @@ -141,7 +119,7 @@ protected final N5Writer createTempN5Writer() { } } - protected final N5Writer createTempN5Writer(String location) { + public final N5Writer createTempN5Writer(String location) { return createTempN5Writer(location, new GsonBuilder()); } @@ -160,6 +138,7 @@ protected final N5Writer createTempN5Writer(String location, GsonBuilder gson) { @After public void removeTempWriters() { + synchronized (tempWriters) { for (final N5Writer writer : tempWriters) { try { @@ -208,7 +187,7 @@ protected Compression[] getCompressions() { @Before public void setUpOnce() { - final Random rnd = new Random(); + final Random rnd = new Random(111); byteBlock = new byte[blockNumElements]; shortBlock = new short[blockNumElements]; intBlock = new int[blockNumElements]; @@ -252,22 +231,107 @@ public void testSetAttributeDoesntCreateGroup() { } @Test - public void testCreateDataset() { + public void testCreateDataset() { - final DatasetAttributes info; - try (N5Writer writer = createTempN5Writer()) { - writer.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression()); + final DatasetAttributes info; + try (N5Writer writer = createTempN5Writer()) { + writer.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression()); + + assertTrue("Dataset does not exist", writer.exists(datasetName)); + + info = writer.getDatasetAttributes(datasetName); + } + assertArrayEquals(dimensions, info.getDimensions()); + assertArrayEquals(blockSize, info.getBlockSize()); + assertEquals(DataType.UINT64, info.getDataType()); + } + + @Test + public void testBlocksLargerThanDimensions() { + + // Test case where block size is larger than dataset dimensions + final long[] smallDimensions = new long[]{2, 3, 4}; + final int[] largeBlockSize = new int[]{5, 7, 10}; + + try (final N5Writer n5 = createTempN5Writer()) { + n5.createDataset(datasetName, smallDimensions, largeBlockSize, DataType.UINT8, new RawCompression()); + final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + + // Create a block that is larger than the dataset dimensions + final int numElements = largeBlockSize[0] * largeBlockSize[1] * largeBlockSize[2]; + final byte[] data = new byte[numElements]; + for (int i = 0; i < numElements; i++) { + data[i] = (byte)(i % 256); + } + + final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(largeBlockSize, new long[]{0, 0, 0}, data); + n5.writeBlock(datasetName, attributes, dataBlock); + + // Read the block back + final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + assertNotNull("Block should be readable", loadedDataBlock); + assertArrayEquals("Block size should match", largeBlockSize, loadedDataBlock.getSize()); + assertArrayEquals("Block data should match", data, (byte[])loadedDataBlock.getData()); + } + } + + @Test + public void testUnalignedBlocksTruncatedAtEnd() { + + // Test case where dimensions don't evenly divide by block size + final long[] unalignedDimensions = new long[]{5, 14, 33}; + final int[] testBlockSize = new int[]{3, 5, 7}; + + try (final N5Writer n5 = createTempN5Writer()) { + n5.createDataset(datasetName, unalignedDimensions, testBlockSize, DataType.INT32, new RawCompression()); + final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); - assertTrue("Dataset does not exist", writer.exists(datasetName)); + // Test writing to the last block in dimension 0 (should be truncated to size 2 instead of 3) + final int[] truncatedBlockSize0 = new int[]{2, 5, 7}; // [3-4] in dim 0 + final int numElements0 = truncatedBlockSize0[0] * truncatedBlockSize0[1] * truncatedBlockSize0[2]; + final int[] data0 = new int[numElements0]; + for (int i = 0; i < numElements0; i++) { + data0[i] = i + 1000; + } + final IntArrayDataBlock dataBlock0 = new IntArrayDataBlock(truncatedBlockSize0, new long[]{1, 0, 0}, data0); + n5.writeBlock(datasetName, attributes, dataBlock0); + + final DataBlock loadedBlock0 = n5.readBlock(datasetName, attributes, 1, 0, 0); + assertNotNull("Truncated block should be readable", loadedBlock0); + assertArrayEquals("Truncated block data should match", data0, (int[])loadedBlock0.getData()); + + // Test writing to the last block in dimension 1 (should be truncated to size 4 instead of 5) + final int[] truncatedBlockSize1 = new int[]{3, 4, 7}; // [10-13] in dim 1 + final int numElements1 = truncatedBlockSize1[0] * truncatedBlockSize1[1] * truncatedBlockSize1[2]; + final int[] data1 = new int[numElements1]; + for (int i = 0; i < numElements1; i++) { + data1[i] = i + 2000; + } + final IntArrayDataBlock dataBlock1 = new IntArrayDataBlock(truncatedBlockSize1, new long[]{0, 2, 0}, data1); + n5.writeBlock(datasetName, attributes, dataBlock1); + + final DataBlock loadedBlock1 = n5.readBlock(datasetName, attributes, 0, 2, 0); + assertNotNull("Truncated block should be readable", loadedBlock1); + assertArrayEquals("Truncated block data should match", data1, (int[])loadedBlock1.getData()); + + // Test writing to the last block in dimension 2 (should be truncated to size 5 instead of 7) + final int[] truncatedBlockSize2 = new int[]{3, 5, 5}; // [28-32] in dim 2 + final int numElements2 = truncatedBlockSize2[0] * truncatedBlockSize2[1] * truncatedBlockSize2[2]; + final int[] data2 = new int[numElements2]; + for (int i = 0; i < numElements2; i++) { + data2[i] = i + 3000; + } + final IntArrayDataBlock dataBlock2 = new IntArrayDataBlock(truncatedBlockSize2, new long[]{0, 0, 4}, data2); + n5.writeBlock(datasetName, attributes, dataBlock2); - info = writer.getDatasetAttributes(datasetName); + final DataBlock loadedBlock2 = n5.readBlock(datasetName, attributes, 0, 0, 4); + assertNotNull("Truncated block should be readable", loadedBlock2); + assertArrayEquals("Truncated block data should match", data2, (int[])loadedBlock2.getData()); } - assertArrayEquals(dimensions, info.getDimensions()); - assertArrayEquals(blockSize, info.getBlockSize()); - assertEquals(DataType.UINT64, info.getDataType()); - assertTrue(info.getCompression() instanceof RawCompression); } + + @Test public void testWriteReadByteBlock() { @@ -283,7 +347,6 @@ public void testWriteReadByteBlock() { final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); - } } } @@ -460,11 +523,63 @@ public void testWriteReadSerializableBlock() throws ClassNotFoundException { } } + @Test + @Ignore // TODO + public void testWriteInvalidBlock() { + + final Compression compression = getCompressions()[0]; + final DataType dataType = DataType.UINT8; + + final int[] biggerBlockSize = Arrays.stream(blockSize).map(x -> x + 2).toArray(); + int nBigger = Arrays.stream(biggerBlockSize).reduce(1, (x, y) -> x * y); + + final int[] smallerBlockSize = Arrays.stream(blockSize).map(x -> x - 2).toArray(); + int nSmaller = Arrays.stream(smallerBlockSize).reduce(1, (x, y) -> x * y); + + int N = Arrays.stream(blockSize).reduce(1, (x, y) -> x * y); + + final Random rnd = new Random(7560); + final byte[] biggerData = new byte[nBigger]; + rnd.nextBytes(biggerData); + + final byte[] smallerData = new byte[nSmaller]; + rnd.nextBytes(smallerData); + + final float[] floatData = new float[N]; + + try (final N5Writer n5 = createTempN5Writer()) { + + n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); + final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + + // write a block that is too large + final ByteArrayDataBlock bigDataBlock = new ByteArrayDataBlock(biggerBlockSize, new long[]{0, 0, 0}, biggerData); + n5.writeBlock(datasetName, attributes, bigDataBlock); + + final DataBlock loadedBigDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + assertArrayEquals(biggerData, (byte[])loadedBigDataBlock.getData()); + + // write a block that is too small + final ByteArrayDataBlock smallDataBlock = new ByteArrayDataBlock(smallerBlockSize, new long[]{0, 0, 0}, smallerData); + n5.writeBlock(datasetName, attributes, smallDataBlock); + + final DataBlock loadedSmallDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); + assertArrayEquals(smallerData, (byte[])loadedSmallDataBlock.getData()); + + // write a block of the wrong type + final FloatArrayDataBlock floatDataBlock = new FloatArrayDataBlock(blockSize, new long[]{0, 0, 0}, floatData); + assertThrows(ClassCastException.class, () -> { + n5.writeBlock(datasetName, attributes, floatDataBlock); + }); + } + } + @Test public void testOverwriteBlock() { + final Compression compression = getCompressions()[0]; try (final N5Writer n5 = createTempN5Writer()) { - n5.createDataset(datasetName, dimensions, blockSize, DataType.INT32, new GzipCompression()); + n5.createDataset(datasetName, dimensions, blockSize, DataType.INT32, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final IntArrayDataBlock randomDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); @@ -472,16 +587,17 @@ public void testOverwriteBlock() { final DataBlock loadedRandomDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); assertArrayEquals(intBlock, (int[])loadedRandomDataBlock.getData()); - // test the case where the resulting file becomes shorter - final IntArrayDataBlock emptyDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, new int[DataBlock.getNumElements(blockSize)]); + // test the case where the resulting file becomes shorter (because the data compresses better) + final int[] emptyBlock = new int[DataBlock.getNumElements(blockSize)]; + final IntArrayDataBlock emptyDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, emptyBlock); n5.writeBlock(datasetName, attributes, emptyDataBlock); final DataBlock loadedEmptyDataBlock = n5.readBlock(datasetName, attributes, 0, 0, 0); - assertArrayEquals(new int[DataBlock.getNumElements(blockSize)], (int[])loadedEmptyDataBlock.getData()); + assertArrayEquals(emptyBlock, (int[])loadedEmptyDataBlock.getData()); } } @Test - public void testAttributeParsingPrimitive() { + public void testAttributeParsingPrimitive() { try (final N5Writer n5 = createTempN5Writer()) { @@ -557,7 +673,7 @@ public void testAttributeParsingPrimitive() { } @Test - public void testAttributes() { + public void testAttributes() { try (final N5Writer n5 = createTempN5Writer()) { assertNull(n5.getAttribute(groupName, "test", String.class)); @@ -623,7 +739,6 @@ public void testAttributes() { } } - @Test public void testNullAttributes() throws URISyntaxException, IOException { @@ -847,7 +962,7 @@ public void testUri() throws IOException, URISyntaxException { } @Test - public void testRemoveGroup() { + public void testRemoveGroup() { try (final N5Writer n5 = createTempN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression()); @@ -907,7 +1022,7 @@ public void testDeepList() throws ExecutionException, InterruptedException { for (final String subGroup : subGroupNames) assertTrue("deepList contents", Arrays.asList(n5.deepList("")).contains(groupName.replaceFirst("/", "") + "/" + subGroup)); - final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64, new RawCompression()); + final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64); final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize, new long[]{0, 0, 0}, new long[blockNumElements]); n5.createDataset(datasetName, datasetAttributes); n5.writeBlock(datasetName, datasetAttributes, dataBlock); @@ -1009,7 +1124,7 @@ public void testDeepList() throws ExecutionException, InterruptedException { } @Test - public void testExists() { + public void testExists() { final String groupName2 = groupName + "-2"; final String datasetName2 = datasetName + "-2"; @@ -1030,7 +1145,7 @@ public void testExists() { } @Test - public void testListAttributes() { + public void testListAttributes() { try (N5Writer n5 = createTempN5Writer()) { final String groupName2 = groupName + "-2"; @@ -1137,14 +1252,14 @@ public void testReaderCreation() throws IOException, URISyntaxException { writer.setAttribute("/", N5Reader.VERSION_KEY, invalidVersion); assertThrows("Incompatible version throws error", N5Exception.class, () -> { try (final N5Reader ignored = createN5Reader(location)) { - /*Only try with resource to ensure `close()` is called.*/ + /*Only try with resource to ensure `close()` is called.*/ } }); } } @Test - public void testDelete() { + public void testDelete() { try (N5Writer n5 = createTempN5Writer()) { final String datasetName = AbstractN5Test.datasetName + "-test-delete"; @@ -1154,8 +1269,7 @@ public void testDelete() { final long[] position2 = {0, 1, 2}; // no blocks should exist to begin with - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1))); - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); + assertNull(n5.readBlock(datasetName, attributes, position1)); final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, position1, byteBlock); n5.writeBlock(datasetName, attributes, dataBlock); @@ -1165,24 +1279,17 @@ public void testDelete() { assertNotNull(readBlock); assertTrue(readBlock instanceof ByteArrayDataBlock); assertArrayEquals(byteBlock, ((ByteArrayDataBlock)readBlock).getData()); - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); - // deletion should report true in all cases - assertTrue(n5.deleteBlock(datasetName, position1)); - assertTrue(n5.deleteBlock(datasetName, position1)); - assertTrue(n5.deleteBlock(datasetName, position2)); + assertTrue("deleting existing block should return true", n5.deleteBlock(datasetName, position1)); + assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position1)); + assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position2)); // no block should exist anymore - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1))); - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); + assertNull(n5.readBlock(datasetName, attributes, position1)); + assertNull(n5.readBlock(datasetName, attributes, position2)); } } - protected boolean testDeleteIsBlockDeleted(final DataBlock dataBlock) { - - return dataBlock == null; - } - public static class TestData { public String groupPath; @@ -1287,7 +1394,7 @@ public void customObjectTest() { } @Test - public void testAttributePaths() { + public void testAttributePaths() { try (final N5Writer writer = createTempN5Writer()) { @@ -1393,13 +1500,13 @@ public void testAttributePaths() { * to try and grab the value as a json structure. I should grab the root, and match the empty string case */ assertEquals(writer.getAttribute(testGroup, "", JsonObject.class), writer.getAttribute(testGroup, "/", JsonObject.class)); - /* Lastly, ensure grabing nonsense returns null */ + /* Lastly, ensure grabbing nonsense returns null */ assertNull(writer.getAttribute(testGroup, "/this/key/does/not/exist", Object.class)); } } @Test - public void testAttributePathEscaping() { + public void testAttributePathEscaping() { final JsonObject emptyObj = new JsonObject(); @@ -1485,8 +1592,7 @@ private String jsonKeyVal(final String key, final String val) { } @Test - public void - testRootLeaves() { + public void testRootLeaves() { /* Test retrieving non-JsonObject root leaves */ try (final N5Writer n5 = createTempN5Writer()) { @@ -1593,13 +1699,17 @@ public void testWriterSeparation() { } } + protected String[] illegalChars() { + return new String[]{" ", "#", "%"}; + } + @Test public void testPathsWithIllegalUriCharacters() throws IOException, URISyntaxException { try (N5Writer writer = createTempN5Writer()) { try (N5Reader reader = createN5Reader(writer.getURI().toString())) { - final String[] illegalChars = {" ", "#", "%"}; + final String[] illegalChars = illegalChars(); for (final String illegalChar : illegalChars) { final String groupWithIllegalChar = "test" + illegalChar + "group"; assertThrows("list over group should throw prior to create", N5Exception.N5IOException.class, () -> writer.list(groupWithIllegalChar)); @@ -1625,10 +1735,40 @@ public void testPathsWithIllegalUriCharacters() throws IOException, URISyntaxExc } } + public static void assertBlockEquals(final DataBlock expected, final DataBlock actual) { + assertEquals("Datablocks are different type", expected.getClass(), actual.getClass()); + + Assert.assertArrayEquals("read block position should be same as block position when unsharded", expected.getGridPosition(), actual.getGridPosition()); + Assert.assertArrayEquals("read block size should equal block size when unsharded", expected.getSize(), actual.getSize()); + + final Object expectedData = expected.getData(); + final Object actualData = actual.getData(); + + final String dataEqualsMsg = "block written through shard should be identical"; + if (expectedData instanceof byte[]) + assertArrayEquals(dataEqualsMsg, (byte[])expectedData, (byte[])expectedData); + else if (expectedData instanceof short[]) + assertArrayEquals(dataEqualsMsg, (short[])expectedData, (short[])actualData); + else if (expectedData instanceof int[]) + assertArrayEquals(dataEqualsMsg, (int[])expectedData, (int[])actualData); + else if (expectedData instanceof long[]) + assertArrayEquals(dataEqualsMsg, (long[])expectedData, (long[])actualData); + else if (expectedData instanceof float[]) + assertArrayEquals(dataEqualsMsg, (float[])expectedData, (float[])actualData, 0f); + else if (expectedData instanceof double[]) + assertArrayEquals(dataEqualsMsg, (double[])expectedData, (double[])actualData, 0d); + else if (expectedData instanceof String[]) + assertArrayEquals(dataEqualsMsg, (String[])expectedData, (String[])actualData); + else + fail("Unsupported data type for block data: " + expectedData.getClass()); + } + protected void assertDatasetAttributesEquals(final DatasetAttributes expected, final DatasetAttributes actual) { assertArrayEquals(expected.getDimensions(), actual.getDimensions()); assertArrayEquals(expected.getBlockSize(), actual.getBlockSize()); assertEquals(expected.getDataType(), actual.getDataType()); - assertEquals(expected.getCompression(), actual.getCompression()); + + // TODO would be nice to check this somehow maybe make a DatasetAttributes.equals method? +// assertArrayEquals(expected.getDataCodecInfos(), actual.getDataCodecInfos()); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java new file mode 100644 index 000000000..9f2d52810 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java @@ -0,0 +1,247 @@ +/*- + * #%L + * Not HDF5 + * %% + * Copyright (C) 2017 - 2025 Stephan Saalfeld + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.janelia.saalfeldlab.n5; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.shard.DefaultShardCodecInfo; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; +import org.janelia.saalfeldlab.n5.util.GridIterator; +import org.janelia.saalfeldlab.n5.util.Position; +import org.junit.Test; + +/** + * Unit tests for DatasetAttributes class. + */ +public class DatasetAttributesTest { + + /** + * Test that validateBlockShardSizes method accepts valid shard and block size combinations. + */ + @Test + public void testValidateBlockShardSizesValid() { + + // Test case 1: shard size equals block size + long[] dimensions = new long[]{100, 200, 300}; + int[] shardSize = new int[]{64, 64, 64}; + int[] blockSize = new int[]{64, 64, 64}; + DataType dataType = DataType.UINT8; + + // This should not throw any exception + DatasetAttributes attrs = shardDatasetAttributes(dimensions, shardSize, blockSize, dataType); +// assertEquals(shardSize, attrs.getShardSize()); + assertEquals(blockSize, attrs.getBlockSize()); +// assertArrayEquals(new int[]{1, 1, 1}, attrs.getBlocksPerShard()); + + // Test case 2: shard size is a multiple of block size + shardSize = new int[]{128}; + blockSize = new int[]{64}; + attrs = shardDatasetAttributes(new long[]{128}, shardSize, blockSize, dataType); +// assertEquals(shardSize, attrs.getShardSize()); + assertEquals(blockSize, attrs.getBlockSize()); +// assertArrayEquals(new int[]{2}, attrs.getBlocksPerShard()); + + // Test case 3: different multiples per dimension + shardSize = new int[]{128, 256, 32, 2}; + blockSize = new int[]{32, 64, 32, 1}; + attrs = shardDatasetAttributes(new long[]{128, 128, 128, 128}, shardSize, blockSize, dataType ); +// assertEquals(shardSize, attrs.getShardSize()); + assertEquals(blockSize, attrs.getBlockSize()); +// assertArrayEquals(new int[]{4, 4, 1, 2}, attrs.getBlocksPerShard()); + + // Test case 4: large multiples + shardSize = new int[]{1024, 2048, 512}; + blockSize = new int[]{32, 64, 16}; + attrs = shardDatasetAttributes(dimensions, shardSize, blockSize, dataType); +// assertEquals(shardSize, attrs.getShardSize()); + assertEquals(blockSize, attrs.getBlockSize()); +// assertArrayEquals(new int[]{32, 32, 32}, attrs.getBlocksPerShard()); + } + + private static DatasetAttributes shardDatasetAttributes( + long[] dimensions, int[] shardSize, int[] blockSize, DataType dataType) { + + DefaultShardCodecInfo blockCodecInfo = new DefaultShardCodecInfo( + blockSize, + new N5BlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + IndexLocation.END); + + return new DatasetAttributes(dimensions, shardSize, dataType, blockCodecInfo); + } + + /** + * Test that validateBlockShardSizes method rejects invalid shard and block size combinations. + */ + @Test + public void testValidateBlockShardSizesInvalid() { + + final long[] dimensions = new long[]{100, 200, 300}; + final DataType dataType = DataType.UINT8; + + // Block size too small + IllegalArgumentException ex0 = assertThrows( + IllegalArgumentException.class, + () -> shardDatasetAttributes(dimensions, new int[]{1, 1, 1}, new int[]{1, 0, -1}, dataType)); + assertTrue(ex0.getMessage().contains("negative")); + + // Different number of dimensions + IllegalArgumentException ex1 = assertThrows( + IllegalArgumentException.class, + () -> shardDatasetAttributes(dimensions, new int[]{64, 64}, new int[]{32, 32, 32}, dataType)); + assertTrue(ex1.getMessage().contains("different length")); + + // Shard size smaller than block size + IllegalArgumentException ex2 = assertThrows( + IllegalArgumentException.class, + () -> shardDatasetAttributes(dimensions, new int[]{32, 64, 64}, new int[]{64, 64, 64}, dataType)); + assertTrue(ex2.getMessage().contains("is smaller than previous")); + + // Shard size not a multiple of block size + IllegalArgumentException ex3 = assertThrows( + IllegalArgumentException.class, + () -> shardDatasetAttributes(dimensions, new int[]{100, 100, 100}, new int[]{64, 64, 64}, dataType)); + assertTrue(ex3.getMessage().contains("not a multiple of previous level")); + + // Multiple violations - shard smaller than block in one dimension + IllegalArgumentException ex4 = assertThrows( + IllegalArgumentException.class, + () -> shardDatasetAttributes(dimensions, new int[]{128, 32, 128}, new int[]{64, 64, 64}, dataType)); + assertTrue(ex4.getMessage().contains("is smaller than previous")); + + // Edge case - shard size of 0 + assertThrows( + IllegalArgumentException.class, + () -> shardDatasetAttributes(dimensions, new int[]{0, 64, 64}, new int[]{64, 64, 64}, dataType)); + } + +// @Test +// public void testShardProperties() { +// +// final long[] arraySize = new long[]{16, 16}; +// final int[] shardSize = new int[]{16, 16}; +// final long[] shardPosition = new long[]{1, 1}; +// final int[] blkSize = new int[]{4, 4}; +// +// final DatasetAttributes dsetAttrs = new DatasetAttributes( +// arraySize, +// shardSize, +// blkSize, +// DataType.UINT8, +// new ShardingCodec( +// blkSize, +// new CodecInfo[]{new N5BlockCodecInfo()}, +// new DeterministicSizeCodecInfo[]{new RawBlockCodecInfo()}, +// IndexLocation.END +// ) +// ); +// +// final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); +// +// assertArrayEquals(new int[]{4, 4}, shard.getBlockGridSize()); +// +// assertArrayEquals(new long[]{0, 0}, dsetAttrs.getShardPositionForBlock(0, 0)); +// assertArrayEquals(new long[]{1, 1}, dsetAttrs.getShardPositionForBlock(5, 5)); +// assertArrayEquals(new long[]{1, 0}, dsetAttrs.getShardPositionForBlock(5, 0)); +// assertArrayEquals(new long[]{0, 1}, dsetAttrs.getShardPositionForBlock(0, 5)); +// +// assertArrayEquals(new int[]{0, 0}, shard.getRelativeBlockPosition(4, 4)); +// assertArrayEquals(new int[]{1, 1}, shard.getRelativeBlockPosition(5, 5)); +// assertArrayEquals(new int[]{2, 2}, shard.getRelativeBlockPosition(6, 6)); +// assertArrayEquals(new int[]{3, 3}, shard.getRelativeBlockPosition(7, 7)); +// } + +// @Test +// public void testShardGrouping() { +// +// final long[] arraySize = new long[]{8, 12}; +// final int[] shardSize = new int[]{4, 6}; +// final int[] blkSize = new int[]{2, 3}; +// +// final DatasetAttributes attrs = new DatasetAttributes( +// arraySize, +// shardSize, +// blkSize, +// DataType.UINT8, +// new ShardingCodec( +// blkSize, +// new CodecInfo[]{ new N5BlockCodecInfo() }, +// new DeterministicSizeCodecInfo[]{new RawBlockCodecInfo()}, +// IndexLocation.END +// ) +// ); +// +// List blockPositions = blockPositions(attrs).collect(Collectors.toList()); +// final Map> result = attrs.groupBlockPositions(blockPositions); +// +// // there are four shards in this image +// assertEquals(4, result.size()); +// +// // there are four blocks per shard in this image +// result.values().forEach(x -> assertEquals(4, x.size())); +// } + +// private static Stream blockPositions( final DatasetAttributes attrs ) { +// +// final int nd = attrs.getNumDimensions(); +// final int[] blocksPerShard = attrs.getBlocksPerShard(); +// return toStream( new GridIterator(attrs.shardsPerDataset())) +// .flatMap( shardPosition -> { +// final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]); +// return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); +// }); +// } + + private static Stream toStream( final Iterator it ) { + + return StreamSupport.stream( + Spliterators.spliteratorUnknownSize(it, Spliterator.ORDERED), + false); + } + +} \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java index 5f069f7b2..546a4072f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import static org.junit.Assert.fail; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 262d2ee18..070516be5 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -136,8 +136,8 @@ public void cacheGroupDatasetTest() throws IOException, URISyntaxException { final String groupName = "gg"; final String tmpLocation = tempN5Location(); - try (N5KeyValueWriter w1 = (N5KeyValueWriter) createN5Writer(tmpLocation); - N5KeyValueWriter w2 = (N5KeyValueWriter) createN5Writer(tmpLocation);) { + try (GsonKeyValueN5Writer w1 = (GsonKeyValueN5Writer) createN5Writer(tmpLocation); + GsonKeyValueN5Writer w2 = (GsonKeyValueN5Writer) createN5Writer(tmpLocation);) { // create a group, both writers know it exists w1.createGroup(groupName); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java new file mode 100644 index 000000000..4cdc1a272 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java @@ -0,0 +1,141 @@ +package org.janelia.saalfeldlab.n5.backward; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Files; +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.N5FSReader; +import org.janelia.saalfeldlab.n5.N5FSWriter; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.junit.Test; + +import com.google.gson.JsonElement; + +public class CompatibilityTest { + + String[][] readVersionsDataset = { + {"data-1.5.0.n5", "raw"}, + {"data-2.5.1.n5", "raw"}, + {"data-3.1.3.n5", "raw"} }; + + String writeVersion = "data-3.1.3.n5"; + String writeDataset = "raw"; + String[] writePathsToTest = {"0/0", "0/1", "1/0", "1/1"}; + + @Test + public void testBackwardReads() throws NumberFormatException, IOException { + + for (String[] versionDset : readVersionsDataset) + backwardReadHelper(versionDset[0], versionDset[1]); + } + + public void backwardReadHelper(final String base, final String dsetPath) throws NumberFormatException, IOException { + + final N5FSReader n5 = new N5FSReader("src/test/resources/backward/" + base); + assertTrue(n5.datasetExists(dsetPath)); + final DatasetAttributes attrs = n5.getDatasetAttributes(dsetPath); + + // equivalent to the assertTrue above, but be extra sure + assertNotNull(attrs); + + byte value = 0; + long[] p = new long[2]; + + DataBlock b00 = n5.readBlock(dsetPath, attrs, p); + assertNotNull(b00); + assertArrayEquals(new int[]{5,4}, b00.getSize()); + assertArrayEquals(expectedData(20, value), b00.getData()); + + p[0] = 1; + p[1] = 0; + value++; + DataBlock b10 = n5.readBlock(dsetPath, attrs, p); + assertNotNull(b10); + assertArrayEquals(new int[]{2,4}, b10.getSize()); + assertArrayEquals(expectedData(8, value), b10.getData()); + + p[0] = 0; + p[1] = 1; + value++; + DataBlock b01 = n5.readBlock(dsetPath, attrs, p); + assertNotNull(b01); + assertArrayEquals(new int[]{5,1}, b01.getSize()); + assertArrayEquals(expectedData(5, value), b01.getData()); + + p[0] = 1; + p[1] = 1; + value++; + DataBlock b11 = n5.readBlock(dsetPath, attrs, p); + assertNotNull(b11); + assertArrayEquals(new int[]{2,1}, b11.getSize()); + assertArrayEquals(expectedData(2, value), b11.getData()); + + n5.close(); + } + + @Test + public void testBlockData() throws IOException { + + final N5FSReader n5Legacy = new N5FSReader("src/test/resources/backward/" + writeVersion); + final URI uriLegacy = n5Legacy.getURI(); + + final File basePath = Files.createTempDirectory("n5-blockDataTest-").toFile(); + basePath.delete(); + basePath.mkdir(); + basePath.deleteOnExit(); + + N5FSWriter n5My = CreateSampleData.createSampleData( + basePath.getCanonicalPath(), writeDataset, new RawCompression()); + URI uriMy = n5My.getURI(); + + // check attributes + final JsonElement attrsLegacy = ((GsonKeyValueN5Reader)n5Legacy).getAttributes(writeDataset); + final JsonElement attrsMy = ((GsonKeyValueN5Reader)n5My).getAttributes(writeDataset); + assertEquals(attrsLegacy, attrsMy); + + final KeyValueAccess kva = n5My.getKeyValueAccess(); + for (final String path : writePathsToTest) { + final byte[] dataMy = read(kva, kva.compose(uriMy, writeDataset, path)); + final byte[] dataLegacy = read(kva, kva.compose(uriLegacy, writeDataset, path)); + assertArrayEquals(dataLegacy, dataMy); + } + + n5My.remove(); + n5My.close(); + n5Legacy.close(); + } + + private byte[] read(KeyValueAccess kva, String path) { + + int N = (int)kva.size(path); + byte[] data = new byte[N]; + try (LockedChannel ch = kva.lockForReading(path); + InputStream is = ch.newInputStream();) { + + is.read(data); + } catch (IOException e) { + return null; + } + return data; + } + + private static byte[] expectedData(int size, byte value ) { + byte[] data = new byte[size]; + Arrays.fill(data, value); + return data; + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java new file mode 100644 index 000000000..73334cdcd --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java @@ -0,0 +1,69 @@ +package org.janelia.saalfeldlab.n5.backward; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.Compression; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.N5FSWriter; +import org.janelia.saalfeldlab.n5.RawCompression; + +public class CreateSampleData { + + public static void main(String[] args) throws IOException { + + File f = new File("src/test/resources/data-4.0.0-alpha-X.n5"); + System.out.println(f.getCanonicalPath()); + createSampleData(f.getCanonicalPath(), "raw", new RawCompression()); + } + + public static N5FSWriter createSampleData(String baseDir, String dataset, Compression compression) throws IOException { + + N5FSWriter n5 = new N5FSWriter(baseDir); + final String dsetPath = compression.getType(); + + long[] dimensions = new long[]{7, 5}; + int[] blkSizeDset = new int[]{5, 4}; + int[] blkSize = new int[]{5, 4}; + + final DatasetAttributes attrs = new DatasetAttributes(dimensions, blkSizeDset, DataType.UINT8, compression); + n5.createDataset(dsetPath, attrs); + + byte val = 0; + long[] pos = new long[]{0, 0}; + n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + + pos[0] = 1; + pos[1] = 0; + blkSize[0] = 2; + blkSize[1] = 4; + val++; + n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + + pos[0] = 0; + pos[1] = 1; + blkSize[0] = 5; + blkSize[1] = 1; + val++; + n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + + pos[0] = 1; + pos[1] = 1; + blkSize[0] = 2; + blkSize[1] = 1; + val++; + n5.writeBlock(dsetPath, attrs, createDataBlock( blkSize, pos, val )); + + return n5; + } + + public static ByteArrayDataBlock createDataBlock(int[] size, long[] gridPosition, byte value) throws IOException { + int N = Arrays.stream(size).reduce(1, (x,y) -> x*y); + final byte[] data = new byte[N]; + Arrays.fill(data, value); + return new ByteArrayDataBlock(size, gridPosition, data); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java new file mode 100644 index 000000000..199563b98 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java @@ -0,0 +1,136 @@ +package org.janelia.saalfeldlab.n5.benchmarks; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@State(Scope.Benchmark) +@Warmup(iterations = 10, time = 100, timeUnit = TimeUnit.MICROSECONDS) +@Measurement(iterations = 100, time = 100, timeUnit = TimeUnit.MICROSECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(1) +public class ReadDataBenchmarks { + + @Param(value = { "10000000" }) + protected int objectSizeBytes; + + protected Path basePath; + protected ArrayList tmpPaths; + protected KeyValueAccess kva; + protected Random random; + + public ReadDataBenchmarks() {} + + public static void main(String... args) throws RunnerException { + + final Options options = new OptionsBuilder().include(ReadDataBenchmarks.class.getSimpleName() + "\\.") + .build(); + + new Runner(options).run(); + } + + @Benchmark + public void run(Blackhole hole) throws IOException { + + hole.consume(read().materialize()); + } + + public ReadData read() throws IOException { + + return kva.createReadData(getPath().toString()); + } + + protected Path getPath() { + + return basePath.resolve("tmp-" + objectSizeBytes); + } + + @Setup(Level.Trial) + public void setup() throws IOException { + + random = new Random(); + kva = new FileSystemKeyValueAccess(FileSystems.getDefault()); + + basePath = Files.createTempDirectory("ReadDataBenchmark-"); + tmpPaths = new ArrayList<>(); + for (final int sz : sizes()) { + Path p = basePath.resolve("tmp-"+sz); + write(p, sz); + tmpPaths.add(p); + } + } + + protected void write(Path path, int numBytes) { + + final byte[] data = new byte[numBytes]; + random.nextBytes(data); + + System.out.println(path.toAbsolutePath().toString()); + System.out.println(numBytes); + try (final LockedChannel ch = kva.lockForWriting(path.toAbsolutePath().toString())) { + final OutputStream os = ch.newOutputStream(); + os.write(data); + os.flush(); + os.close(); + } catch (final IOException e) { + e.printStackTrace(); + } + } + + @TearDown(Level.Trial) + public void teardown() { + + for ( Path p : tmpPaths ) { + p.toFile().delete(); + } + basePath.toFile().delete(); + } + + public int[] sizes() { + + try { + final Param ann = ReadDataBenchmarks.class.getDeclaredField("objectSizeBytes").getAnnotation(Param.class); + System.out.println(Arrays.toString(ann.value())); + return Arrays.stream(ann.value()).mapToInt(Integer::parseInt).toArray(); + + } catch (final NoSuchFieldException e) { + e.printStackTrace(); + } catch (final SecurityException e) { + e.printStackTrace(); + } + + return null; + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java new file mode 100644 index 000000000..da0bf9958 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarksKvaReadData.java @@ -0,0 +1,43 @@ +package org.janelia.saalfeldlab.n5.benchmarks; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@State(Scope.Benchmark) +@Warmup(iterations = 10, time = 100, timeUnit = TimeUnit.MICROSECONDS) +@Measurement(iterations = 100, time = 100, timeUnit = TimeUnit.MICROSECONDS) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(1) +public class ReadDataBenchmarksKvaReadData extends ReadDataBenchmarks { + + public static void main(String... args) throws RunnerException { + + final Options options = new OptionsBuilder().include(ReadDataBenchmarksKvaReadData.class.getSimpleName() + "\\.") + .build(); + + new Runner(options).run(); + } + + public ReadData read() throws IOException { + + return ((FileSystemKeyValueAccess)kva).createReadData(getPath().toString()); + } + +} \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java index ed6259f2d..d3f996ede 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java @@ -209,7 +209,7 @@ public void testChildManagement() { // Test addChildIfPresent on cached parent without children list cache.exists("parent2", null); - children = cache.list("parent2"); // initialize children array + children = cache.list("parent2"); // create children array cache.addChildIfPresent("parent2", "child"); children = cache.list("parent2"); assertTrue(Arrays.asList(children).contains("child")); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java new file mode 100644 index 000000000..3ff34059f --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java @@ -0,0 +1,291 @@ +package org.janelia.saalfeldlab.n5.codec; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.Random; + +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock; +import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; +import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.saalfeldlab.n5.IntArrayDataBlock; +import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; +import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; +import org.janelia.saalfeldlab.n5.shard.DatasetAccess; +import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; +import org.janelia.saalfeldlab.n5.shard.RawShardTest; +import org.janelia.saalfeldlab.n5.shard.TestPositionValueAccess; +import org.junit.Test; + +public class BlockCodecTests { + + static Random random = new Random(12345); + + final int[] blockSize = {11, 7, 5}; + private final BitShiftBytesCodec shiftCodec = new BitShiftBytesCodec(3); + private final GzipCompression compressor = new GzipCompression(); + private final DataCodecInfo[][] dataCodecInfos = new DataCodecInfo[][]{ + {}, // empty: "raw" compression + {compressor}, + {shiftCodec}, + {shiftCodec, compressor} + }; + + private final DataType[] dataTypes = { + DataType.INT8, DataType.UINT8, + DataType.INT16, DataType.UINT16, + DataType.INT32, DataType.UINT32, + DataType.INT64, DataType.UINT64, + DataType.FLOAT32, DataType.FLOAT64 + }; + + @Test + public void testN5BlockCodec() throws Exception { + for (DataType dataType : dataTypes) { + for (DataCodecInfo[] dataCodecInfo : dataCodecInfos) { + + final DatasetAttributes attributes = new DatasetAttributes( + new long[]{32, 32, 32}, + blockSize, + dataType, + new N5BlockCodecInfo(), + dataCodecInfo); + + testBlockCodecHelper(attributes); + } + } + } + + @Test + public void testRawBytesBlockCodec() throws Exception { + // Test RawBlockCodecInfo codec with different byte orders and DataTypes + final ByteOrder[] byteOrders = {ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}; + for (DataType dataType : dataTypes) { + for (ByteOrder byteOrder : byteOrders) { + for (DataCodecInfo[] codecs : dataCodecInfos) { + + final RawBlockCodecInfo codec = new RawBlockCodecInfo(byteOrder); + final DatasetAttributes attributes = new DatasetAttributes( + new long[]{32, 32, 32}, + blockSize, + dataType, + codec, + codecs); + + testBlockCodecHelper(attributes); + } + } + } + } + + private void testBlockCodecHelper(DatasetAttributes attributes) throws Exception { + + // TODO +// final int[] blockSize = attributes.getBlockSize(); +// final DataType dataType = attributes.getDataType(); +// final long[] gridPosition = {3, 2, 1}; +// +// // Create appropriate data block based on type +// DataBlock originalBlock = ((DataBlock)createRandomDataBlock(dataType, blockSize, gridPosition)); +// final BlockCodec codec = attributes.getBlockCodec(); +// +// // Test encode/decode roundtrip +// final ReadData encoded = codec.encode(originalBlock); +// assertNotNull(encoded); +// +// final DataBlock decoded = codec.decode(encoded, gridPosition); +// assertNotNull(decoded); +// +// assertArrayEquals("Block size should match", blockSize, decoded.getSize()); +// assertArrayEquals("Grid position should match", gridPosition, decoded.getGridPosition()); +// assertDataEquals(originalBlock, decoded); +// verifyCompatibleDataType(dataType, decoded); + } + + @SuppressWarnings("unchecked") + @Test + public void testEmptyBlock() throws Exception { + // Test handling of empty blocks + final int[] blockSize = {0, 0}; + final long[] gridPosition = {0, 0}; + final N5BlockCodecInfo blockCodecInfo = new N5BlockCodecInfo(); + final RawShardTest.TestDatasetAttributes attributes = new RawShardTest.TestDatasetAttributes( + new long[]{64, 64}, + new int[]{8, 8}, + DataType.UINT8, + blockCodecInfo, + new RawCompression()); + + final PositionValueAccess store = new TestPositionValueAccess(); + DatasetAccess access = attributes.datasetAccess(); + + // Test encode/decode + final ByteArrayDataBlock emptyBlock = new ByteArrayDataBlock(blockSize, gridPosition, new byte[0]); + + access.writeBlock(store, emptyBlock); + final DataBlock decoded = access.readBlock(store, gridPosition); + + assertEquals("Empty block should have 0 elements", 0, decoded.getNumElements()); + } + + @Test + public void testEncodedSizeCalculation() throws Exception { + + // TODO + + // Test that encoded size calculations are correct +// final int[] blockSize = {64, 64}; +// final DatasetAttributes n5ArrayAttrs = new DatasetAttributes( +// new long[]{512, 512}, +// blockSize, +// blockSize, +// DataType.INT16, +// new N5BlockCodecInfo()); +// +// +// final DatasetAttributes rawArrayAttrs = new DatasetAttributes( +// new long[]{512, 512}, +// blockSize, +// blockSize, +// DataType.INT16, +// new RawBlockCodecInfo()); +// +// // Calculate expected sizes +// final long rawDataSize = blockSize[0] * blockSize[1] * 2; // INT16 has 2 bytes per element +// +// // N5BlockCodecInfo adds a header +// // the estimate of the encoded size +// final long n5EncodedSize = n5ArrayAttrs.getBlockCodecInfo().encodedSize(rawDataSize); +// assertTrue("N5 encoded size should be larger than raw size", n5EncodedSize > rawDataSize); +// +// DataBlock dataBlock = ((DataBlock)createRandomDataBlock(n5ArrayAttrs.getDataType(), blockSize, new long[]{0, 0})); +// ReadData n5EncodedDataBlock = n5ArrayAttrs.getBlockCodec().encode(dataBlock); +// assertEquals("N5 actual encoded size should equal estimated size", n5EncodedSize, n5EncodedDataBlock.length()); +// +// // RawBlockCodecInfo should not change size +// final long rawEncodedSize = rawArrayAttrs.getBlockCodecInfo().encodedSize(rawDataSize); +// assertEquals("Raw encoded size should equal input size", rawDataSize, rawEncodedSize); +// +// ReadData rawEncodedDataBlock = rawArrayAttrs.getBlockCodec().encode(dataBlock); +// assertEquals("Raw actual encoded size should equal estimated size", rawEncodedSize, rawEncodedDataBlock.length()); + } + + private static DataBlock createRandomDataBlock(DataType dataType, int[] blockSize, long[] gridPosition) { + final int numElements = Arrays.stream(blockSize).reduce(1, (a, b) -> a * b); + + switch (dataType) { + case INT8: + case UINT8: + byte[] uint8Data = new byte[numElements]; + for (int i = 0; i < numElements; i++) { + uint8Data[i] = (byte) random.nextInt(256); + } + return new ByteArrayDataBlock(blockSize, gridPosition, uint8Data); + + case INT16: + case UINT16: + short[] uint16Data = new short[numElements]; + for (int i = 0; i < numElements; i++) { + uint16Data[i] = (short) random.nextInt(65536); + } + return new ShortArrayDataBlock(blockSize, gridPosition, uint16Data); + + case INT32: + case UINT32: + int[] uint32Data = new int[numElements]; + for (int i = 0; i < numElements; i++) { + uint32Data[i] = random.nextInt(); + } + return new IntArrayDataBlock(blockSize, gridPosition, uint32Data); + + case INT64: + case UINT64: + long[] uint64Data = new long[numElements]; + for (int i = 0; i < numElements; i++) { + uint64Data[i] = random.nextLong(); + } + return new LongArrayDataBlock(blockSize, gridPosition, uint64Data); + + case FLOAT32: + float[] floatData = new float[numElements]; + for (int i = 0; i < numElements; i++) { + floatData[i] = random.nextFloat(); + } + return new FloatArrayDataBlock(blockSize, gridPosition, floatData); + + case FLOAT64: + double[] doubleData = new double[numElements]; + for (int i = 0; i < numElements; i++) { + doubleData[i] = random.nextDouble(); + } + return new DoubleArrayDataBlock(blockSize, gridPosition, doubleData); + + default: + throw new IllegalArgumentException("Unsupported data type: " + dataType); + } + } + + private static void verifyCompatibleDataType(DataType expectedType, DataBlock block) { + + Object data = block.getData(); + switch (expectedType) { + case INT8: + case UINT8: + assertTrue("Expected byte array for " + expectedType, data instanceof byte[]); + break; + case INT16: + case UINT16: + assertTrue("Expected short array for " + expectedType, data instanceof short[]); + break; + case INT32: + case UINT32: + assertTrue("Expected int array for " + expectedType, data instanceof int[]); + break; + case INT64: + case UINT64: + assertTrue("Expected long array for " + expectedType, data instanceof long[]); + break; + case FLOAT32: + assertTrue("Expected float array for " + expectedType, data instanceof float[]); + break; + case FLOAT64: + assertTrue("Expected double array for " + expectedType, data instanceof double[]); + break; + default: + throw new IllegalArgumentException("Unsupported data type: " + expectedType); + } + } + + private static void assertDataEquals(DataBlock expected, DataBlock actual) { + + Object expectedData = expected.getData(); + Object actualData = actual.getData(); + + if (expectedData instanceof byte[]) { + assertArrayEquals((byte[]) expectedData, (byte[]) actualData); + } else if (expectedData instanceof short[]) { + assertArrayEquals((short[]) expectedData, (short[]) actualData); + } else if (expectedData instanceof int[]) { + assertArrayEquals((int[]) expectedData, (int[]) actualData); + } else if (expectedData instanceof long[]) { + assertArrayEquals((long[]) expectedData, (long[]) actualData); + } else if (expectedData instanceof float[]) { + assertArrayEquals((float[]) expectedData, (float[]) actualData, 0.0f); + } else if (expectedData instanceof double[]) { + assertArrayEquals((double[]) expectedData, (double[]) actualData, 0.0); + } else { + throw new IllegalArgumentException("Unknown data type"); + } + } + + +} \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java new file mode 100644 index 000000000..90692c710 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BytesCodecTests.java @@ -0,0 +1,208 @@ +package org.janelia.saalfeldlab.n5.codec; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Random; +import java.util.function.IntUnaryOperator; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.ReadData.OutputStreamOperator; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; +import org.junit.BeforeClass; +import org.junit.Test; + +public class BytesCodecTests { + + static Random random; + + @BeforeClass + public static void setup() { + random = new Random(7777); + } + + @Test + public void testEncodeDecodeBytes() { + + // Create a BitShiftBytesCodec with shift value + final BitShiftBytesCodec originalCodec = new BitShiftBytesCodec(3); + + // Test encode/decode roundtrip + final byte[] testData = new byte[12]; + random.nextBytes(testData); + + final ReadData original = ReadData.from(testData); + final ReadData encoded = originalCodec.encode(original); + final ReadData decoded = originalCodec.decode(encoded); + + final byte[] result = decoded.allBytes(); + assertEquals("Length should match", testData.length, result.length); + assertArrayEquals("encoded-decoded bytes should match original", testData, result); + } + + @Test + public void concatenatedBytesCodecTest() throws IOException { + + int N = 16; + ReadData data = ReadData.from( new InputStream() { + @Override + public int read() throws IOException { + return Math.abs(random.nextInt()) % 32; + } + }, N ).materialize(); + + final byte[] bytes = data.allBytes(); + final byte[] expected = new byte[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + expected[i] = (byte)(2 * bytes[i] + 3); + } + + final DataCodec a = new ByteFunctionCodec(x -> 2 * x, x -> x / 2); + final DataCodec b = new ByteFunctionCodec(x -> x + 3, x -> x - 3 ); + final ConcatenatedDataCodec ab = new ConcatenatedDataCodec(new DataCodec[]{a, b}); + + final ReadData encodedData = ab.encode(data).materialize(); + assertArrayEquals(expected, encodedData.allBytes()); + + final ReadData decodedData = ab.decode(encodedData).materialize(); + assertArrayEquals(bytes, decodedData.allBytes()); + } + + public static class ByteFunctionCodec implements DataCodec, DataCodecInfo { + + IntUnaryOperator encoder; + IntUnaryOperator decoder; + + public ByteFunctionCodec( IntUnaryOperator encoder, IntUnaryOperator decoder ) { + this.encoder = encoder; + this.decoder = decoder; + } + + @Override + public String getType() { + return "byteFunction"; + } + + public ReadData decode(ReadData data) { + return data.encode(new ByteFun(decoder)); + } + + public ReadData encode(ReadData data) { + return data.encode(new ByteFun(encoder)); + } + + @Override public DataCodec create() { + + return this; + } + } + + private static class ByteFun implements OutputStreamOperator { + + IntUnaryOperator fun; + public ByteFun(IntUnaryOperator fun) { + this.fun = fun; + } + + @Override + public OutputStream apply(OutputStream o) { + return new OutputStream() { + @Override + public void write(int b) throws IOException { + o.write(fun.applyAsInt(b)); + } + }; + } + } + + @NameConfig.Name(BitShiftBytesCodec.TYPE) + public static class BitShiftBytesCodec implements DataCodec, DataCodecInfo { + @Override public DataCodec create() { + + return this; + } + + private static final String TYPE = "bitshift"; + + @NameConfig.Parameter + private int shift; + + public BitShiftBytesCodec() { + + shift = 0; + } + + public BitShiftBytesCodec(int shift) { + + this.shift = shift; + } + + @Override + public String getType() { + + return TYPE; + } + + @Override + public ReadData decode(ReadData readData) throws N5IOException { + + if (shift == 0) { + return readData; + } + + final byte[] data = readData.allBytes(); + final byte[] decoded = new byte[data.length]; + + // Apply inverse bit shift (right rotate) to decode + for (int i = 0; i < data.length; i++) { + int b = data[i] & 0xFF; + decoded[i] = (byte)((b >>> shift) | (b << (8 - shift))); + } + + return ReadData.from(decoded); + } + + @Override + public ReadData encode(ReadData readData) throws N5IOException { + + if (shift == 0) { + return readData; + } + + byte[] data = readData.allBytes(); + byte[] encoded = new byte[data.length]; + + // Apply bit shift (left rotate) to encode + for (int i = 0; i < data.length; i++) { + int b = data[i] & 0xFF; + encoded[i] = (byte)((b << shift) | (b >>> (8 - shift))); + } + return ReadData.from(encoded); + } + + @Override + public boolean equals(Object obj) { + + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + BitShiftBytesCodec other = (BitShiftBytesCodec)obj; + return shift == other.shift; + } + + @Override + public int hashCode() { + + return Integer.hashCode(shift); + } + + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/compression/CompressionTypesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/compression/CompressionTypesTest.java index 25ecea36f..af467c856 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/compression/CompressionTypesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/compression/CompressionTypesTest.java @@ -26,9 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * - */ package org.janelia.saalfeldlab.n5.compression; import java.lang.reflect.Field; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java new file mode 100644 index 000000000..20b69082a --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java @@ -0,0 +1,93 @@ +package org.janelia.saalfeldlab.n5.demo; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo; +import org.janelia.saalfeldlab.n5.util.GridIterator; + +public class BlockIterators { + + public static void main(String[] args) { + +// blockIterator(); +// shardBlockIterator(); + } + + public static void shardBlockIterator() { + +// final DatasetAttributes attrs = new DatasetAttributes( +// new long[] {12, 8}, // image size +// new int[] {6, 4}, // shard size +// new int[] {2, 2}, // block size +// DataType.UINT8, +// new ShardingCodec( +// new int[] {2, 2}, +// new CodecInfo[] { new N5BlockCodecInfo() }, +// new DeterministicSizeCodecInfo[] { new RawBlockCodecInfo() }, +// IndexLocation.END +// )); +// +// shardPositions(attrs) +// .forEach(x -> System.out.println(Arrays.toString(x))); + } + +// public static void blockIterator() { +// +// final DatasetAttributes attrs = new DatasetAttributes( +// new long[] {12, 8}, +// new int[] {2, 2}, +// DataType.UINT8, +// new RawCompression()); +// +// blockPositions(attrs).forEach(x -> System.out.println(Arrays.toString(x))); +// } +// +// public static long[] blockGridSize(final DatasetAttributes attrs ) { +// // this could be a nice method for DatasetAttributes +// +// return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getBlockSize()[i])).toArray(); +// +// } +// +// public static long[] shardGridSize(final DatasetAttributes attrs ) { +// // this could be a nice method for DatasetAttributes +// +// return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getShardSize()[i])).toArray(); +// +// } +// +// public static Stream blockPositions( DatasetAttributes attrs ) { +// return toStream(new GridIterator(blockGridSize(attrs))); +// } +// +// public static Stream shardPositions( DatasetAttributes attrs ) { +// +// final int[] blocksPerShard = attrs.getBlocksPerShard(); +// return toStream( new GridIterator(shardGridSize(attrs))) +// .flatMap( shardPosition -> { +// +// final int nd = attrs.getNumDimensions(); +// final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]); +// return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); +// }); +// } +// +// public static Stream toStream( final Iterator it ) { +// return StreamSupport.stream( Spliterators.spliteratorUnknownSize( +// it, Spliterator.ORDERED), +// false); +// } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index 68edb9d3f..573cc1f29 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -29,9 +29,9 @@ package org.janelia.saalfeldlab.n5.http; import com.google.gson.Gson; +import com.google.gson.JsonElement; import org.janelia.saalfeldlab.n5.CachedGsonKeyValueN5Reader; import org.janelia.saalfeldlab.n5.CachedGsonKeyValueN5Writer; -import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; @@ -39,6 +39,8 @@ import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import java.io.Serializable; import java.lang.reflect.Field; @@ -50,6 +52,8 @@ import java.util.concurrent.ExecutorService; import java.util.function.Predicate; +import static org.janelia.saalfeldlab.n5.N5KeyValueReader.ATTRIBUTES_JSON; + public class HttpReaderFsWriter implements GsonKeyValueN5Writer { private final GsonKeyValueN5Writer writer; @@ -81,6 +85,11 @@ public HttpRead } + @Override public String getAttributesKey() { + + return writer.getAttributesKey(); + } + @Override public Version getVersion() throws N5Exception { return reader.getVersion(); @@ -255,11 +264,6 @@ public HttpRead writer.createDataset(datasetPath, datasetAttributes); } - @Override public void createDataset(String datasetPath, long[] dimensions, int[] blockSize, DataType dataType, Compression compression) throws N5Exception { - - writer.createDataset(datasetPath, dimensions, blockSize, dataType, compression); - } - @Override public void writeBlock(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { writer.writeBlock(datasetPath, datasetAttributes, dataBlock); } @@ -273,4 +277,29 @@ public HttpRead writer.writeSerializedBlock(object, datasetPath, datasetAttributes, gridPosition); } + + @Override public void setVersion(String path) { + + writer.setVersion(path); + } + + @Override public void writeAttributes(String normalGroupPath, JsonElement attributes) throws N5Exception { + + writer.writeAttributes(normalGroupPath, attributes); + } + + @Override public void setAttributes(String path, JsonElement attributes) throws N5Exception { + + writer.setAttributes(path, attributes); + } + + @Override public void writeAttributes(String normalGroupPath, Map attributes) throws N5Exception { + + writer.writeAttributes(normalGroupPath, attributes); + } + + @Override public void writeBlocks(String datasetPath, DatasetAttributes datasetAttributes, DataBlock... dataBlocks) throws N5Exception { + + writer.writeBlocks(datasetPath, datasetAttributes, dataBlocks); + } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java index 31f3e01af..ca65fd766 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/AbstractKeyValueAccessTest.java @@ -26,9 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * - */ package org.janelia.saalfeldlab.n5.kva; import org.janelia.saalfeldlab.n5.KeyValueAccess; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java index 7e0c7a912..ff1bf517f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java @@ -26,9 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * - */ package org.janelia.saalfeldlab.n5.kva; import static org.junit.Assert.assertArrayEquals; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java index 19e439799..486ae34f3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/HttpKeyValueAccessTest.java @@ -26,9 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * - */ package org.janelia.saalfeldlab.n5.kva; import org.janelia.saalfeldlab.n5.AbstractN5Test; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java new file mode 100644 index 000000000..b478c5189 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java @@ -0,0 +1,92 @@ +package org.janelia.saalfeldlab.n5.serialization; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.janelia.saalfeldlab.n5.Compression; +import org.janelia.saalfeldlab.n5.CompressionAdapter; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.GsonUtils; +import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.saalfeldlab.n5.NameConfigAdapter; +import org.janelia.saalfeldlab.n5.codec.DataCodec; +import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.codec.IdentityCodec; +import org.junit.Before; +import org.junit.Test; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +public class CodecSerialization { + + private Gson gson; + + @Before + public void before() { + + final GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(DataCodecInfo.class, NameConfigAdapter.getJsonAdapter(DataCodecInfo.class)); + gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); + gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.disableHtmlEscaping(); + gson = gsonBuilder.create(); + } + + @Test + public void testCodecSerialization() { + + final IdentityCodec id = new IdentityCodec(); + final JsonObject jsonId = gson.toJsonTree(id).getAsJsonObject(); + final JsonElement expected = gson.fromJson("{\"name\":\"id\"}", JsonElement.class); + assertEquals("identity json", expected, jsonId.getAsJsonObject()); + + final BitShiftBytesCodec codec = new BitShiftBytesCodec(3); + final JsonObject bitShiftJson = gson.toJsonTree(codec).getAsJsonObject(); + final JsonElement expectedBitShift = gson.fromJson( + "{\"name\":\"bitshift\",\"configuration\":{\"shift\":3}}", + JsonElement.class); + assertEquals("bitshift json", expectedBitShift, bitShiftJson); + + final DataCodecInfo deserializedCodecInfo = gson.fromJson(bitShiftJson, DataCodecInfo.class); + // Verify deserialized codec + assertEquals("Deserialized codec should equal original", codec, deserializedCodecInfo); + } + + @Test + public void testSerializeCodecArray() { + + CodecInfo[] codecs = new CodecInfo[]{ + new IdentityCodec() + }; + JsonArray jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); + JsonElement expected = gson.fromJson( + "[{\"name\":\"id\"}]", + JsonElement.class); + assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); + + CodecInfo[] codecsDeserialized = gson.fromJson(expected, CodecInfo[].class); + assertEquals("codecs length not 1", 1, codecsDeserialized.length); + assertTrue("first codec not identity", codecsDeserialized[0] instanceof IdentityCodec); + + codecs = new CodecInfo[]{ + new GzipCompression() + }; + jsonCodecArray = gson.toJsonTree(codecs).getAsJsonArray(); + expected = gson.fromJson( + "[{\"name\":\"gzip\",\"configuration\":{\"level\":-1,\"useZlib\":false}}]", + JsonElement.class); + assertEquals("codec array", expected, jsonCodecArray.getAsJsonArray()); + + codecsDeserialized = gson.fromJson(expected, CodecInfo[].class); + assertEquals("codecs length not 1", 1, codecsDeserialized.length); + assertTrue("second codec not gzip", codecsDeserialized[0] instanceof GzipCompression); + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java new file mode 100644 index 000000000..d213c0713 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java @@ -0,0 +1,88 @@ +package org.janelia.saalfeldlab.n5.shard; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertThrows; + +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; +import org.junit.Assert; +import org.junit.Test; + +public class NestedGridTest { + + private static long absPosition1D(final NestedGrid grid, final int sourcePos, final int targetLevel) { + return grid.absolutePosition(new long[] {sourcePos}, targetLevel)[0]; + } + + @Test + public void testValidateInput() { + int[][] blockSizes = {{3}, {7}, {11}}; + assertThrows(IllegalArgumentException.class, () -> new NestedGrid(blockSizes)); + } + + @Test + public void testAbsolutePosition() { + int[][] blockSizes = {{1}, {3}, {6}, {24}}; + NestedGrid grid = new NestedGrid(blockSizes); + + Assert.assertEquals(38, absPosition1D(grid, 38, 0)); + + Assert.assertEquals(12, absPosition1D(grid, 36, 1)); + Assert.assertEquals(12, absPosition1D(grid, 37, 1)); + Assert.assertEquals(12, absPosition1D(grid, 38, 1)); + + Assert.assertEquals(6, absPosition1D(grid, 38, 2)); + Assert.assertEquals(1, absPosition1D(grid, 38, 3)); + } + + @Test + public void testAbsolutePositionChunkSize() { + int[][] blockSizes = {{10}, {30}, {60}, {240}}; + NestedGrid grid = new NestedGrid(blockSizes); + + Assert.assertEquals(38, absPosition1D(grid, 38, 0)); + Assert.assertEquals(12, absPosition1D(grid, 38, 1)); + Assert.assertEquals(6, absPosition1D(grid, 38, 2)); + Assert.assertEquals(1, absPosition1D(grid, 38, 3)); + } + + private static long relPosition1D(final NestedGrid grid, final int sourcePos, final int targetLevel) { + return grid.relativePosition(new long[] {sourcePos}, targetLevel)[0]; + } + + @Test + public void testRelativePosition() { + int[][] blockSizes = {{1}, {3}, {6}, {24}}; + NestedGrid grid = new NestedGrid(blockSizes); + + Assert.assertEquals(2, relPosition1D(grid, 38, 0)); + Assert.assertEquals(0, relPosition1D(grid, 38, 1)); + Assert.assertEquals(2, relPosition1D(grid, 38, 2)); + Assert.assertEquals(1, relPosition1D(grid, 38, 3)); + + } + + @Test + public void testRelativePositionChunkSize() { + int[][] blockSizes = {{10}, {30}, {60}, {240}}; + NestedGrid grid = new NestedGrid(blockSizes); + + Assert.assertEquals(2, relPosition1D(grid, 38, 0)); + Assert.assertEquals(0, relPosition1D(grid, 38, 1)); + Assert.assertEquals(2, relPosition1D(grid, 38, 2)); + Assert.assertEquals(1, relPosition1D(grid, 38, 3)); + } + + @Test + public void testNd() { + + int[][] blockSizes = {{5, 7}, {5*3, 7*2}}; + NestedGrid grid = new NestedGrid(blockSizes); + System.out.println(grid); + assertArrayEquals(new long[]{1, 2}, grid.absolutePosition(new long[]{1, 2}, 0)); + assertArrayEquals(new long[]{99, 99}, grid.absolutePosition(new long[]{99, 99}, 0)); + + assertArrayEquals(new long[]{0, 0}, grid.absolutePosition(new long[]{0, 1}, 1)); + assertArrayEquals(new long[]{0, 1}, grid.absolutePosition(new long[]{0, 2}, 1)); + assertArrayEquals(new long[]{1, 1}, grid.absolutePosition(new long[]{3, 2}, 1)); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java new file mode 100644 index 000000000..eaa895420 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java @@ -0,0 +1,142 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.Arrays; +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; + +public class RawShardTest { + + + public static void main(String[] args) { + + int[] datablockSize = {3, 3, 3}; + int[] level1ShardSize = {6, 6, 6}; + int[] level2ShardSize = {24, 24, 24}; + + // DataBlocks are 3x3x3 + // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) + // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) + final BlockCodecInfo c0 = new N5BlockCodecInfo(); + final ShardCodecInfo c1 = new DefaultShardCodecInfo( + datablockSize, + c0, + new DataCodecInfo[] {new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[] {new RawCompression()}, + IndexLocation.END + ); + final ShardCodecInfo c2 = new DefaultShardCodecInfo( + level1ShardSize, + c1, + new DataCodecInfo[] {new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[] {new RawCompression()}, + IndexLocation.START + ); + + TestDatasetAttributes attributes = new TestDatasetAttributes( + new long[] {}, + level2ShardSize, + DataType.INT8, + c2, + new RawCompression()); + + final DatasetAccess datasetAccess = attributes.datasetAccess(); + + // TODO: N5Reader/Writer needs to provide a PositionValueAccess implementation on top of its KVA. + // The read/write/deleteBlock methods would getDataAccess() from the DatasetAttributes and call it with that PositionValueAccess. + final PositionValueAccess store = new TestPositionValueAccess(); + + + // --------------------------------------------------------------- + // Some "tests" + // TODO: Turn into unit tests + // --------------------------------------------------------------- + + // write some blocks, filled with constant values + final int[] dataBlockSize = c1.getInnerBlockSize(); + datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 0, 0}, 1)); + datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 0, 0}, 2)); + datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 1, 0}, 3)); + datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 1, 0}, 4)); + datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {3, 2, 1}, 5)); + datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {8, 4, 1}, 6)); + + // verify that the written blocks can be read back with the correct values + checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), true, 1); + checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); + checkBlock(datasetAccess.readBlock(store, new long[] {0, 1, 0}), true, 3); + checkBlock(datasetAccess.readBlock(store, new long[] {1, 1, 0}), true, 4); + checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5); + checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6); + + // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) + datasetAccess.deleteBlock(store, new long[] {0, 0, 0}); + checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); + checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); + + // if a shard becomes empty the corresponding key should be deleted + if ( store.get(new long[] {1, 0, 0}) == null ) { + throw new IllegalStateException("expected non-null readData"); + } + datasetAccess.deleteBlock(store, new long[] {8, 4, 1}); + if ( store.get(new long[] {1, 0, 0}) != null ) { + throw new IllegalStateException("expected null readData"); + } + + // deleting a non-existent block should not fail + datasetAccess.deleteBlock(store, new long[] {0, 0, 8}); + + System.out.println("all good"); + } + + private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { + + if (dataBlock == null) { + if (expectedNonNull) { + throw new IllegalStateException("expected non-null dataBlock"); + } + } else { + if (!expectedNonNull) { + throw new IllegalStateException("expected null dataBlock"); + } + final byte[] bytes = dataBlock.getData(); + for (byte b : bytes) { + if (b != (byte) expectedFillValue) { + throw new IllegalStateException("expected all values to be " + expectedFillValue); + } + } + } + } + + private static DataBlock createDataBlock(int[] size, long[] gridPosition, int fillValue) { + final byte[] bytes = new byte[DataBlock.getNumElements(size)]; + Arrays.fill(bytes, (byte) fillValue); + return new ByteArrayDataBlock(size, gridPosition, bytes); + } + + public static class TestDatasetAttributes extends DatasetAttributes { + + public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType dataType, BlockCodecInfo blockCodecInfo, + DataCodecInfo... dataCodecInfos) { + + super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); + } + + public DatasetAccess datasetAccess() { + + // to make this accessible for the test + return createDatasetAccess(); + } + + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java new file mode 100644 index 000000000..ea1406d83 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java @@ -0,0 +1,105 @@ +package org.janelia.saalfeldlab.n5.shard; + +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.N5FSTest; +import org.janelia.saalfeldlab.n5.N5KeyValueWriter; +import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; +import org.janelia.saalfeldlab.n5.util.GridIterator; +import org.junit.After; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; + +public class ShardIndexTest { + + private static final N5FSTest tempN5Factory = new N5FSTest(); + + @After + public void removeTempWriters() { + + tempN5Factory.removeTempWriters(); + } + + @Test + @Ignore + public void testOffsetIndex() { + + // TODO +// int[] shardBlockGridSize = new int[]{5, 4, 3}; +// ShardIndex index = new ShardIndex( +// shardBlockGridSize, +// IndexLocation.END, new RawBlockCodecInfo()); +// +// GridIterator it = new GridIterator(shardBlockGridSize); +// int i = 0; +// while (it.hasNext()) { +// int j = index.getOffsetIndex(GridIterator.long2int(it.next())); +// assertEquals(i, j); +// i += 2; +// } +// +// shardBlockGridSize = new int[]{5, 4, 3, 13}; +// index = new ShardIndex( +// shardBlockGridSize, +// IndexLocation.END, new RawBlockCodecInfo()); +// +// it = new GridIterator(shardBlockGridSize); +// i = 0; +// while (it.hasNext()) { +// int j = index.getOffsetIndex(GridIterator.long2int(it.next())); +// assertEquals(i, j); +// i += 2; +// } + + } + + @Test + @Ignore + public void writeReadTest() throws IOException { + + // TODO + +// final N5KeyValueWriter writer = (N5KeyValueWriter)tempN5Factory.createTempN5Writer(); +// final KeyValueAccess kva = writer.getKeyValueAccess(); +// +// final int[] shardBlockGridSize = new int[]{6, 5}; +// final IndexLocation indexLocation = IndexLocation.END; +// final IndexCodecAdapter indexCodecAdapter = new IndexCodecAdapter( +// new RawBlockCodecInfo(), +// new Crc32cChecksumCodec() +// ); +// +// final ShardIndex index = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecAdapter); +// index.set(0, 6, new int[]{0, 0}); +// index.set(19, 32, new int[]{1, 0}); +// index.set(93, 111, new int[]{3, 0}); +// index.set(143, 1, new int[]{1, 2}); +// +// final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "indexTest").toString(); +// try ( +// final LockedChannel channel = kva.lockForWriting(path); +// final OutputStream out = channel.newOutputStream() +// ) { +// +// ShardIndex.write(out, index); +// } +// +// final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecAdapter); +// try ( +// final LockedChannel channel = kva.lockForReading(path); +// final InputStream in = channel.newInputStream() +// ) { +// ShardIndex.read(in, indexRead); +// } +// assertEquals(index, indexRead); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java new file mode 100644 index 000000000..76b935d3c --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -0,0 +1,427 @@ +package org.janelia.saalfeldlab.n5.shard; + +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.N5FSTest; +import org.janelia.saalfeldlab.n5.N5FSWriter; +import org.janelia.saalfeldlab.n5.N5KeyValueWriter; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.NameConfigAdapter; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; +import org.janelia.saalfeldlab.n5.codec.CodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import java.io.File; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RunWith(Parameterized.class) +public class ShardTest { + + private static final boolean LOCAL_DEBUG = false; + + private static final N5FSTest tempN5Factory = new N5FSTest() { + + @Override public N5Writer createTempN5Writer() { + + if (LOCAL_DEBUG) { + final N5Writer writer = new ShardedN5Writer("src/test/resources/test.n5"); + writer.remove(""); // Clear old when starting new test + return writer; + } + + final String basePath = new File(tempN5PathName()).toURI().normalize().getPath(); + try { + String uri = new URI("file", null, basePath, null).toString(); + return new ShardedN5Writer(uri); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + return null; + } + + private String tempN5PathName() { + + try { + final File tmpFile = Files.createTempDirectory("n5-shard-test-").toFile(); + tmpFile.delete(); + tmpFile.mkdir(); + tmpFile.deleteOnExit(); + return tmpFile.getCanonicalPath(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + }; + + public static GsonBuilder gsonBuilder() { + return new GsonBuilder(); + } + + @Parameterized.Parameters(name = "IndexLocation({0}), Index ByteOrder({1})") + public static Collection data() { + + final ArrayList params = new ArrayList<>(); + for (IndexLocation indexLoc : IndexLocation.values()) { + for (ByteOrder indexByteOrder : new ByteOrder[]{ByteOrder.BIG_ENDIAN, ByteOrder.LITTLE_ENDIAN}) { + params.add(new Object[]{indexLoc, indexByteOrder}); + } + } + final int numParams = params.size(); + final Object[][] paramArray = new Object[numParams][]; + Arrays.setAll(paramArray, params::get); + return Arrays.asList(paramArray); + } + + @Parameterized.Parameter() + public IndexLocation indexLocation; + + @Parameterized.Parameter(1) + public ByteOrder indexByteOrder; + + @After + public void removeTempWriters() { + + tempN5Factory.removeTempWriters(); + } + + private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] blockSize) { + + DefaultShardCodecInfo blockCodec = new DefaultShardCodecInfo( + blockSize, + new N5BlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[]{new RawCompression()}, + IndexLocation.END); + + return new DatasetAttributes( + dimensions, + shardSize, + DataType.UINT8, + blockCodec); + } + + protected DatasetAttributes getTestAttributes() { + + return getTestAttributes(new long[]{8, 8}, new int[]{4, 4}, new int[]{2, 2}); + } + + @Test + public void writeReadBlocksTest() { + + final N5Writer writer = tempN5Factory.createTempN5Writer(); + + final DatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + new int[]{2, 2} + ); + + final String dataset = "writeReadBlocks"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); + + final int[] blockSize = datasetAttributes.getBlockSize(); + final int numElements = blockSize[0] * blockSize[1]; + + final byte[] data = new byte[numElements]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)((100) + (10) + i); + } + + writer.writeBlocks( + dataset, + datasetAttributes, + /* shard (0, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + + /* shard (1, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), + + /* shard (2, 2) */ + new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) + ); + + final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); + + long[][] keys = new long[][]{ + {0, 0}, + {1, 0}, + {2, 2} + }; + + final long[][] someUnusedKeys = new long[][]{ + {0, 1}, + {1, 1}, + {1, 2}, + {2, 1} + }; + + ensureKeysExist(kva, writer.getURI(), dataset, datasetAttributes, keys); + ensureKeysDoNotExist(kva, writer.getURI(), dataset, datasetAttributes, someUnusedKeys); + + final long[][] blockIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; + for (long[] blockIndex : blockIndices) { + final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + } + + final byte[] data2 = new byte[numElements]; + for (int i = 0; i < data2.length; i++) { + data2[i] = (byte)(10 + i); + } + + writer.writeBlocks( + dataset, + datasetAttributes, + /* shard (0, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data2), + new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data2), + + /* shard (0, 1) */ + new ByteArrayDataBlock(blockSize, new long[]{0, 4}, data2), + new ByteArrayDataBlock(blockSize, new long[]{0, 5}, data2), + + /* shard (2, 2) */ + new ByteArrayDataBlock(blockSize, new long[]{10, 10}, data2)); + + long[][] keys2 = new long[][]{ + {0, 0}, + {1, 0}, + {0, 1}, + {2, 2} + }; + + long[][] someUnusedKeys2 = new long[][]{ + {1, 1}, + {1, 2}, + {2, 1} + }; + + ensureKeysExist(kva, writer.getURI(), dataset, datasetAttributes, keys2); + ensureKeysDoNotExist(kva, writer.getURI(), dataset, datasetAttributes, someUnusedKeys2); + + final long[][] oldBlockIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; + for (long[] blockIndex : oldBlockIndices) { + final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + } + + final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; + final List newBlockIndexList = Arrays.asList(newBlockIndices); + final List> readBlocks = writer.readBlocks(dataset, datasetAttributes, newBlockIndexList); + for (int i = 0; i < newBlockIndices.length; i++) { + final long[] blockIndex = newBlockIndices[i]; + final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])block.getData()); + final DataBlock blockFromReadBlocks = readBlocks.get(i); + Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])blockFromReadBlocks.getData()); + } + } + + private void ensureKeysExist(KeyValueAccess kva, URI uri, String dataset, + DatasetAttributes datasetAttributes, long[][] keys) { + + for (long[] key : keys) { + final String shard = kva.compose(uri, dataset, datasetAttributes.relativeBlockPath(key)); + Assert.assertTrue("Shard at" + shard + "Does not exist", kva.exists(shard)); + } + } + + private void ensureKeysDoNotExist(KeyValueAccess kva, URI uri, String dataset, + DatasetAttributes datasetAttributes, long[][] keys) { + + for (long[] key : keys) { + final String shard = kva.compose(uri, dataset, datasetAttributes.relativeBlockPath(key)); + Assert.assertFalse("Shard at" + shard + " exists but should not.", kva.exists(shard)); + } + } + + @Test + public void writeShardDataSizeTest() { + + // note: this test depends on the use of raw compression + final N5Writer writer = tempN5Factory.createTempN5Writer(); + + int numBlocksPerShard = 16; + final int n5HeaderSizeBytes = 12; // 2 + 2 + 4*2 + final DatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + new int[]{2, 2} + ); + + final String dataset = "writeBlocksShardSize"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); + final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); + + final int[] blockSize = datasetAttributes.getBlockSize(); + final int numElements = blockSize[0] * blockSize[1]; + + final byte[] data = new byte[numElements]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)((100) + (10) + i); + } + + writer.writeBlocks( + dataset, + datasetAttributes, + /* shard (0, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + + /* shard (1, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), + + /* shard (2, 2) */ + new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) + ); + + final int indexSizeBytes = numBlocksPerShard * 16; // 8 bytes per long * + final int blockDataSizeBytes = numElements + n5HeaderSizeBytes; + + // shard 0,0 has 4 blocks so should be this size: + long shard00SizeBytes = indexSizeBytes + 4 * blockDataSizeBytes; + long shard10SizeBytes = indexSizeBytes + 2 * blockDataSizeBytes; + long shard22SizeBytes = indexSizeBytes + 1 * blockDataSizeBytes; + + final String[][] keys = new String[][]{ + {dataset, "0", "0"}, + {dataset, "1", "0"}, + {dataset, "2", "2"} + }; + + long[] shardSizes = new long[]{ + shard00SizeBytes, + shard10SizeBytes, + shard22SizeBytes + }; + + int i = 0; + for (String[] key : keys) { + final String shardPath = kva.compose(writer.getURI(), key); + Assert.assertEquals("shard at " + shardPath + " was the wrong size", shardSizes[i++], kva.size(shardPath)); + } + + } + + @Test + public void readBlocksTest() { + + final N5Writer n5 = tempN5Factory.createTempN5Writer(); + final DatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + new int[]{2, 2}); + + final String dataset = "writeReadBlocks"; + final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; + final List> readBlocks = n5.readBlocks(dataset, datasetAttributes, Arrays.asList(newBlockIndices)); + } + + @Test + public void writeReadBlockTest() { + + final GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)tempN5Factory.createTempN5Writer(); + final DatasetAttributes datasetAttributes = getTestAttributes(); + + final String dataset = "writeReadBlock"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); + + final int[] blockSize = datasetAttributes.getBlockSize(); + final DataType dataType = datasetAttributes.getDataType(); + final int numElements = 2 * 2; + + final HashMap writtenBlocks = new HashMap<>(); + + for (int idx1 = 1; idx1 >= 0; idx1--) { + for (int idx2 = 1; idx2 >= 0; idx2--) { + final long[] gridPosition = {idx1, idx2}; + final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(blockSize, gridPosition, numElements); + byte[] data = dataBlock.getData(); + for (int i = 0; i < data.length; i++) { + data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); + } + writer.writeBlock(dataset, datasetAttributes, dataBlock); + + final DataBlock block = writer.readBlock(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + + for (Map.Entry entry : writtenBlocks.entrySet()) { + final long[] otherGridPosition = entry.getKey(); + final byte[] otherData = entry.getValue(); + final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); + Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); + } + + writtenBlocks.put(gridPosition, data); + } + } + } + + /** + * An N5Writer that serializing the sharding codecs, enabling testing of + * shard functionality, despite the fact that the N5 format does not support + * sharding. + */ + public static class ShardedN5Writer extends N5FSWriter { + + Gson gson; + + public ShardedN5Writer(String basePath) { + + this(basePath, new GsonBuilder()); + } + + public ShardedN5Writer(String basePath, GsonBuilder gsonBuilder) { + + super(basePath); + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); + gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBlockCodecInfo.byteOrderAdapter); + gsonBuilder.disableHtmlEscaping(); + gson = gsonBuilder.create(); + } + + @Override + public Gson getGson() { + + // the super constructor needs the gson instance, unfortunately + return gson == null ? super.gson : gson; + } + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java new file mode 100644 index 000000000..29e609395 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java @@ -0,0 +1,57 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; + +public class TestPositionValueAccess implements PositionValueAccess { + + private final Map map = new HashMap<>(); + + @Override + public ReadData get(final long[] key) { + final byte[] bytes = map.get(new Key(key)); + return bytes == null ? null : ReadData.from(bytes); + } + + @Override + public void put(final long[] key, final ReadData data) { + final byte[] bytes = data == null ? null : data.allBytes(); + map.put(new Key(key), bytes); + } + + @Override + public boolean remove(final long[] key) throws N5IOException { + return map.remove(new Key(key)) != null; + } + + private static class Key { + + private final long[] data; + + Key(long[] data) { + + this.data = data; + } + + @Override + public final boolean equals(final Object o) { + + if (!(o instanceof Key)) { + return false; + } + final Key key = (Key)o; + return Arrays.equals(data, key.data); + } + + @Override + public int hashCode() { + + return Arrays.hashCode(data); + } + } +} diff --git a/src/test/resources/backward/data-1.5.0.n5/attributes.json b/src/test/resources/backward/data-1.5.0.n5/attributes.json new file mode 100644 index 000000000..c050c3d21 --- /dev/null +++ b/src/test/resources/backward/data-1.5.0.n5/attributes.json @@ -0,0 +1 @@ +{"n5":"1.5.0"} \ No newline at end of file diff --git a/src/test/resources/backward/data-1.5.0.n5/raw/0/0 b/src/test/resources/backward/data-1.5.0.n5/raw/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..a129c42df4f256d1b075f9af34ea7c8440562ebb GIT binary patch literal 32 UcmZQzU|?ckU| Date: Mon, 3 Nov 2025 16:03:49 -0500 Subject: [PATCH 416/423] test: DatasetAttributesTest test NestedGrid --- .../saalfeldlab/n5/DatasetAttributesTest.java | 119 ++---------------- 1 file changed, 13 insertions(+), 106 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java index 9f2d52810..63d3d7e5d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java @@ -28,29 +28,16 @@ */ package org.janelia.saalfeldlab.n5; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; -import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; import org.janelia.saalfeldlab.n5.shard.DefaultShardCodecInfo; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; -import org.janelia.saalfeldlab.n5.util.GridIterator; -import org.janelia.saalfeldlab.n5.util.Position; import org.junit.Test; /** @@ -72,33 +59,37 @@ public void testValidateBlockShardSizesValid() { // This should not throw any exception DatasetAttributes attrs = shardDatasetAttributes(dimensions, shardSize, blockSize, dataType); -// assertEquals(shardSize, attrs.getShardSize()); assertEquals(blockSize, attrs.getBlockSize()); -// assertArrayEquals(new int[]{1, 1, 1}, attrs.getBlocksPerShard()); + NestedGrid grid = attrs.getNestedBlockGrid(); + assertEquals(blockSize, grid.getBlockSize(0)); + assertEquals(shardSize, grid.getBlockSize(1)); // Test case 2: shard size is a multiple of block size shardSize = new int[]{128}; blockSize = new int[]{64}; attrs = shardDatasetAttributes(new long[]{128}, shardSize, blockSize, dataType); -// assertEquals(shardSize, attrs.getShardSize()); assertEquals(blockSize, attrs.getBlockSize()); -// assertArrayEquals(new int[]{2}, attrs.getBlocksPerShard()); + grid = attrs.getNestedBlockGrid(); + assertEquals(blockSize, grid.getBlockSize(0)); + assertEquals(shardSize, grid.getBlockSize(1)); // Test case 3: different multiples per dimension shardSize = new int[]{128, 256, 32, 2}; blockSize = new int[]{32, 64, 32, 1}; attrs = shardDatasetAttributes(new long[]{128, 128, 128, 128}, shardSize, blockSize, dataType ); -// assertEquals(shardSize, attrs.getShardSize()); assertEquals(blockSize, attrs.getBlockSize()); -// assertArrayEquals(new int[]{4, 4, 1, 2}, attrs.getBlocksPerShard()); + grid = attrs.getNestedBlockGrid(); + assertEquals(blockSize, grid.getBlockSize(0)); + assertEquals(shardSize, grid.getBlockSize(1)); // Test case 4: large multiples shardSize = new int[]{1024, 2048, 512}; blockSize = new int[]{32, 64, 16}; attrs = shardDatasetAttributes(dimensions, shardSize, blockSize, dataType); -// assertEquals(shardSize, attrs.getShardSize()); assertEquals(blockSize, attrs.getBlockSize()); -// assertArrayEquals(new int[]{32, 32, 32}, attrs.getBlocksPerShard()); + grid = attrs.getNestedBlockGrid(); + assertEquals(blockSize, grid.getBlockSize(0)); + assertEquals(shardSize, grid.getBlockSize(1)); } private static DatasetAttributes shardDatasetAttributes( @@ -160,88 +151,4 @@ public void testValidateBlockShardSizesInvalid() { () -> shardDatasetAttributes(dimensions, new int[]{0, 64, 64}, new int[]{64, 64, 64}, dataType)); } -// @Test -// public void testShardProperties() { -// -// final long[] arraySize = new long[]{16, 16}; -// final int[] shardSize = new int[]{16, 16}; -// final long[] shardPosition = new long[]{1, 1}; -// final int[] blkSize = new int[]{4, 4}; -// -// final DatasetAttributes dsetAttrs = new DatasetAttributes( -// arraySize, -// shardSize, -// blkSize, -// DataType.UINT8, -// new ShardingCodec( -// blkSize, -// new CodecInfo[]{new N5BlockCodecInfo()}, -// new DeterministicSizeCodecInfo[]{new RawBlockCodecInfo()}, -// IndexLocation.END -// ) -// ); -// -// final InMemoryShard shard = new InMemoryShard(dsetAttrs, shardPosition, null); -// -// assertArrayEquals(new int[]{4, 4}, shard.getBlockGridSize()); -// -// assertArrayEquals(new long[]{0, 0}, dsetAttrs.getShardPositionForBlock(0, 0)); -// assertArrayEquals(new long[]{1, 1}, dsetAttrs.getShardPositionForBlock(5, 5)); -// assertArrayEquals(new long[]{1, 0}, dsetAttrs.getShardPositionForBlock(5, 0)); -// assertArrayEquals(new long[]{0, 1}, dsetAttrs.getShardPositionForBlock(0, 5)); -// -// assertArrayEquals(new int[]{0, 0}, shard.getRelativeBlockPosition(4, 4)); -// assertArrayEquals(new int[]{1, 1}, shard.getRelativeBlockPosition(5, 5)); -// assertArrayEquals(new int[]{2, 2}, shard.getRelativeBlockPosition(6, 6)); -// assertArrayEquals(new int[]{3, 3}, shard.getRelativeBlockPosition(7, 7)); -// } - -// @Test -// public void testShardGrouping() { -// -// final long[] arraySize = new long[]{8, 12}; -// final int[] shardSize = new int[]{4, 6}; -// final int[] blkSize = new int[]{2, 3}; -// -// final DatasetAttributes attrs = new DatasetAttributes( -// arraySize, -// shardSize, -// blkSize, -// DataType.UINT8, -// new ShardingCodec( -// blkSize, -// new CodecInfo[]{ new N5BlockCodecInfo() }, -// new DeterministicSizeCodecInfo[]{new RawBlockCodecInfo()}, -// IndexLocation.END -// ) -// ); -// -// List blockPositions = blockPositions(attrs).collect(Collectors.toList()); -// final Map> result = attrs.groupBlockPositions(blockPositions); -// -// // there are four shards in this image -// assertEquals(4, result.size()); -// -// // there are four blocks per shard in this image -// result.values().forEach(x -> assertEquals(4, x.size())); -// } - -// private static Stream blockPositions( final DatasetAttributes attrs ) { -// -// final int nd = attrs.getNumDimensions(); -// final int[] blocksPerShard = attrs.getBlocksPerShard(); -// return toStream( new GridIterator(attrs.shardsPerDataset())) -// .flatMap( shardPosition -> { -// final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]); -// return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); -// }); -// } - - private static Stream toStream( final Iterator it ) { - - return StreamSupport.stream( - Spliterators.spliteratorUnknownSize(it, Spliterator.ORDERED), - false); - } - } \ No newline at end of file From fd9b715d4ac6e8eb43708a43e99183cb31acfe14 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 3 Nov 2025 16:37:35 -0500 Subject: [PATCH 417/423] doc: javadoc fixes --- .../org/janelia/saalfeldlab/n5/N5Reader.java | 4 ++ .../org/janelia/saalfeldlab/n5/N5Writer.java | 1 - .../readdata/segment/SegmentedReadData.java | 63 +++++++++++++------ .../n5/shard/PositionValueAccess.java | 19 +++--- .../saalfeldlab/n5/shard/RawShard.java | 11 +++- .../saalfeldlab/n5/shard/ShardCodecInfo.java | 13 +++- 6 files changed, 77 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index e7004cdf3..c378dda68 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -284,6 +284,8 @@ T getAttribute( /** * Reads a {@link DataBlock}. * + * @param + * the DataBlock data type * @param pathName * dataset path * @param datasetAttributes @@ -305,6 +307,8 @@ DataBlock readBlock( * Implementations may optimize / batch read operations when possible, e.g. * in the case that the datasets are sharded. * + * @param + * the DataBlock data type * @param pathName * dataset path * @param datasetAttributes diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index aec867d40..6c3112441 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -220,7 +220,6 @@ default void createDataset( * @param dimensions the dataset dimensions * @param blockSize the block size * @param dataType the data type - * @param blockCodecInfo the block codec * @param compression the compression */ default void createDataset( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java index a2305d655..cbd5caf56 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/segment/SegmentedReadData.java @@ -24,9 +24,14 @@ interface SegmentsAndData { } /** - * Wrap {@code readData} and create one segment comprising the entire {@code + * Wrap a {@link ReadData} and create one segment comprising the entire + * {@code * readData}. The segment can be retrieved as the first (and only) element * of {@link SegmentedReadData#segments()}. + * + * @param readData + * the ReadData to wrap + * @return the SegmentedReadData */ static SegmentedReadData wrap(ReadData readData) { return new DefaultSegmentedReadData(readData); @@ -35,8 +40,15 @@ static SegmentedReadData wrap(ReadData readData) { /** * Wrap {@code readData} and create segments at the given locations. The * order of segments in the returned {@link SegmentsAndData#segments()} list - * matches the order of the given {@code locations} (while the {@link - * #segments} in the {@link SegmentsAndData#data()} are ordered by offset). + * matches the order of the given {@code locations} (while the + * {@link #segments} in the {@link SegmentsAndData#data()} are ordered by + * offset). + * + * @param readData + * the ReadData to wrap + * @param locations + * the ranges for segments + * @return the SegmentsAndData */ static SegmentsAndData wrap(ReadData readData, Range... locations) { return wrap(readData, Arrays.asList(locations)); @@ -45,8 +57,15 @@ static SegmentsAndData wrap(ReadData readData, Range... locations) { /** * Wrap {@code readData} and create segments at the given locations. The * order of segments in the returned {@link SegmentsAndData#segments()} list - * matches the order of the given {@code locations} (while the {@link - * #segments} in the {@link SegmentsAndData#data()} are ordered by offset). + * matches the order of the given {@code locations} (while the + * {@link #segments} in the {@link SegmentsAndData#data()} are ordered by + * offset). + * + * @param readData + * the ReadData to wrap + * @param locations + * the ranges for segments + * @return the SegmentsAndData */ static SegmentsAndData wrap(ReadData readData, List locations) { return DefaultSegmentedReadData.wrap(readData, locations); @@ -57,41 +76,45 @@ static SegmentsAndData wrap(ReadData readData, List locations) { * given {@code readDatas}. The concatenation contains the segments of all * concatenated {@code readData}s with appropriately offset locations. *

        - * In particular, it is also possible to concatenate {@code SegmentedReadData}s - * with (yet) unknown length. (This is useful for postponing compression of - * DataBlocks until they are actually written.) In that case, segment locations - * are only available after all lengths become known. This happens when - * concatenation (or all its constituents) is {@link #materialize() - * materialized} or {@link #writeTo(OutputStream) written}. + * In particular, it is also possible to concatenate + * {@code SegmentedReadData}s with (yet) unknown length. (This is useful for + * postponing compression of DataBlocks until they are actually written.) In + * that case, segment locations are only available after all lengths become + * known. This happens when concatenation (or all its constituents) is + * {@link #materialize() materialized} or {@link #writeTo(OutputStream) + * written}. + * + * @param readDatas + * a list of ReadDatra to concatenate + * @return the SegmentedReadData comprising all the input readDatas */ static SegmentedReadData concatenate(List readDatas) { return new ConcatenatedReadData(readDatas); } - - /** * Returns the location of {@code segment} in this {@code ReadData}. *

        - * Note that this {@code ReadData} is not necessarily the source of the segment. + * Note that this {@code ReadData} is not necessarily the source of the + * segment. *

        * The returned {@code Range} may be {@code {offset=0, length=-1}}, which - * means that the segment comprises this whole {@code ReadData} (and the length of - * this {@code ReadData} is not yet known). + * means that the segment comprises this whole {@code ReadData} (and the + * length of this {@code ReadData} is not yet known). * * @param segment - * the segment id + * the segment id * * @return location of the segment, or null * * @throws IllegalArgumentException - * if the segment is not contained in this ReadData + * if the segment is not contained in this ReadData */ Range location(Segment segment) throws IllegalArgumentException; /** - * Return all segments (fully) contained in this {@code ReadData}, ordered by location - * (that is, sorted by {@link Range#COMPARATOR}). + * Return all segments (fully) contained in this {@code ReadData}, ordered + * by location (that is, sorted by {@link Range#COMPARATOR}). * * @return all segments contained in this {@code ReadData}. */ diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java index f39819067..a9cd70319 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java @@ -17,7 +17,15 @@ public interface PositionValueAccess { /** - * @return ReadData for the given key or {@code null} if the key doesn't exist + * Gets the {@link ReadData} for the DataBlock (or shard) at the given + * position in the block (or shard) grid. + * + * @param key + * The position of the data block or shard + * @return ReadData for the given key or {@code null} if the key doesn't + * exist + * @throws N5Exception.N5IOException + * if an error occurs while reading */ ReadData get(long[] key) throws N5Exception.N5IOException; @@ -53,14 +61,9 @@ class KvaPositionValueAccess implements PositionValueAccess { } /** - * Constructs the path for a shard or data block at a given - * grid position. - *
        - * If the gridPosition passed in refers to shard position in a sharded - * dataset, this will return the path to the shard key. + * Constructs the absolute path for a data block (or shard) at a given grid + * position. * - * @param normalPath - * normalized dataset path * @param gridPosition * to the target data block * @return the absolute path to the data block ad gridPosition diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java index 690b3d180..0a8cb5a7d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java @@ -26,15 +26,20 @@ public class RawShard { } /** - * The ReadData from which the shard was constructed, or {@code null} - * for a new empty shard. + * The ReadData from which the shard was constructed, or {@code null} for a + * new empty shard. + * + * @return this shard's source ReadData, or null. */ public SegmentedReadData sourceData() { return sourceData; } /** - * Maps grid position of shard elements to {@link Segment}s. + * Maps grid position of shard elements to {@link Segment}s that give the + * byte range for the blocks in this shard. + * + * @return an NDArray of segments */ public NDArray index() { return index; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java index 0f2d2150f..dcc40f3d7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java @@ -9,26 +9,35 @@ public interface ShardCodecInfo extends BlockCodecInfo { /** * Chunk size of each shard element (either nested shard or DataBlock) + * + * @return the size of each shard element */ int[] getInnerBlockSize(); /** - * BlockCodec for shard elements (either nested shard or DataBlock) + * BlockCodecInfo for shard elements (either nested shard or DataBlock) + * + * @return the BlockCodecInfo for DataBlocks in this shard */ BlockCodecInfo getInnerBlockCodecInfo(); /** - * DataCodecs for inner BlockCodec + * @return the collection of DataCodecInfos applied to data blocks for this + * shard. */ DataCodecInfo[] getInnerDataCodecInfos(); /** * BlockCodec for shard index + * + * @return the BlockCodecInfo for this shard's index */ BlockCodecInfo getIndexBlockCodecInfo(); /** * Deterministic-size DataCodecs for index BlockCodec + * + * @return the collection of DataCodecInfos for this shard's index */ DataCodecInfo[] getIndexDataCodecInfos(); From 0b8d4c9eea6ce630340c4bd45ebaddf25ff3efb0 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 4 Nov 2025 15:05:50 -0500 Subject: [PATCH 418/423] fix: correctly handle redirect for `isFile` and `isDirectory()` call for HTTP KVA Signed-off-by: Caleb Hulbert --- .../org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index 2d50cb9ad..a1462aa16 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -158,7 +158,7 @@ public boolean exists(final String normalPath) { public boolean isDirectory(final String normalPath) { try { - requireValidHttpResponse(getDirectoryPath(normalPath), HEAD, (code, msg,http) -> { + requireValidHttpResponse(getDirectoryPath(normalPath), HEAD, false, (code, msg,http) -> { final N5Exception cause = validExistsResponse(code, "Error checking directory: " + normalPath, msg, true); if (code >= 300 && code < 400) { final String redirectLocation = http.getHeaderField("Location"); @@ -200,7 +200,7 @@ public boolean isFile(final String normalPath) { /* Files must not end in `/` And Don't accept a redirect to a location ending in `/` */ try { - requireValidHttpResponse(getFilePath(normalPath), HEAD, (code, msg, http) -> { + requireValidHttpResponse(getFilePath(normalPath), HEAD, false, (code, msg, http) -> { final N5Exception cause = validExistsResponse(code, "Error accessing file: " + normalPath, msg, true); if (code >= 300 && code < 400) { final String redirectLocation = http.getHeaderField("Location"); @@ -316,12 +316,17 @@ private HttpURLConnection requireValidHttpResponse(String uri, String method, St } private HttpURLConnection requireValidHttpResponse(String uri, String method, TriFunction filterCode) throws N5Exception { + return requireValidHttpResponse(uri, method, true, filterCode); + } + + private HttpURLConnection requireValidHttpResponse(String uri, String method, boolean followRedirects, TriFunction filterCode) throws N5Exception { final int code; final HttpURLConnection http; final String responseMsg; try { http = httpRequest(uri, method); + http.setInstanceFollowRedirects(followRedirects); code = http.getResponseCode(); responseMsg = http.getResponseMessage(); } catch (IOException e) { From 18702dc879cd1d87362ada430be70b19c94491f9 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 4 Nov 2025 15:06:26 -0500 Subject: [PATCH 419/423] fix(test): get cache field from N5KeyValueReader class Signed-off-by: Caleb Hulbert --- .../saalfeldlab/n5/http/HttpReaderFsWriter.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index 573cc1f29..aba51a4d0 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -39,8 +39,7 @@ import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.N5KeyValueReader; import java.io.Serializable; import java.lang.reflect.Field; @@ -70,8 +69,13 @@ public HttpRead if (cachedReader.cacheMeta()) { /* Hack necessary to test HTTP reader caching without creating the data entirely first */ try { - // Access the private 'cache' field in the reader - final Field cacheField = reader.getClass().getDeclaredField("cache"); + // Access the private 'cache' field in the reader (or the N5KeyValueReader as a fallback) + Field cacheField; + try { + cacheField = reader.getClass().getDeclaredField("cache"); + } catch (NoSuchFieldException e) { + cacheField = N5KeyValueReader.class.getDeclaredField("cache"); + } cacheField.setAccessible(true); // Set the value of 'cache' to the one from writer.getCache() From 19903c417bac4301400053ac2a86d8f2389fad48 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 4 Nov 2025 15:09:22 -0500 Subject: [PATCH 420/423] refactor: don't explicitly handle the cache case in createGroup guard clauses, the normal calls handle this when it is cached Signed-off-by: Caleb Hulbert --- .../n5/CachedGsonKeyValueN5Writer.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index 3801e671b..d402eeddf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -57,19 +57,11 @@ default void createGroup(final String path) throws N5Exception { // avoid hitting the backend if this path is already a group according to the cache // else if exists is true (then a dataset is present) so throw an exception to avoid // overwriting / invalidating existing data - if (cacheMeta()) { - if (getCache().isGroup(normalPath, getAttributesKey())) - return; - else if (getCache().exists(normalPath, getAttributesKey())) { - throw new N5Exception("Can't make a group on existing path."); - } - } + if (groupExists(normalPath)) + return; + else if (datasetExists(normalPath)) + throw new N5Exception("Can't make a group on existing dataset."); - // N5Writer.super.createGroup(path); - /* - * the lines below duplicate the single line above but would have to call - * normalizeGroupPath again the below duplicates code, but avoids extra work - */ getKeyValueAccess().createDirectories(absoluteGroupPath(normalPath)); if (cacheMeta()) { From 453e8394f65ce9d462fd39b5d629da381b36702d Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 5 Nov 2025 15:33:49 -0500 Subject: [PATCH 421/423] style: remove commented old methods --- .../saalfeldlab/n5/DatasetAttributes.java | 149 ------------------ 1 file changed, 149 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index b3e1890c1..ad02e518f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -218,160 +218,11 @@ public int getNumDimensions() { return dimensions.length; } -// public int[] getShardSize() { -// -// return shardSize; -// } - public int[] getBlockSize() { return blockSize; } -// /** -// * Returns the number of blocks per dimension for each shard. -// * -// * @return the blocks per shard -// */ -// public int[] getBlocksPerShard() { -// -// final int[] shardSize = getShardSize(); -// final int nd = getNumDimensions(); -// final int[] blocksPerShard = new int[nd]; -// final int[] blockSize = getBlockSize(); -// for (int i = 0; i < nd; i++) -// blocksPerShard[i] = shardSize[i] / blockSize[i]; -// -// return blocksPerShard; -// } - -// /** -// * Returns the number of shards per dimension for this dataset. -// * -// * @return shards per dataset -// */ -// public long[] shardsPerDataset() { -// -// return IntStream.range(0, getNumDimensions()) -// .mapToLong(i -> (long)Math.ceil((double)getDimensions()[i] / getShardSize()[i])) -// .toArray(); -// } -// -// /** -// * Returns the total number of blocks in each shard. -// * -// * @return number of blocks in a shard -// */ -// public long getNumBlocksPerShard() { -// -// return Arrays.stream(getBlocksPerShard()).reduce(1, (x, y) -> x * y); -// } -// -// /** -// * Given a block's position relative to the dataset, returns the position of -// * the shard containing that block. -// * -// * @param blockGridPosition position of a block relative to the dataset -// * @return the position of the containing shard in the shard grid -// */ -// public long[] getShardPositionForBlock(final long... blockGridPosition) { -// -// final int[] blocksPerShard = getBlocksPerShard(); -// final long[] shardGridPosition = new long[blockGridPosition.length]; -// for (int i = 0; i < shardGridPosition.length; i++) { -// shardGridPosition[i] = (int)Math.floor((double)blockGridPosition[i] / blocksPerShard[i]); -// } -// -// return shardGridPosition; -// } -// -// /** -// * Given a {@code datasetRelativeBlockPosition} returns the position -// * relative the the shard at position {@code shardPosition}. -// * -// * @param shardPosition position of the shard -// * @param datasetRelativeBlockPosition position of the block relative to the dataset -// * @return position of the block relative to the shard -// */ -// public int[] getShardRelativeBlockPosition(final long[] shardPosition, final long[] datasetRelativeBlockPosition) { -// -// final long[] shardPos = getShardPositionForBlock(datasetRelativeBlockPosition); -// if (!Arrays.equals(shardPosition, shardPos)) -// return null; -// -// final int[] shardSize = getBlocksPerShard(); -// final int[] shardRelativeBlockPosition = new int[shardSize.length]; -// for (int i = 0; i < shardSize.length; i++) { -// shardRelativeBlockPosition[i] = (int)(datasetRelativeBlockPosition[i] % shardSize[i]); -// } -// return shardRelativeBlockPosition; -// } -// -// /** -// * Given a {@code shardRelativeBlockPosition} relative to the shard at -// * position {@code shardPosition}, returns the block' position relative the -// * dataset. -// * -// * @param shardPosition position of the shard -// * @param shardRelativeBlockPosition position of the block relative to the shard -// * @return position of the block relative to the dataset -// */ -// public long[] getBlockPositionFromShardPosition(final long[] shardPosition, final int[] shardRelativeBlockPosition) { -// -// final int[] shardBlockSize = getBlocksPerShard(); -// final long[] datasetRelativeBlockPosition = new long[getNumDimensions()]; -// for (int i = 0; i < getNumDimensions(); i++) { -// datasetRelativeBlockPosition[i] = (shardPosition[i] * shardBlockSize[i]) + (shardRelativeBlockPosition[i]); -// } -// -// return datasetRelativeBlockPosition; -// } -// -// /** -// * Returns the number of shards per dimension for the dataset. -// * -// * @return the size of the shard grid of a dataset -// */ -// public int[] getShardBlockGridSize() { -// -// final int nd = getNumDimensions(); -// final int[] shardBlockGridSize = new int[nd]; -// final int[] blockSize = getBlockSize(); -// for (int i = 0; i < nd; i++) -// shardBlockGridSize[i] = (int)(Math.ceil((double)getDimensions()[i] / blockSize[i])); -// -// return shardBlockGridSize; -// } -// -// public Map> groupBlockPositions(final List blockPositions) { -// -// final TreeMap> map = new TreeMap<>(); -// for (final long[] blockPos : blockPositions) { -// Position shardPos = Position.wrap(getShardPositionForBlock(blockPos)); -// if (!map.containsKey(shardPos)) { -// map.put(shardPos, new ArrayList<>()); -// } -// map.get(shardPos).add(blockPos); -// } -// -// return map; -// } -// -// public Map>> groupBlocks(final List> blocks) { -// -// // figure out how to re-use groupBlockPositions here? -// final TreeMap>> map = new TreeMap<>(); -// for (final DataBlock block : blocks) { -// Position shardPos = Position.wrap(getShardPositionForBlock(block.getGridPosition())); -// if (!map.containsKey(shardPos)) { -// map.put(shardPos, new ArrayList<>()); -// } -// map.get(shardPos).add(block); -// } -// -// return map; -// } - public boolean isSharded() { return blockCodecInfo instanceof ShardCodecInfo; From 944c3aab05bc4a4dc74505b9618b817a930ecb16 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 5 Nov 2025 15:37:25 -0500 Subject: [PATCH 422/423] test: backend shard read behavior * logs number of backend reads when reading blocks from shards --- .../saalfeldlab/n5/shard/ShardTest.java | 170 +++++++++++++++++- 1 file changed, 165 insertions(+), 5 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 76b935d3c..18c0a86cf 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -4,18 +4,22 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.KeyValueAccessReadData; +import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5FSTest; -import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.NameConfigAdapter; import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer; import org.janelia.saalfeldlab.n5.codec.CodecInfo; import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; import org.junit.After; import org.junit.Assert; @@ -27,13 +31,21 @@ import com.google.gson.GsonBuilder; import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -392,23 +404,83 @@ public void writeReadBlockTest() { } } + /** + * Checks how many read calls to the backend are performed for a particular readBlocks + * call. At this time (Nov 4 2025), one read for the index, and one read per block are performed. + */ + public void numReadsTest() { + + final ShardedN5Writer writer = (ShardedN5Writer)tempN5Factory.createTempN5Writer(); + + final DatasetAttributes datasetAttributes = getTestAttributes( + new long[]{24, 24}, + new int[]{8, 8}, + new int[]{2, 2} + ); + + final String dataset = "writeReadBlocks"; + writer.remove(dataset); + writer.createDataset(dataset, datasetAttributes); + + final int[] blockSize = datasetAttributes.getBlockSize(); + final int numElements = blockSize[0] * blockSize[1]; + + final byte[] data = new byte[numElements]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)((100) + (10) + i); + } + + writer.writeBlocks( + dataset, + datasetAttributes, + /* shard (0, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + + /* shard (1, 0) */ + new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), + new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), + + /* shard (2, 2) */ + new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) + ); + + writer.resetNumMaterializeCalls(); + writer.readBlocks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0})); + System.out.println(writer.getNumMaterializeCalls()); + + ArrayList ptList = new ArrayList<>(); + ptList.add(new long[] {0,0}); + ptList.add(new long[] {0,1}); + ptList.add(new long[] {1,0}); + ptList.add(new long[] {1,1}); + + writer.resetNumMaterializeCalls(); + writer.readBlocks(dataset, datasetAttributes, ptList); + System.out.println(writer.getNumMaterializeCalls()); + System.out.println(""); + } + /** * An N5Writer that serializing the sharding codecs, enabling testing of * shard functionality, despite the fact that the N5 format does not support * sharding. */ - public static class ShardedN5Writer extends N5FSWriter { + public static class ShardedN5Writer extends N5KeyValueWriter { Gson gson; public ShardedN5Writer(String basePath) { - this(basePath, new GsonBuilder()); + super( new TrackingFileSystemKeyValueAccess(FileSystems.getDefault()), + basePath, new GsonBuilder(), false); } public ShardedN5Writer(String basePath, GsonBuilder gsonBuilder) { - super(basePath); + this(basePath); gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(CodecInfo.class, NameConfigAdapter.getJsonAdapter(CodecInfo.class)); gsonBuilder.registerTypeHierarchyAdapter(ByteOrder.class, RawBlockCodecInfo.byteOrderAdapter); @@ -416,12 +488,100 @@ public ShardedN5Writer(String basePath, GsonBuilder gsonBuilder) { gson = gsonBuilder.create(); } + public void resetNumMaterializeCalls() { + ((TrackingFileSystemKeyValueAccess)super.getKeyValueAccess()).numMaterializeCalls = 0; + } + + public int getNumMaterializeCalls() { + return ((TrackingFileSystemKeyValueAccess)super.getKeyValueAccess()).numMaterializeCalls; + } + @Override public Gson getGson() { - // the super constructor needs the gson instance, unfortunately return gson == null ? super.gson : gson; } } + private static class TrackingFileSystemKeyValueAccess extends FileSystemKeyValueAccess { + + private int numMaterializeCalls = 0; + + protected TrackingFileSystemKeyValueAccess(FileSystem fileSystem) { + super(fileSystem); + } + + @Override + public ReadData createReadData(final String normalPath) { + return new KeyValueAccessReadData(new TrackingFileLazyRead(normalPath)); + } + + private class TrackingFileLazyRead implements LazyRead { + + private final String normalKey; + + TrackingFileLazyRead(String normalKey) { + this.normalKey = normalKey; + } + + @Override + public long size() { + return TrackingFileSystemKeyValueAccess.this.size(normalKey); + } + + @Override + public ReadData materialize(final long offset, final long length) { + + numMaterializeCalls++; + + try (final TrackingLockedFileChannel lfs = new TrackingLockedFileChannel(normalKey, true); + final FileChannel channel = lfs.getFileChannel()) { + + channel.position(offset); + if (length > Integer.MAX_VALUE) + throw new IOException("Attempt to materialize too large data"); + + final long channelSize = channel.size(); + if (!validBounds(channelSize, offset, length)) + throw new IndexOutOfBoundsException(); + + final int sz = (int)(length < 0 ? channelSize : length); + final byte[] data = new byte[sz]; + final ByteBuffer buf = ByteBuffer.wrap(data); + channel.read(buf); + return ReadData.from(data); + + } catch (final NoSuchFileException e) { + throw new N5NoSuchKeyException("No such file", e); + } catch (IOException | UncheckedIOException e) { + throw new N5Exception.N5IOException(e); + } + } + } + + private class TrackingLockedFileChannel extends LockedFileChannel { + + protected TrackingLockedFileChannel(String path, boolean readOnly) throws IOException { + super(path, readOnly); + } + + @Override + protected FileChannel getFileChannel() { + return channel; + } + } + + private static boolean validBounds(long channelSize, long offset, long length) { + + if (offset < 0) + return false; + else if (channelSize > 0 && offset >= channelSize) // offset == 0 and arrayLength == 0 is okay + return false; + else if (length >= 0 && offset + length > channelSize) + return false; + + return true; + } + } + } From 8477283b1e618c86fd61b13a12b9e27bb4e68ad5 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 5 Nov 2025 15:36:01 -0500 Subject: [PATCH 423/423] refactor: expose `groupInnerPositions` and move it to `DatasetAccess` interface as a static method Signed-off-by: Caleb Hulbert --- .../saalfeldlab/n5/shard/DatasetAccess.java | 39 +++++++++++++++++++ .../n5/shard/DefaultDatasetAccess.java | 36 +---------------- 2 files changed, 41 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index 541c1cd8d..a7a2f6d99 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -3,8 +3,13 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; +import java.util.TreeMap; /** * Wrap an instantiated DataBlock/shard codec hierarchy to implement (single and @@ -26,4 +31,38 @@ public interface DatasetAccess { void writeBlocks(PositionValueAccess kva, List> blocks); NestedGrid getGrid(); + + /** + * Sort a list of {@link NestedPosition}s by their parent level {@link NestedPosition}. + * nestedPositions are grouped at `outerLevel`. + * If {@code NestedPosition.level()} level is already equivalent to {@code NestedGrid.numLevels() - 1}, nestedPositions is returned. + * + * @param grid to sort the blocky by + * @param innerPositions to group per shard. + * @param outerLevel of the outerLevel shard position to group by. must be in range {@code [1, NestedGrid.numLevels() - 1]} + * @return map of outerLevel shard positions to inner level block positions + */ + static Collection> groupInnerPositions(final NestedGrid grid, final List innerPositions, final int outerLevel) { + + if (outerLevel < 1 || outerLevel >= grid.numLevels()) + throw new IllegalArgumentException("outerLevel must be in range [1, grid.numLevels() - 1]"); + + if (innerPositions.isEmpty()) + return Collections.emptyList(); + + final TreeMap> blocksPerShard = new TreeMap<>(); + for (T nestedPosition : innerPositions) { + final NestedPosition outerNestedPosition; + if (nestedPosition.level() == outerLevel) + outerNestedPosition = nestedPosition; + else + outerNestedPosition = new NestedPosition(grid, nestedPosition.absolute(0), outerLevel); + + + final List blocks = blocksPerShard.computeIfAbsent(outerNestedPosition, it -> new ArrayList<>()); + blocks.add(nestedPosition); + } + return blocksPerShard.values(); + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index e14e835a1..43029582d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -5,7 +5,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.TreeMap; import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.N5Exception; @@ -16,6 +15,8 @@ import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; +import static org.janelia.saalfeldlab.n5.shard.DatasetAccess.*; + public class DefaultDatasetAccess implements DatasetAccess { private final NestedGrid grid; @@ -328,39 +329,6 @@ private static ReadData getExistingReadData(final PositionValueAccess pva, final } } - /** - * Sort a list of {@link NestedPosition}s by their parent level {@link NestedPosition}. - * nestedPositions are grouped at `outerLevel`. - * If {@code NestedPosition.level()} level is already equivalent to {@code NestedGrid.numLevels() - 1}, nestedPositions is returned. - * - * @param grid to sort the blocky by - * @param nestedPositions to group per shard. - * @param outerLevel of the outerLevel shard position to group by. must be in range {@code [1, NestedGrid.numLevels() - 1]} - * @return map of outerLevel shard positions to inner level block positions - */ - private static Collection> groupInnerPositions(final NestedGrid grid, final List nestedPositions, final int outerLevel) { - - if (outerLevel < 1 || outerLevel >= grid.numLevels()) - throw new IllegalArgumentException("outerLevel must be in range [1, grid.numLevels() - 1]"); - - if (nestedPositions.isEmpty()) - return Collections.emptyList(); - - final TreeMap> blocksPerShard = new TreeMap<>(); - for (T nestedPosition : nestedPositions) { - final NestedPosition outerNestedPosition; - if (nestedPosition.level() == outerLevel) - outerNestedPosition = nestedPosition; - else - outerNestedPosition = new NestedPosition(grid, nestedPosition.absolute(0), outerLevel); - - - final List blocks = blocksPerShard.computeIfAbsent(outerNestedPosition, it -> new ArrayList<>()); - blocks.add(nestedPosition); - } - return blocksPerShard.values(); - } - /** * NestedPosition wrapper for a DataBlock. Useful for grouping DataBlock by nested shard position. *