diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index ae5d6acdf..8ccddd5a8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -54,19 +54,22 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; -import java.io.InputStream; + import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.janelia.saalfeldlab.n5.Compression.CompressionType; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("bzip2") +@NameConfig.Name("bzip2") public class Bzip2Compression implements Compression { private static final long serialVersionUID = -4873117458390529118L; @CompressionParameter + @NameConfig.Parameter private final int blockSize; public Bzip2Compression(final int blockSize) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java index d66467cb8..92edc5d8b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Compression.java @@ -53,7 +53,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; import java.io.Serializable; import java.lang.annotation.ElementType; import java.lang.annotation.Inherited; @@ -61,16 +60,24 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.Codec; import org.scijava.annotations.Indexable; /** - * Compression scheme interface. + * This interface is used to indicate that a {@link BytesCodec} can be + * serialized as a "compression" for the N5 format (using the N5 API). + *

+ * N5Readers and N5Writers for the N5 format can declare BytesCodecs that + * implement this interface so that the {@link CompressionAdapter} is used for + * serialization. + *

+ * See also: an alternative method for serializing general {@link Codec}s is + * with the {@link NameConfigAdapter}. * * @author Stephan Saalfeld */ -public interface Compression extends Serializable { +public interface Compression extends Serializable, BytesCodec { /** * Annotation for runtime discovery of compression schemes. @@ -94,6 +101,7 @@ public interface Compression extends Serializable { @Target(ElementType.FIELD) @interface CompressionParameter {} + @Override default String getType() { final CompressionType compressionType = getClass().getAnnotation(CompressionType.class); @@ -102,41 +110,4 @@ default String getType() { else return compressionType.value(); } - - // -------------------------------------------------- - // - - /** - * Decode the given {@code readData}. - *

- * The returned decoded {@code ReadData} reports {@link ReadData#length() - * length()}{@code == decodedLength}. Decoding may be lazy or eager, - * depending on the {@code BytesCodec} implementation. - * - * @param readData - * data to decode - * - * @return decoded ReadData - * - * @throws N5IOException - * if any I/O error occurs - */ - ReadData decode(ReadData readData) throws N5IOException; - - /** - * Encode the given {@code readData}. - *

- * Encoding may be lazy or eager, depending on the {@code BytesCodec} - * implementation. - * - * @param readData - * data to encode - * - * @return encoded ReadData - * - * @throws N5IOException - * if any I/O error occurs - */ - ReadData encode(ReadData readData) throws N5IOException; - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index d68dd5527..181efdf7b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -6,32 +6,6 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. @@ -42,7 +16,7 @@ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS @@ -50,6 +24,7 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. + * #L% */ package org.janelia.saalfeldlab.n5; @@ -57,8 +32,11 @@ import java.util.Arrays; import java.util.HashMap; -import org.janelia.saalfeldlab.n5.codec.DataBlockCodec; -import org.janelia.saalfeldlab.n5.codec.N5Codecs; +import org.janelia.saalfeldlab.n5.codec.Codec; +import org.janelia.saalfeldlab.n5.codec.ArrayCodec; +import org.janelia.saalfeldlab.n5.codec.BytesCodec; +import org.janelia.saalfeldlab.n5.codec.DataBlockSerializer; +import org.janelia.saalfeldlab.n5.codec.N5ArrayCodec; /** * Mandatory dataset attributes: @@ -88,30 +66,39 @@ public class DatasetAttributes implements Serializable { private final long[] dimensions; private final int[] blockSize; private final DataType dataType; - private final DataBlockCodec dataBlockCodec; - private final Compression compression; + + private final ArrayCodec arrayCodec; + private final BytesCodec[] byteCodecs; + + private final DataBlockSerializer dataBlockSerializer; public DatasetAttributes( final long[] dimensions, final int[] blockSize, final DataType dataType, - final Compression compression) { + final ArrayCodec arrayCodec, + final BytesCodec... codecs) { + + this.dimensions = dimensions; + this.blockSize = blockSize; + this.dataType = dataType; - this(dimensions, blockSize, dataType, compression, N5Codecs.createDataBlockCodec(dataType, compression)); + this.arrayCodec = arrayCodec == null ? defaultArrayCodec() : arrayCodec; + byteCodecs = Arrays.stream(codecs).filter(it -> !(it instanceof RawCompression)).toArray(BytesCodec[]::new); + dataBlockSerializer = this.arrayCodec.initialize(this, byteCodecs); } - protected DatasetAttributes( + public DatasetAttributes( final long[] dimensions, final int[] blockSize, final DataType dataType, - final Compression compression, - final DataBlockCodec dataBlockCodec) { + final BytesCodec compression) { - this.dimensions = dimensions; - this.blockSize = blockSize; - this.dataType = dataType; - this.compression = compression; - this.dataBlockCodec = dataBlockCodec; + this(dimensions, blockSize, dataType, null, compression); + } + + protected ArrayCodec defaultArrayCodec() { + return new N5ArrayCodec(); } public long[] getDimensions() { @@ -131,7 +118,11 @@ public int[] getBlockSize() { public Compression getCompression() { - return compression; + return Arrays.stream(byteCodecs) + .filter(it -> it instanceof Compression) + .map(it -> (Compression)it) + .findFirst() + .orElse(new RawCompression()); } public DataType getDataType() { @@ -140,17 +131,18 @@ public DataType getDataType() { } /** - * Get the {@link DataBlockCodec} for this dataset. + * Get the {@link ArrayCodec} for this dataset. * - * @param - * the returned codec is cast to {@code DataBlockCodec} for convenience - * (that is, the caller doesn't have to do the cast explicitly). - * @return the {@code DataBlockCodec} for this dataset + * @return the {@code ArrayCodec} for this dataset */ - @SuppressWarnings("unchecked") - public DataBlockCodec getDataBlockCodec() { + public ArrayCodec getArrayCodec() { - return (DataBlockCodec) dataBlockCodec; + return arrayCodec; + } + + @SuppressWarnings("unchecked") + DataBlockSerializer getDataBlockSerializer() { + return (DataBlockSerializer) dataBlockSerializer; } public HashMap asMap() { @@ -159,7 +151,7 @@ public HashMap asMap() { map.put(DIMENSIONS_KEY, dimensions); map.put(BLOCK_SIZE_KEY, blockSize); map.put(DATA_TYPE_KEY, dataType); - map.put(COMPRESSION_KEY, compression); + map.put(COMPRESSION_KEY, getCompression()); return map; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java deleted file mode 100644 index b58828c47..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ /dev/null @@ -1,92 +0,0 @@ -/*- - * #%L - * Not HDF5 - * %% - * Copyright (C) 2017 - 2025 Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import java.io.InputStream; - -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.codec.DataBlockCodec; -import org.janelia.saalfeldlab.n5.readdata.ReadData; - -/** - * Default implementation of block reading (N5 format). - * - * @author Stephan Saalfeld - * @author Igor Pisarev - */ -public interface DefaultBlockReader { - - /** - * Reads a {@link DataBlock} from an {@link InputStream}. - * - * @param in - * the input stream - * @param datasetAttributes - * the dataset attributes - * @param gridPosition - * the grid position - * @return the block - * @throws N5IOException - * the exception - */ - static DataBlock readBlock( - final InputStream in, - final DatasetAttributes datasetAttributes, - final long[] gridPosition) throws N5IOException { - - final DataBlockCodec codec = datasetAttributes.getDataBlockCodec(); - return codec.decode(ReadData.from(in), gridPosition); - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java deleted file mode 100644 index 6cab03beb..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ /dev/null @@ -1,91 +0,0 @@ -/*- - * #%L - * Not HDF5 - * %% - * Copyright (C) 2017 - 2025 Stephan Saalfeld - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import java.io.OutputStream; - -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.codec.DataBlockCodec; - -/** - * Default implementation of block writing (N5 format). - * - * @author Stephan Saalfeld - * @author Igor Pisarev - */ -public interface DefaultBlockWriter { - - /** - * Writes a {@link DataBlock} into an {@link OutputStream}. - * - * @param the type of data - * @param out - * the output stream - * @param datasetAttributes - * the dataset attributes - * @param dataBlock - * the data block the block data type - * @throws N5IOException - * the exception - */ - static void writeBlock( - final OutputStream out, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws N5IOException { - - final DataBlockCodec codec = datasetAttributes.getDataBlockCodec(); - codec.encode(dataBlock).writeTo(out); - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index e5a503205..a742cc299 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -29,10 +29,12 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.UncheckedIOException; -import java.util.Arrays; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.ReadData; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -79,14 +81,13 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { final String groupPath = N5URI.normalizeGroupPath(pathName); final String attributesPath = absoluteAttributesPath(groupPath); - try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(attributesPath)) { - return GsonUtils.readAttributes(lockedChannel.newReader(), getGson()); + try ( final InputStream in = getKeyValueAccess().createReadData(attributesPath).inputStream() ) { + return GsonUtils.readAttributes(new InputStreamReader(in), getGson()); } catch (final N5Exception.N5NoSuchKeyException e) { return null; } catch (final IOException | UncheckedIOException | N5IOException e) { throw new N5IOException("Failed to read attributes from dataset " + pathName, e); } - } @Override @@ -97,14 +98,11 @@ default DataBlock readBlock( final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), gridPosition); - try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(path)) { - return DefaultBlockReader.readBlock(lockedChannel.newInputStream(), datasetAttributes, gridPosition); - } catch (final N5Exception.N5NoSuchKeyException e) { + try { + final ReadData blockData = getKeyValueAccess().createReadData(path); + return datasetAttributes.getDataBlockSerializer().decode(blockData, gridPosition); + } catch (N5Exception.N5NoSuchKeyException e) { return null; - } catch (final IOException | UncheckedIOException | N5IOException e) { - throw new N5IOException( - "Failed to read block " + Arrays.toString(gridPosition) + " from dataset " + path, - e); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 4c30bfe93..f7bc044a6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -246,8 +246,8 @@ default void writeBlock( final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath); final OutputStream out = lock.newOutputStream() ) { - DefaultBlockWriter.writeBlock(out, datasetAttributes, dataBlock); - } catch (final IOException | UncheckedIOException | N5IOException e) { + datasetAttributes.getDataBlockSerializer().encode(dataBlock).writeTo(out); + } catch (final IOException | UncheckedIOException e) { throw new N5IOException( "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, e); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index b7cfa15bb..e017a7c25 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -71,6 +71,7 @@ import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; import org.janelia.saalfeldlab.n5.N5Exception.N5JsonParseException; +import org.janelia.saalfeldlab.n5.codec.Codec; /** * Utility class for working with JSON. @@ -82,6 +83,7 @@ public interface GsonUtils { static Gson registerGson(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.disableHtmlEscaping(); return gsonBuilder.create(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index 2312e5f20..9c81fef76 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -6,32 +6,6 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. @@ -42,7 +16,7 @@ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS @@ -50,6 +24,7 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. + * #L% */ package org.janelia.saalfeldlab.n5; @@ -64,16 +39,24 @@ import org.janelia.saalfeldlab.n5.Compression.CompressionType; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("gzip") +@NameConfig.Name("gzip") public class GzipCompression implements Compression { private static final long serialVersionUID = 8630847239813334263L; @CompressionParameter + @NameConfig.Parameter + //TODO Caleb: How to handle serialization of parameter-less constructor. + // For N5 the default is -1. + // For zarr the range is 0-9 and is required. + // How to map -1 to some default (1?) when serializing to zarr? private final int level; @CompressionParameter + @NameConfig.Parameter(optional = true) private final boolean useZlib; private final transient GzipParameters parameters = new GzipParameters(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java index 8826ebaf4..e7bc39355 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java @@ -36,6 +36,7 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import java.io.Closeable; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -382,6 +383,8 @@ public InputStream newInputStream() throws N5IOException { } } return conn.getInputStream(); + } catch (FileNotFoundException e) { + throw new N5Exception.N5NoSuchKeyException("Could not open stream for " + uri, e); } catch (IOException e) { throw new N5IOException("Could not open stream for " + uri, e); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index 50c1880bc..1039713b5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import net.jpountz.lz4.LZ4BlockInputStream; @@ -58,13 +33,18 @@ import org.janelia.saalfeldlab.n5.Compression.CompressionType; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + +import java.io.IOException; @CompressionType("lz4") +@NameConfig.Name("lz4") public class Lz4Compression implements Compression { private static final long serialVersionUID = -9071316415067427256L; @CompressionParameter + @NameConfig.Parameter private final int blockSize; public Lz4Compression(final int blockSize) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java index 7d992fc96..1eac58a68 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java @@ -583,6 +583,8 @@ public static URI encodeAsUriPath(final String path) { * @param uri * to encode * @return the {@link URI} created from encoding the {@link String uri} + * @throws URISyntaxException + * if the provided String is not a valid URI */ public static URI encodeAsUri(final String uri) throws URISyntaxException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java new file mode 100644 index 000000000..7fd05a262 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/NameConfigAdapter.java @@ -0,0 +1,251 @@ +/*- + * #%L + * Not HDF5 + * %% + * Copyright (C) 2017 - 2025 Stephan Saalfeld + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.JsonArray; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import org.janelia.saalfeldlab.n5.serialization.N5Annotations; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; +import org.scijava.annotations.Index; +import org.scijava.annotations.IndexItem; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map.Entry; + +/** + * T adapter, auto-discovers annotated T implementations in the classpath. + * + * @author Caleb Hulbert + */ +public class NameConfigAdapter implements JsonDeserializer, JsonSerializer { + + private static HashMap, NameConfigAdapter> adapters = new HashMap<>(); + + private static void registerAdapter(Class cls) { + + adapters.put(cls, new NameConfigAdapter<>(cls)); + update(adapters.get(cls)); + } + private final HashMap> constructors = new HashMap<>(); + + private final HashMap> parameters = new HashMap<>(); + private final HashMap> parameterNames = new HashMap<>(); + private static ArrayList getDeclaredFields(Class clazz) { + + final ArrayList fields = new ArrayList<>(); + fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + for (clazz = clazz.getSuperclass(); clazz != null; clazz = clazz.getSuperclass()) + fields.addAll(Arrays.asList(clazz.getDeclaredFields())); + return fields; + } + + @SuppressWarnings("unchecked") + public static synchronized void update(final NameConfigAdapter adapter) { + + final String prefix = adapter.type.getAnnotation(NameConfig.Prefix.class).value(); + final ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + final Index annotationIndex = Index.load(NameConfig.Name.class, classLoader); + for (final IndexItem item : annotationIndex) { + Class clazz; + try { + clazz = (Class)Class.forName(item.className()); + final String name = clazz.getAnnotation(NameConfig.Name.class).value(); + final String type = prefix + "." + name; + + final Constructor constructor = clazz.getDeclaredConstructor(); + + final HashMap parameters = new HashMap<>(); + final HashMap parameterNames = new HashMap<>(); + final ArrayList fields = getDeclaredFields(clazz); + for (final Field field : fields) { + final NameConfig.Parameter parameter = field.getAnnotation(NameConfig.Parameter.class); + if (parameter != null) { + + final String parameterName; + if (parameter.value().equals("")) + parameterName = field.getName(); + else + parameterName = parameter.value(); + + parameterNames.put(field.getName(), parameterName); + + parameters.put(field.getName(), field); + } + } + + adapter.constructors.put(type, constructor); + adapter.parameters.put(type, parameters); + adapter.parameterNames.put(type, parameterNames); + } catch (final ClassNotFoundException | NoSuchMethodException | ClassCastException + | UnsatisfiedLinkError e) { + + System.err.println("T '" + item.className() + "' could not be registered"); + e.printStackTrace(System.err); + } + } + } + + private final Class type; + + public NameConfigAdapter(Class cls) { + this.type = cls; + } + + @Override + public JsonElement serialize( + final T object, + final Type typeOfSrc, + final JsonSerializationContext context) { + + final Class clazz = (Class)object.getClass(); + + final String name = clazz.getAnnotation(NameConfig.Name.class).value(); + final String prefix = type.getAnnotation(NameConfig.Prefix.class).value(); + final String type = prefix + "." + name; + + final JsonObject json = new JsonObject(); + json.addProperty("name", name); + final JsonObject configuration = new JsonObject(); + + final HashMap parameterTypes = parameters.get(type); + final HashMap parameterNameMap = parameterNames.get(type); + try { + for (final Entry parameterType : parameterTypes.entrySet()) { + final String fieldName = parameterType.getKey(); + final Field field = clazz.getDeclaredField(fieldName); + final boolean isAccessible = field.isAccessible(); + field.setAccessible(true); + final Object value = field.get(object); + field.setAccessible(isAccessible); + final JsonElement serialized = context.serialize(value); + if (field.getAnnotation(N5Annotations.ReverseArray.class) != null) { + final JsonArray reversedArray = reverseJsonArray(serialized.getAsJsonArray()); + configuration.add(parameterNameMap.get(fieldName), reversedArray); + } else + configuration.add(parameterNameMap.get(fieldName), serialized); + + } + if (!configuration.isEmpty()) + json.add("configuration", configuration); + } catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) { + new RuntimeException("Could not serialize " + clazz.getName(), e).printStackTrace(System.err); + return null; + } + + return json; + } + + @Override + public T deserialize( + final JsonElement json, + final Type typeOfT, + final JsonDeserializationContext context) throws JsonParseException { + + final String prefix = type.getAnnotation(NameConfig.Prefix.class).value(); + + final JsonObject objectJson = json.getAsJsonObject(); + final String name = objectJson.getAsJsonPrimitive("name").getAsString(); + if (name == null) { + return null; + } + + final String type = prefix + "." + name; + + final JsonObject configuration = objectJson.getAsJsonObject("configuration"); + /* It's ok to be null if all parameters are optional. + * Otherwise, return*/ + if (configuration == null) { + for (final Field field : parameters.get(type).values()) { + if (!field.getAnnotation(NameConfig.Parameter.class).optional()) + return null; + } + } + + final Constructor constructor = constructors.get(type); + constructor.setAccessible(true); + final T object; + try { + object = constructor.newInstance(); + final HashMap parameterTypes = parameters.get(type); + final HashMap parameterNameMap = parameterNames.get(type); + for (final Entry parameterType : parameterTypes.entrySet()) { + final String fieldName = parameterType.getKey(); + final String paramName = parameterNameMap.get(fieldName); + final JsonElement paramJson = configuration == null ? null : configuration.get(paramName); + final Field field = parameterType.getValue(); + if (paramJson != null) { + final Object parameter; + if (field.getAnnotation(N5Annotations.ReverseArray.class) != null) { + final JsonArray reversedArray = reverseJsonArray(paramJson); + parameter = context.deserialize(reversedArray, field.getType()); + } else + parameter = context.deserialize(paramJson, field.getType()); + ReflectionUtils.setFieldValue(object, fieldName, parameter); + } else if (!field.getAnnotation(NameConfig.Parameter.class).optional()) { + /* if param is null, and not optional, return null */ + return null; + } + } + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException + | SecurityException | NoSuchFieldException e) { + e.printStackTrace(System.err); + return null; + } + + return object; + } + + private static JsonArray reverseJsonArray(JsonElement paramJson) { + + final JsonArray reversedJson = new JsonArray(paramJson.getAsJsonArray().size()); + for (int i = paramJson.getAsJsonArray().size() - 1; i >= 0; i--) { + reversedJson.add(paramJson.getAsJsonArray().get(i)); + } + return reversedJson; + } + + public static NameConfigAdapter getJsonAdapter(Class cls) { + + if (adapters.get(cls) == null) + registerAdapter(cls); + return (NameConfigAdapter) adapters.get(cls); + } +} \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index 8fe80b900..f13452fdd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -26,31 +26,6 @@ * POSSIBILITY OF SUCH DAMAGE. * #L% */ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ package org.janelia.saalfeldlab.n5; import java.io.IOException; @@ -59,13 +34,16 @@ import org.janelia.saalfeldlab.n5.Compression.CompressionType; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; @CompressionType("xz") +@NameConfig.Name("xz") public class XzCompression implements Compression { private static final long serialVersionUID = -7272153943564743774L; @CompressionParameter + @NameConfig.Parameter private final int preset; public XzCompression(final int preset) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ArrayCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ArrayCodec.java new file mode 100644 index 000000000..c02fece1f --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ArrayCodec.java @@ -0,0 +1,15 @@ +package org.janelia.saalfeldlab.n5.codec; + +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +/** + * {@code ArrayCodec}s encode {@link DataBlock}s into {@link ReadData} and + * decode {@link ReadData} into {@link DataBlock}s. + */ +public interface ArrayCodec extends Codec { + + DataBlockSerializer initialize(final DatasetAttributes attributes, final BytesCodec... codecs); + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java new file mode 100644 index 000000000..800c0965d --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/BytesCodec.java @@ -0,0 +1,63 @@ +package org.janelia.saalfeldlab.n5.codec; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +/** + * {@code BytesCodec}s transform one {@link ReadData} into another, + * for example, compressing it. + */ +public interface BytesCodec extends Codec { + + /** + * Decode the given {@link ReadData}. + *

+ * The returned decoded {@code ReadData} reports {@link ReadData#length() + * length()}{@code == decodedLength}. Decoding may be lazy or eager, + * depending on the {@code BytesCodec} implementation. + * + * @param readData + * data to decode + * + * @return decoded ReadData + * + * @throws N5IOException + * if any I/O error occurs + */ + ReadData decode(ReadData readData) throws N5IOException; + + /** + * Encode the given {@link ReadData}. + *

+ * Encoding may be lazy or eager, depending on the {@code BytesCodec} + * implementation. + * + * @param readData + * data to encode + * + * @return encoded ReadData + * + * @throws N5IOException + * if any I/O error occurs + */ + ReadData encode(ReadData readData) throws N5IOException; + + /** + * Create a {@code BytesCodec} that sequentially applies {@code codecs} in + * the given order for encoding, and in reverse order for decoding. + * + * @param codecs + * a list of BytesCodecs + * @return the concatenated BytesCodec + */ + static BytesCodec concatenate(final BytesCodec... codecs) { + + if (codecs == null) + throw new NullPointerException(); + + if (codecs.length == 1) + return codecs[0]; + + return new ConcatenatedBytesCodec(codecs); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java new file mode 100644 index 000000000..7b40b5818 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/Codec.java @@ -0,0 +1,17 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.io.Serializable; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + +/** + * {@code Codec}s can encode and decode {@link ReadData} objects. + *

+ * Modeled after Codecs in + * Zarr. + */ +@NameConfig.Prefix("codec") +public interface Codec extends Serializable { + + String getType(); +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java new file mode 100644 index 000000000..2daad7cdb --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/ConcatenatedBytesCodec.java @@ -0,0 +1,41 @@ +package org.janelia.saalfeldlab.n5.codec; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +class ConcatenatedBytesCodec implements BytesCodec { + + private final BytesCodec[] codecs; + + ConcatenatedBytesCodec(final BytesCodec[] codecs) { + + if (codecs == null) { + throw new NullPointerException(); + } + this.codecs = codecs; + } + + @Override + public ReadData encode(ReadData readData) { + + for (BytesCodec codec : codecs) { + readData = codec.encode(readData); + } + return readData; + } + + @Override + public ReadData decode(ReadData readData) { + + for (int i = codecs.length - 1; i >= 0; i--) { + final BytesCodec codec = codecs[i]; + readData = codec.decode(readData); + } + return readData; + } + + @Override + public String getType() { + + return "internal-concatenated-codecs"; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockSerializer.java similarity index 95% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockSerializer.java index a8ef5b582..7c66b892b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataBlockSerializer.java @@ -29,7 +29,6 @@ package org.janelia.saalfeldlab.n5.codec; import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -39,7 +38,7 @@ * @param * type of the data contained in the DataBlock */ -public interface DataBlockCodec { +public interface DataBlockSerializer { ReadData encode(DataBlock dataBlock) throws N5IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/FlatArraySerializer.java similarity index 74% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/FlatArraySerializer.java index 25dfc4126..3573df913 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/FlatArraySerializer.java @@ -38,7 +38,6 @@ import java.util.Arrays; import java.util.function.IntFunction; import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.readdata.ReadData; @@ -53,7 +52,7 @@ * @param * type of the data contained in the DataBlock */ -public abstract class DataCodec { +public abstract class FlatArraySerializer { public abstract ReadData serialize(T data) throws N5IOException; @@ -70,40 +69,40 @@ public T createData(final int numElements) { // ------------------- instances -------------------- // - public static final DataCodec BYTE = new ByteDataCodec(); - public static final DataCodec SHORT_BIG_ENDIAN = new ShortDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec INT_BIG_ENDIAN = new IntDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec LONG_BIG_ENDIAN = new LongDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec FLOAT_BIG_ENDIAN = new FloatDataCodec(ByteOrder.BIG_ENDIAN); - public static final DataCodec DOUBLE_BIG_ENDIAN = new DoubleDataCodec(ByteOrder.BIG_ENDIAN); + public static final FlatArraySerializer BYTE = new ByteArraySerializer(); + public static final FlatArraySerializer SHORT_BIG_ENDIAN = new ShortArraySerializer(ByteOrder.BIG_ENDIAN); + public static final FlatArraySerializer INT_BIG_ENDIAN = new IntArraySerializer(ByteOrder.BIG_ENDIAN); + public static final FlatArraySerializer LONG_BIG_ENDIAN = new LongArraySerializer(ByteOrder.BIG_ENDIAN); + public static final FlatArraySerializer FLOAT_BIG_ENDIAN = new FloatArraySerializer(ByteOrder.BIG_ENDIAN); + public static final FlatArraySerializer DOUBLE_BIG_ENDIAN = new DoubleArraySerializer(ByteOrder.BIG_ENDIAN); - public static final DataCodec SHORT_LITTLE_ENDIAN = new ShortDataCodec(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec INT_LITTLE_ENDIAN = new IntDataCodec(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec LONG_LITTLE_ENDIAN = new LongDataCodec(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec FLOAT_LITTLE_ENDIAN = new FloatDataCodec(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec DOUBLE_LITTLE_ENDIAN = new DoubleDataCodec(ByteOrder.LITTLE_ENDIAN); + public static final FlatArraySerializer SHORT_LITTLE_ENDIAN = new ShortArraySerializer(ByteOrder.LITTLE_ENDIAN); + public static final FlatArraySerializer INT_LITTLE_ENDIAN = new IntArraySerializer(ByteOrder.LITTLE_ENDIAN); + public static final FlatArraySerializer LONG_LITTLE_ENDIAN = new LongArraySerializer(ByteOrder.LITTLE_ENDIAN); + public static final FlatArraySerializer FLOAT_LITTLE_ENDIAN = new FloatArraySerializer(ByteOrder.LITTLE_ENDIAN); + public static final FlatArraySerializer DOUBLE_LITTLE_ENDIAN = new DoubleArraySerializer(ByteOrder.LITTLE_ENDIAN); - public static final DataCodec STRING = new N5StringDataCodec(); - public static final DataCodec ZARR_STRING = new ZarrStringDataCodec(); - public static final DataCodec OBJECT = new ObjectDataCodec(); + public static final FlatArraySerializer STRING = new N5StringArraySerializer(); + public static final FlatArraySerializer ZARR_STRING = new ZarrStringArraySerializer(); + public static final FlatArraySerializer OBJECT = new ObjectArraySerializer(); - public static DataCodec SHORT(ByteOrder order) { + public static FlatArraySerializer SHORT(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? SHORT_BIG_ENDIAN : SHORT_LITTLE_ENDIAN; } - public static DataCodec INT(ByteOrder order) { + public static FlatArraySerializer INT(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? INT_BIG_ENDIAN : INT_LITTLE_ENDIAN; } - public static DataCodec LONG(ByteOrder order) { + public static FlatArraySerializer LONG(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? LONG_BIG_ENDIAN : LONG_LITTLE_ENDIAN; } - public static DataCodec FLOAT(ByteOrder order) { + public static FlatArraySerializer FLOAT(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? FLOAT_BIG_ENDIAN : FLOAT_LITTLE_ENDIAN; } - public static DataCodec DOUBLE(ByteOrder order) { + public static FlatArraySerializer DOUBLE(ByteOrder order) { return order == ByteOrder.BIG_ENDIAN ? DOUBLE_BIG_ENDIAN : DOUBLE_LITTLE_ENDIAN; } @@ -113,14 +112,14 @@ public static DataCodec DOUBLE(ByteOrder order) { private final int bytesPerElement; private final IntFunction dataFactory; - private DataCodec(int bytesPerElement, IntFunction dataFactory) { + private FlatArraySerializer(int bytesPerElement, IntFunction dataFactory) { this.bytesPerElement = bytesPerElement; this.dataFactory = dataFactory; } - private static final class ByteDataCodec extends DataCodec { + private static final class ByteArraySerializer extends FlatArraySerializer { - private ByteDataCodec() { + private ByteArraySerializer() { super(Byte.BYTES, byte[]::new); } @@ -141,11 +140,11 @@ public byte[] deserialize(final ReadData readData, int numElements) throws N5IOE } } - private static final class ShortDataCodec extends DataCodec { + private static final class ShortArraySerializer extends FlatArraySerializer { private final ByteOrder order; - ShortDataCodec(ByteOrder order) { + ShortArraySerializer(ByteOrder order) { super(Short.BYTES, short[]::new); this.order = order; } @@ -165,11 +164,11 @@ public short[] deserialize(final ReadData readData, int numElements) throws N5IO } } - private static final class IntDataCodec extends DataCodec { + private static final class IntArraySerializer extends FlatArraySerializer { private final ByteOrder order; - IntDataCodec(ByteOrder order) { + IntArraySerializer(ByteOrder order) { super(Integer.BYTES, int[]::new); this.order = order; } @@ -191,11 +190,11 @@ public int[] deserialize(final ReadData readData, int numElements) throws N5IOEx } } - private static final class LongDataCodec extends DataCodec { + private static final class LongArraySerializer extends FlatArraySerializer { private final ByteOrder order; - LongDataCodec(ByteOrder order) { + LongArraySerializer(ByteOrder order) { super(Long.BYTES, long[]::new); this.order = order; } @@ -215,11 +214,11 @@ public long[] deserialize(final ReadData readData, int numElements) throws N5IOE } } - private static final class FloatDataCodec extends DataCodec { + private static final class FloatArraySerializer extends FlatArraySerializer { private final ByteOrder order; - FloatDataCodec(ByteOrder order) { + FloatArraySerializer(ByteOrder order) { super(Float.BYTES, float[]::new); this.order = order; } @@ -239,11 +238,11 @@ public float[] deserialize(final ReadData readData, int numElements) throws N5IO } } - private static final class DoubleDataCodec extends DataCodec { + private static final class DoubleArraySerializer extends FlatArraySerializer { private final ByteOrder order; - DoubleDataCodec(ByteOrder order) { + DoubleArraySerializer(ByteOrder order) { super(Double.BYTES, double[]::new); this.order = order; } @@ -263,12 +262,12 @@ public double[] deserialize(final ReadData readData, int numElements) throws N5I } } - private static final class N5StringDataCodec extends DataCodec { + private static final class N5StringArraySerializer extends FlatArraySerializer { private static final Charset ENCODING = StandardCharsets.UTF_8; private static final String NULLCHAR = "\0"; - N5StringDataCodec() { + N5StringArraySerializer() { super( -1, String[]::new); } @@ -286,11 +285,11 @@ public String[] deserialize(ReadData readData, int numElements) throws N5IOExcep } } - private static final class ZarrStringDataCodec extends DataCodec { + private static final class ZarrStringArraySerializer extends FlatArraySerializer { private static final Charset ENCODING = StandardCharsets.UTF_8; - ZarrStringDataCodec() { + ZarrStringArraySerializer() { super( -1, String[]::new); } @@ -334,9 +333,9 @@ public String[] deserialize(ReadData readData, int numElements) throws N5IOExcep } } - private static final class ObjectDataCodec extends DataCodec { + private static final class ObjectArraySerializer extends FlatArraySerializer { - ObjectDataCodec() { + ObjectArraySerializer() { super(-1, byte[]::new); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5ArrayCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5ArrayCodec.java new file mode 100644 index 000000000..7d8d5209a --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5ArrayCodec.java @@ -0,0 +1,24 @@ +package org.janelia.saalfeldlab.n5.codec; + +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + +@NameConfig.Name(value = N5ArrayCodec.TYPE) +public class N5ArrayCodec implements ArrayCodec { + + private static final long serialVersionUID = 3523505403978222360L; + + public static final String TYPE = "n5bytes"; + + @Override + public String getType() { + + return TYPE; + } + + @Override + public DataBlockSerializer initialize(final DatasetAttributes attributes, final BytesCodec... bytesCodecs) { + return N5DataBlockSerializers.create(attributes.getDataType(), BytesCodec.concatenate(bytesCodecs)); + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5DataBlockSerializers.java similarity index 67% rename from src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java rename to src/main/java/org/janelia/saalfeldlab/n5/codec/N5DataBlockSerializers.java index 27a5d97d3..95117238c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/N5Codecs.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/N5DataBlockSerializers.java @@ -35,7 +35,6 @@ import java.io.OutputStream; import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; -import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataBlock.DataBlockFactory; import org.janelia.saalfeldlab.n5.DataType; @@ -49,91 +48,86 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import static org.janelia.saalfeldlab.n5.N5Exception.*; -import static org.janelia.saalfeldlab.n5.codec.N5Codecs.BlockHeader.MODE_DEFAULT; -import static org.janelia.saalfeldlab.n5.codec.N5Codecs.BlockHeader.MODE_OBJECT; -import static org.janelia.saalfeldlab.n5.codec.N5Codecs.BlockHeader.MODE_VARLENGTH; +import static org.janelia.saalfeldlab.n5.codec.N5DataBlockSerializers.BlockHeader.MODE_DEFAULT; +import static org.janelia.saalfeldlab.n5.codec.N5DataBlockSerializers.BlockHeader.MODE_OBJECT; +import static org.janelia.saalfeldlab.n5.codec.N5DataBlockSerializers.BlockHeader.MODE_VARLENGTH; -public class N5Codecs { +public class N5DataBlockSerializers { - private static final DataBlockCodecFactory BYTE = c -> new DefaultDataBlockCodec<>(DataCodec.BYTE, ByteArrayDataBlock::new, c); - private static final DataBlockCodecFactory SHORT = c -> new DefaultDataBlockCodec<>(DataCodec.SHORT_BIG_ENDIAN, ShortArrayDataBlock::new, c); - private static final DataBlockCodecFactory INT = c -> new DefaultDataBlockCodec<>(DataCodec.INT_BIG_ENDIAN, IntArrayDataBlock::new, c); - private static final DataBlockCodecFactory LONG = c -> new DefaultDataBlockCodec<>(DataCodec.LONG_BIG_ENDIAN, LongArrayDataBlock::new, c); - private static final DataBlockCodecFactory FLOAT = c -> new DefaultDataBlockCodec<>(DataCodec.FLOAT_BIG_ENDIAN, FloatArrayDataBlock::new, c); - private static final DataBlockCodecFactory DOUBLE = c -> new DefaultDataBlockCodec<>(DataCodec.DOUBLE_BIG_ENDIAN, DoubleArrayDataBlock::new, c); - private static final DataBlockCodecFactory STRING = c -> new StringDataBlockCodec(c); - private static final DataBlockCodecFactory OBJECT = c -> new ObjectDataBlockCodec(c); + private static final DataBlockSerializerFactory BYTE = c -> new DefaultDataBlockSerializer<>(FlatArraySerializer.BYTE, ByteArrayDataBlock::new, c); + private static final DataBlockSerializerFactory SHORT = c -> new DefaultDataBlockSerializer<>(FlatArraySerializer.SHORT_BIG_ENDIAN, ShortArrayDataBlock::new, c); + private static final DataBlockSerializerFactory INT = c -> new DefaultDataBlockSerializer<>(FlatArraySerializer.INT_BIG_ENDIAN, IntArrayDataBlock::new, c); + private static final DataBlockSerializerFactory LONG = c -> new DefaultDataBlockSerializer<>(FlatArraySerializer.LONG_BIG_ENDIAN, LongArrayDataBlock::new, c); + private static final DataBlockSerializerFactory FLOAT = c -> new DefaultDataBlockSerializer<>(FlatArraySerializer.FLOAT_BIG_ENDIAN, FloatArrayDataBlock::new, c); + private static final DataBlockSerializerFactory DOUBLE = c -> new DefaultDataBlockSerializer<>(FlatArraySerializer.DOUBLE_BIG_ENDIAN, DoubleArrayDataBlock::new, c); + private static final DataBlockSerializerFactory STRING = c -> new StringDataBlockSerializer(c); + private static final DataBlockSerializerFactory OBJECT = c -> new ObjectDataBlockSerializer(c); - private N5Codecs() {} + private N5DataBlockSerializers() {} - public static DataBlockCodec createDataBlockCodec( + public static DataBlockSerializer create( final DataType dataType, - final Compression compression) { + final BytesCodec codec) { - final DataBlockCodecFactory factory; + final DataBlockSerializerFactory factory; switch (dataType) { case UINT8: case INT8: - factory = N5Codecs.BYTE; + factory = N5DataBlockSerializers.BYTE; break; case UINT16: case INT16: - factory = N5Codecs.SHORT; + factory = N5DataBlockSerializers.SHORT; break; case UINT32: case INT32: - factory = N5Codecs.INT; + factory = N5DataBlockSerializers.INT; break; case UINT64: case INT64: - factory = N5Codecs.LONG; + factory = N5DataBlockSerializers.LONG; break; case FLOAT32: - factory = N5Codecs.FLOAT; + factory = N5DataBlockSerializers.FLOAT; break; case FLOAT64: - factory = N5Codecs.DOUBLE; + factory = N5DataBlockSerializers.DOUBLE; break; case STRING: - factory = N5Codecs.STRING; + factory = N5DataBlockSerializers.STRING; break; case OBJECT: - factory = N5Codecs.OBJECT; + factory = N5DataBlockSerializers.OBJECT; break; default: throw new IllegalArgumentException("Unsupported data type: " + dataType); } @SuppressWarnings("unchecked") - final DataBlockCodecFactory tFactory = (DataBlockCodecFactory) factory; - return tFactory.createDataBlockCodec(compression); + final DataBlockSerializerFactory tFactory = (DataBlockSerializerFactory)factory; + return tFactory.create(codec); } - private interface DataBlockCodecFactory { + private interface DataBlockSerializerFactory { /** - * Get a {@link DataBlockCodec}, with the specified {@code compression}, for de/serializing DataBlocks to N5 format, + * Create a {@link DataBlockSerializer} that uses the specified {@code + * BytesCodec} and de/serializes {@code DataBlock} to N5 format. * - * @return N5 {@code DataBlockCodec} for the specified {@code compression} + * @return N5 {@code DataBlockSerializer} for the specified {@code codec} */ - DataBlockCodec createDataBlockCodec(Compression compression); + DataBlockSerializer create(BytesCodec codec); } - private abstract static class AbstractDataBlockCodec implements DataBlockCodec { + abstract static class N5AbstractDataBlockSerializer implements DataBlockSerializer { - private static final int VAR_OBJ_BYTES_PER_ELEMENT = 1; - - private final DataCodec dataCodec; + private final FlatArraySerializer dataCodec; private final DataBlockFactory dataBlockFactory; - private final Compression compression; + private final BytesCodec codec; - AbstractDataBlockCodec( - final DataCodec dataCodec, - final DataBlockFactory dataBlockFactory, - final Compression compression - ) { + N5AbstractDataBlockSerializer(FlatArraySerializer dataCodec, DataBlockFactory dataBlockFactory, BytesCodec codec) { this.dataCodec = dataCodec; this.dataBlockFactory = dataBlockFactory; - this.compression = compression; + this.codec = codec; } abstract BlockHeader createBlockHeader(final DataBlock dataBlock, ReadData blockData) throws N5IOException; @@ -142,10 +136,10 @@ private abstract static class AbstractDataBlockCodec implements DataBlockCode public ReadData encode(DataBlock dataBlock) throws N5IOException { return ReadData.from(out -> { final ReadData dataReadData = dataCodec.serialize(dataBlock.getData()); - final ReadData encodedData = compression.encode(dataReadData); final BlockHeader header = createBlockHeader(dataBlock, dataReadData); header.writeTo(out); + final ReadData encodedData = codec.encode(dataReadData); encodedData.writeTo(out); }); } @@ -158,13 +152,10 @@ public DataBlock decode(final ReadData readData, final long[] gridPosition) t try(final InputStream in = readData.inputStream()) { final BlockHeader header = decodeBlockHeader(in); - final int bytesPerElement - = dataCodec.bytesPerElement() == -1 - ? VAR_OBJ_BYTES_PER_ELEMENT - : dataCodec.bytesPerElement(); - final int numElements = header.numElements(); - final ReadData decodeData = compression.decode(ReadData.from(in)); + final ReadData decodeData = codec.decode(ReadData.from(in)); + + // the dataCodec knows the number of bytes per element final T data = dataCodec.deserialize(decodeData, numElements); return dataBlockFactory.createDataBlock(header.blockSize(), gridPosition, data); } catch (IOException e) { @@ -173,21 +164,22 @@ public DataBlock decode(final ReadData readData, final long[] gridPosition) t } } + /** * DataBlockCodec for all N5 data types, except STRING and OBJECT */ - private static class DefaultDataBlockCodec extends AbstractDataBlockCodec { + private static class DefaultDataBlockSerializer extends N5AbstractDataBlockSerializer { - DefaultDataBlockCodec( - final DataCodec dataCodec, + DefaultDataBlockSerializer( + final FlatArraySerializer dataCodec, final DataBlockFactory dataBlockFactory, - final Compression compression) { + final BytesCodec codec) { - super(dataCodec, dataBlockFactory, compression); + super(dataCodec, dataBlockFactory, codec); } @Override - protected BlockHeader createBlockHeader(final DataBlock dataBlock, ReadData blockData) { + protected BlockHeader createBlockHeader(final DataBlock dataBlock, ReadData blockData) throws N5IOException { return new BlockHeader(dataBlock.getSize(), dataBlock.getNumElements()); } @@ -202,17 +194,17 @@ protected BlockHeader decodeBlockHeader(final InputStream in) throws N5IOExcepti /** * DataBlockCodec for N5 data type STRING */ - private static class StringDataBlockCodec extends AbstractDataBlockCodec { + private static class StringDataBlockSerializer extends N5AbstractDataBlockSerializer { - StringDataBlockCodec(final Compression compression) { + StringDataBlockSerializer(final BytesCodec codec) { - super(DataCodec.STRING, StringDataBlock::new, compression); + super(FlatArraySerializer.STRING, StringDataBlock::new, codec); } @Override protected BlockHeader createBlockHeader(final DataBlock dataBlock, ReadData blockData) throws N5IOException { - return new BlockHeader(dataBlock.getSize(), (int)blockData.length()); + return new BlockHeader(MODE_VARLENGTH, dataBlock.getSize(), (int)blockData.length()); } @Override @@ -225,11 +217,11 @@ protected BlockHeader decodeBlockHeader(final InputStream in) throws N5IOExcepti /** * DataBlockCodec for N5 data type OBJECT */ - private static class ObjectDataBlockCodec extends AbstractDataBlockCodec { + private static class ObjectDataBlockSerializer extends N5AbstractDataBlockSerializer { - ObjectDataBlockCodec(final Compression compression) { + ObjectDataBlockSerializer(final BytesCodec codec) { - super(DataCodec.OBJECT, ByteArrayDataBlock::new, compression); + super(FlatArraySerializer.OBJECT, ByteArrayDataBlock::new, codec); } @Override @@ -275,6 +267,20 @@ static class BlockHeader { this.numElements = numElements; } + public int getSize() { + + switch (mode) { + case MODE_DEFAULT: + return 2 + 4 * blockSize.length; + case MODE_VARLENGTH: + return 2 + 4 * blockSize.length + 4; + case MODE_OBJECT: + return 2 + 4; + default: + throw new IllegalArgumentException("Unexpected mode: " + mode); + } + } + public int[] blockSize() { return blockSize; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java new file mode 100644 index 000000000..8bf7662c1 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java @@ -0,0 +1,58 @@ +package org.janelia.saalfeldlab.n5.codec; + +import java.nio.ByteOrder; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.serialization.NameConfig; + + +@NameConfig.Name(value = RawBytesArrayCodec.TYPE) +public class RawBytesArrayCodec implements ArrayCodec { + + private static final long serialVersionUID = 3282569607795127005L; + + public static final String TYPE = "bytes"; + + @NameConfig.Parameter(value = "endian", optional = true) + protected final ByteOrder byteOrder; + + public RawBytesArrayCodec() { + + this(ByteOrder.BIG_ENDIAN); + } + + public RawBytesArrayCodec(final ByteOrder byteOrder) { + + this.byteOrder = byteOrder; + } + + @Override + public String getType() { + + return TYPE; + } + + public ByteOrder getByteOrder() { + return byteOrder; + } + + @Override + public DataBlockSerializer initialize(final DatasetAttributes attributes, final BytesCodec... bytesCodecs) { + ensureValidByteOrder(attributes.getDataType(), getByteOrder()); + return N5DataBlockSerializers.create(attributes.getDataType(), BytesCodec.concatenate(bytesCodecs)); + } + + private static void ensureValidByteOrder(final DataType dataType, final ByteOrder byteOrder) { + + switch (dataType) { + case INT8: + case UINT8: + case STRING: + case OBJECT: + return; + } + + if (byteOrder == null) + throw new IllegalArgumentException("DataType (" + dataType + ") requires ByteOrder, but was null"); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawDataBlockSerializers.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawDataBlockSerializers.java new file mode 100644 index 000000000..609b2e583 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawDataBlockSerializers.java @@ -0,0 +1,119 @@ +package org.janelia.saalfeldlab.n5.codec; + +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock; +import org.janelia.saalfeldlab.n5.FloatArrayDataBlock; +import org.janelia.saalfeldlab.n5.IntArrayDataBlock; +import org.janelia.saalfeldlab.n5.LongArrayDataBlock; +import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; +import org.janelia.saalfeldlab.n5.StringDataBlock; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +import java.nio.ByteOrder; + +public class RawDataBlockSerializers { + + private static final DataBlockSerializerFactory BYTE = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.BYTE, ByteArrayDataBlock::new, blockSize, codec); + private static final DataBlockSerializerFactory SHORT = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.SHORT(byteOrder), ShortArrayDataBlock::new, blockSize, codec); + private static final DataBlockSerializerFactory INT = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.INT(byteOrder), IntArrayDataBlock::new, blockSize, codec); + private static final DataBlockSerializerFactory LONG = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.LONG(byteOrder), LongArrayDataBlock::new, blockSize, codec); + private static final DataBlockSerializerFactory FLOAT = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.FLOAT(byteOrder), FloatArrayDataBlock::new, blockSize, codec); + private static final DataBlockSerializerFactory DOUBLE = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.DOUBLE(byteOrder), DoubleArrayDataBlock::new, blockSize, codec); + private static final DataBlockSerializerFactory STRING = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.ZARR_STRING, StringDataBlock::new, blockSize, codec); + private static final DataBlockSerializerFactory OBJECT = (byteOrder, blockSize, codec) -> new RawDataBlockSerializer<>(FlatArraySerializer.OBJECT, ByteArrayDataBlock::new, blockSize, codec); + + private RawDataBlockSerializers() {} + + public static DataBlockSerializer create( + final DataType dataType, + final ByteOrder byteOrder, + final int[] blockSize, + final BytesCodec codec) { + final DataBlockSerializerFactory factory; + switch (dataType) { + case UINT8: + case INT8: + factory = RawDataBlockSerializers.BYTE; + break; + case UINT16: + case INT16: + factory = RawDataBlockSerializers.SHORT; + break; + case UINT32: + case INT32: + factory = RawDataBlockSerializers.INT; + break; + case UINT64: + case INT64: + factory = RawDataBlockSerializers.LONG; + break; + case FLOAT32: + factory = RawDataBlockSerializers.FLOAT; + break; + case FLOAT64: + factory = RawDataBlockSerializers.DOUBLE; + break; + case STRING: + factory = RawDataBlockSerializers.STRING; + break; + // TODO: What about OBJECT? + default: + throw new IllegalArgumentException("Unsupported data type: " + dataType); + } + final DataBlockSerializerFactory tFactory = (DataBlockSerializerFactory) factory; + return tFactory.create(byteOrder, blockSize, codec); + } + + private interface DataBlockSerializerFactory { + + /** + * Create a {@link DataBlockSerializer} that uses the specified {@code + * ByteOrder} and {@code BytesCodes} and de/serializes {@code + * DataBlock} of the specified {@code blockSize} to raw format. + * + * @return Raw {@code DataBlockSerializer} + */ + DataBlockSerializer create(ByteOrder byteOrder, int[] blockSize, BytesCodec codec); + } + + private static class RawDataBlockSerializer implements DataBlockSerializer { + + private final FlatArraySerializer dataCodec; + private final DataBlock.DataBlockFactory dataBlockFactory; + private final int[] blockSize; + private final int numElements; + private final BytesCodec codec; + + RawDataBlockSerializer( + final FlatArraySerializer dataCodec, + final DataBlock.DataBlockFactory dataBlockFactory, + final int[] blockSize, + final BytesCodec codec) { + + this.dataCodec = dataCodec; + this.dataBlockFactory = dataBlockFactory; + this.blockSize = blockSize; + this.numElements = DataBlock.getNumElements(blockSize); + this.codec = codec; + } + + @Override + public ReadData encode(DataBlock dataBlock) { + + return ReadData.from(out -> { + final ReadData blockData = dataCodec.serialize(dataBlock.getData()); + codec.encode(blockData).writeTo(out); + }); + } + + @Override + public DataBlock decode(ReadData readData, long[] gridPosition) { + + final ReadData decodeData = codec.decode(readData); + final T data = dataCodec.deserialize(decodeData, numElements); + return dataBlockFactory.createDataBlock(blockSize, gridPosition, data); + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java index 724e71cdc..a298897a1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/ReadData.java @@ -32,8 +32,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; - -import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; /** @@ -70,7 +68,7 @@ default long length() throws N5IOException { /** * Returns a {@link ReadData} whose length is limited to the given value. - * + * * @param length * the length of the resulting ReadData * @return a length-limited ReadData @@ -160,7 +158,7 @@ default ByteBuffer toByteBuffer() throws N5IOException, IllegalStateException { *

* The returned {@code ReadData} has a known {@link #length} and multiple * {@link #inputStream InputStreams} can be opened on it. - * + * * @return * a materialized ReadData. * @throws N5IOException diff --git a/src/main/java/org/janelia/saalfeldlab/n5/serialization/JsonArrayUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/serialization/JsonArrayUtils.java new file mode 100644 index 000000000..3d6711e2a --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/serialization/JsonArrayUtils.java @@ -0,0 +1,26 @@ +package org.janelia.saalfeldlab.n5.serialization; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; + +public class JsonArrayUtils { + + /** + * Reverses the order of elements in a JSON array in-place. + * + * @param array the JSON array to reverse; must not be null + * @see N5Annotations.ReverseArray + */ + public static void reverse(final JsonArray array) { + + JsonElement a; + final int max = array.size() - 1; + for (int i = (max - 1) / 2; i >= 0; --i) { + final int j = max - i; + a = array.get(i); + array.set(i, array.get(j)); + array.set(j, a); + } + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/serialization/N5Annotations.java b/src/main/java/org/janelia/saalfeldlab/n5/serialization/N5Annotations.java new file mode 100644 index 000000000..54e199816 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/serialization/N5Annotations.java @@ -0,0 +1,37 @@ +package org.janelia.saalfeldlab.n5.serialization; + +import java.io.Serializable; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Provides specialized annotations for N5 serialization behaviors. + *

+ * This interface defines annotations that control specific serialization + * transformations needed for N5 compatibility across different storage + * formats, for example, when dealing with dimension ordering conventions. + * + * @see ReverseArray + */ +public interface N5Annotations extends Serializable { + + /** + * Indicates that an array field should be reversed during serialization/deserialization. + *

+ * This annotation is used to handle dimension ordering differences between storage formats. + * For example, Zarr uses C-order (row-major) dimension ordering [Z, Y, X], while N5 uses + * F-order (column-major) dimension ordering [X, Y, Z]. + *

+ * This ensures that dimension-related arrays maintain the correct semantic meaning + * across different storage format conventions. + */ + @Inherited + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.FIELD) + @interface ReverseArray { + } +} + diff --git a/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java b/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java new file mode 100644 index 000000000..f19776906 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/serialization/NameConfig.java @@ -0,0 +1,113 @@ +package org.janelia.saalfeldlab.n5.serialization; + +import org.scijava.annotations.Indexable; + +import java.io.Serializable; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Configuration interface for N5 serialization naming and parameter annotations. + *

+ * This interface provides a standardized way to configure serialization names and parameters + * for N5 components such as compression algorithms and codecs. It defines annotations that + * control how classes and their fields are serialized and deserialized for the N5 API. + *

+ * Classes implementing this interface can use the provided annotations to: + *

+ * + * @see Name + * @see Prefix + * @see Parameter + */ +public interface NameConfig extends Serializable { + + /** + * Defines a namespace prefix for serialization. + *

+ * This annotation specifies a prefix that is prepended to the serialization + * type name, creating a namespaced identifier. This is useful for organizing + * related components into logical groups, usually all the components implementing + * a particular interface. + * + * @see Name + */ + @Retention(RetentionPolicy.RUNTIME) + @Inherited + @Target(ElementType.TYPE) + @interface Prefix { + String value(); + } + + /** + * Specifies the serialization type name for a class. + *

+ * This annotation defines the string identifier used during serialization and + * deserialization to identify the type. The name should be unique within its + * namespace and is typically a short, descriptive identifier. + * + * @see Prefix + */ + @Retention(RetentionPolicy.RUNTIME) + @Inherited + @Target(ElementType.TYPE) + @Indexable + @interface Name { + String value(); + } + + /** + * Marks a field as a parameter to be serialized. + *

+ * This annotation identifies fields that should be included during serialization + * and deserialization. It supports both required and optional parameters. + *

+ * The {@code value} attribute can be used to specify an alternative name for + * the parameter during serialization. If not specified, the field name is used. + * + * @see Name + */ + @Retention(RetentionPolicy.RUNTIME) + @Inherited + @Target(ElementType.FIELD) + @interface Parameter { + /** + * Alternative name for the parameter during serialization. + * If empty, the field name is used. + * + * @return the parameter name, or empty string for field name + */ + String value() default ""; + + /** + * Whether this parameter is optional. + * Optional parameters may be omitted during deserialization. + * + * @return {@code true} if the parameter is optional, {@code false} otherwise + */ + boolean optional() default false; + } + + /** + * Returns the serialization type name for this instance. + *

+ * This method retrieves the value from the {@link Name @Name} annotation + * if present on the class. + * + * @return the type name from the {@code @Name} annotation, or {@code null} if not annotated + */ + default String getType() { + + final Name type = getClass().getAnnotation(Name.class); + return type == null ? null : type.value(); + + } +} + diff --git a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java index 4e9417eeb..ed6259f2d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java @@ -29,7 +29,16 @@ package org.janelia.saalfeldlab.n5.cache; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +import org.janelia.saalfeldlab.n5.N5Exception; import org.junit.Test; import com.google.gson.JsonElement; @@ -126,6 +135,165 @@ public void cacheBackingTest() { } + @Test + public void testCopyOnReadPreventsExternalModification() { + + final DummyBackingStorage backingStorage = new DummyBackingStorage(); + final N5JsonCache cache = new N5JsonCache(backingStorage); + + // Get attributes and modify the returned object + JsonElement attrs1 = cache.getAttributes("path", "key"); + attrs1.getAsJsonObject().addProperty("modified", "value"); + + // Get attributes again - should not contain the modification + JsonElement attrs2 = cache.getAttributes("path", "key"); + assertFalse(attrs2.getAsJsonObject().has("modified")); + + // Verify both calls return different instances + assertNotSame(attrs1, attrs2); + } + + @Test + public void testCacheManipulationMethods() { + + final DummyBackingStorage backingStorage = new DummyBackingStorage(); + final N5JsonCache cache = new N5JsonCache(backingStorage); + + // First, ensure the path exists in cache + assertTrue(cache.exists("path", null)); + + // Test setAttributes + JsonObject newAttrs = new JsonObject(); + newAttrs.addProperty("custom", "value"); + cache.setAttributes("path", "key", newAttrs); + JsonElement retrievedAttrs = cache.getAttributes("path", "key"); + assertTrue(retrievedAttrs.getAsJsonObject().has("custom")); + assertEquals("value", retrievedAttrs.getAsJsonObject().get("custom").getAsString()); + + // Test updateCacheInfo + JsonObject updatedAttrs = new JsonObject(); + updatedAttrs.addProperty("updated", "updated-value"); + cache.updateCacheInfo("path", "key2", updatedAttrs); + JsonElement retrievedUpdated = cache.getAttributes("path", "key2"); + assertTrue(retrievedUpdated.getAsJsonObject().has("updated")); + assertEquals("updated-value", retrievedUpdated.getAsJsonObject().get("updated").getAsString()); + + // Test initializeNonemptyCache + cache.initializeNonemptyCache("newPath", "newKey"); + assertTrue(cache.exists("newPath", null)); + } + + @Test + public void testChildManagement() { + + final DummyBackingStorage backingStorage = new DummyBackingStorage(); + final N5JsonCache cache = new N5JsonCache( backingStorage ); + + // Initialize parent and children + cache.exists("parent", null); + cache.list("parent"); + + // Test addChild + cache.addChild( "parent", "child1" ); + String[] children = cache.list( "parent" ); + assertTrue( Arrays.asList( children ).contains( "child1" ) ); + + // Note: addChildIfPresent doesn't check or create the parent, + // it only adds to existing cache entries + + // Test addChildIfPresent on non-cached parent + // This should not throw and should not create the parent + cache.addChildIfPresent("nonexistent", "child"); + children = cache.list("nonexistent"); + assertFalse(Arrays.asList(children).contains("child")); + + // Test addChildIfPresent on cached parent without children list + cache.exists("parent2", null); + children = cache.list("parent2"); // initialize children array + cache.addChildIfPresent("parent2", "child"); + children = cache.list("parent2"); + assertTrue(Arrays.asList(children).contains("child")); + } + + @Test + public void testRemoveCacheHierarchy() { + final DummyBackingStorage backingStorage = new DummyBackingStorage(); + final N5JsonCache cache = new N5JsonCache(backingStorage); + + // Setup hierarchy + cache.exists("root", null); + cache.exists("root/child1", null); + cache.exists("root/child1/grandchild", null); + cache.exists("root/child2", null); + + // Add children relationships + cache.list("root"); + cache.addChild("root", "child1"); + cache.addChild("root", "child2"); + + // Remove child1 and its descendants + cache.removeCache("root", "root/child1"); + + // Verify removal - paths should not exist anymore + assertFalse(cache.exists("root/child1", null)); + assertFalse(cache.exists("root/child1/grandchild", null)); + + // Verify parent's children list updated + String[] remaining = cache.list("root"); + assertFalse(Arrays.asList(remaining).contains("child1")); + assertTrue(Arrays.asList(remaining).contains("child2")); + + // Verify child2 unaffected + assertTrue(cache.exists("root/child2", null)); + } + + @Test(expected = N5Exception.N5IOException.class) + public void testListNonExistentGroupThrows() { + + final DummyNonExistentBackingStorage backingStorage = new DummyNonExistentBackingStorage(); + final N5JsonCache cache = new N5JsonCache(backingStorage); + cache.list("nonexistent"); + } + + @Test + public void testEmptyCacheInfoBehavior() { + final DummyNonExistentBackingStorage backingStorage = new DummyNonExistentBackingStorage(); + final N5JsonCache cache = new N5JsonCache(backingStorage); + + // Non-existent path should return emptyCacheInfo + assertFalse(cache.exists("nonexistent", null)); + assertFalse(cache.isGroup("nonexistent", null)); + assertFalse(cache.isDataset("nonexistent", null)); + assertNull(cache.getAttributes("nonexistent", "key")); + } + + @Test(expected = N5Exception.class) + public void testEmptyJsonDeepCopyThrows() { + N5JsonCache.emptyJson.deepCopy(); + } + + @Test + public void testCacheStateTransitions() { + final DummyBackingStorage backingStorage = new DummyBackingStorage(); + final N5JsonCache cache = new N5JsonCache(backingStorage); + + // Start with emptyCacheInfo + cache.addNewCacheInfo("path", null, null); + + // Transition to a nonempty cache + cache.initializeNonemptyCache("path", "key"); + assertTrue(cache.exists("path", null)); + + // Update existing cache + JsonObject attrs = new JsonObject(); + attrs.addProperty("version", "1"); + cache.setAttributes("path", "key", attrs); + + attrs.addProperty("version", "2"); + cache.updateCacheInfo("path", "key", attrs); + assertEquals("2", cache.getAttributes("path", "key").getAsJsonObject().get("version").getAsString()); + } + protected static class DummyBackingStorage implements N5JsonCacheableContainer { int attrCallCount = 0; @@ -139,7 +307,7 @@ protected static class DummyBackingStorage implements N5JsonCacheableContainer { public DummyBackingStorage() { } - public JsonElement getAttributesFromContainer(final String key, final String cacheKey) { + public JsonElement getAttributesFromContainer(final String path, final String cacheKey) { attrCallCount++; final JsonObject obj = new JsonObject(); obj.addProperty("key", "value"); @@ -151,17 +319,17 @@ public boolean existsFromContainer(final String path, final String cacheKey) { return true; } - public boolean isGroupFromContainer(final String key) { + public boolean isGroupFromContainer(final String path) { isGroupCallCount++; return true; } - public boolean isDatasetFromContainer(final String key) { + public boolean isDatasetFromContainer(final String path) { isDatasetCallCount++; return true; } - public String[] listFromContainer(final String key) { + public String[] listFromContainer(final String path) { listCallCount++; return new String[] { "list" }; } @@ -179,4 +347,20 @@ public boolean isDatasetFromAttributes(final String cacheKey, final JsonElement } } + // Helper class for non-existent paths + protected static class DummyNonExistentBackingStorage extends DummyBackingStorage { + + @Override + public JsonElement getAttributesFromContainer(String key, String cacheKey) { + attrCallCount++; + return null; + } + + @Override + public boolean existsFromContainer(String path, String cacheKey) { + existsCallCount++; + return false; + } + } + }