Shard support

pom.xml

@@ -5,7 +5,7 @@
 	<parent>
 		<groupId>org.scijava</groupId>
 		<artifactId>pom-scijava</artifactId>
-		<version>40.0.0</version>
+		<version>43.0.0</version>
 		<relativePath />
 	</parent>
 
@@ -161,13 +161,36 @@
 			<groupId>com.google.code.gson</groupId>
 			<artifactId>gson</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>org.scijava</groupId>
+			<artifactId>scijava-common</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-compress</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>commons-io</groupId>
+			<artifactId>commons-io</artifactId>
+		</dependency>
 
 		<!-- Test dependencies -->
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.janelia.saalfeldlab</groupId>
+			<artifactId>n5-universe</artifactId>
+			<exclusions>
+				<exclusion>
+					<groupId>org.janelia.saalfeldlab</groupId>
+					<artifactId>n5</artifactId>
+				</exclusion>
+			</exclusions>
+			<scope>test</scope>
+		</dependency>
 		<dependency>
 			<groupId>net.imagej</groupId>
 			<artifactId>ij</artifactId>
@@ -194,13 +217,16 @@
 			<version>${commons-collections4.version}</version>
 			<scope>test</scope>
 		</dependency>
+		<!-- JMH -->
 		<dependency>
-			<groupId>org.scijava</groupId>
-			<artifactId>scijava-common</artifactId>
+			<groupId>org.openjdk.jmh</groupId>
+			<artifactId>jmh-core</artifactId>
+			<scope>test</scope>
 		</dependency>
 		<dependency>
-			<groupId>org.apache.commons</groupId>
-			<artifactId>commons-compress</artifactId>
+			<groupId>org.openjdk.jmh</groupId>
+			<artifactId>jmh-generator-annprocess</artifactId>
+			<scope>test</scope>
 		</dependency>
 
 		<!-- JMH -->

src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java

@@ -26,31 +26,6 @@
  * POSSIBILITY OF SUCH DAMAGE.
  * #L%
  */
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- *    this list of conditions and the following disclaimer in the documentation
- *    and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
 package org.janelia.saalfeldlab.n5;
 
 import java.util.function.ToIntFunction;
@@ -65,9 +40,9 @@
  */
 public abstract class AbstractDataBlock<T> implements DataBlock<T> {
 
-	private final int[] size;
-	private final long[] gridPosition;
-	private final T data;
+	protected final int[] size;
+	protected final long[] gridPosition;
+	protected final T data;
 	private final ToIntFunction<T> numElements;
 
 	public AbstractDataBlock(

src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java

@@ -26,31 +26,6 @@
  * POSSIBILITY OF SUCH DAMAGE.
  * #L%
  */
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- *    this list of conditions and the following disclaimer in the documentation
- *    and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
 package org.janelia.saalfeldlab.n5;
 
 public class ByteArrayDataBlock extends AbstractDataBlock<byte[]> {

src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java

@@ -26,31 +26,6 @@
  * POSSIBILITY OF SUCH DAMAGE.
  * #L%
  */
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- *    this list of conditions and the following disclaimer in the documentation
- *    and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
 package org.janelia.saalfeldlab.n5;
 
 import java.io.IOException;

src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java

@@ -73,7 +73,7 @@ default DatasetAttributes getDatasetAttributes(final String pathName) {
 			return null;
 
 		if (cacheMeta()) {
-			attributes = getCache().getAttributes(normalPath, N5KeyValueReader.ATTRIBUTES_JSON);
+			attributes = getCache().getAttributes(normalPath, getAttributesKey());
 		} else {
 			attributes = GsonKeyValueN5Reader.super.getAttributes(normalPath);
 		}
@@ -99,7 +99,7 @@ default <T> T getAttribute(
 
 		final JsonElement attributes;
 		if (cacheMeta()) {
-			attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON);
+			attributes = getCache().getAttributes(normalPathName, getAttributesKey());
 		} else {
 			attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName);
 		}
@@ -120,7 +120,7 @@ default <T> T getAttribute(
 		final String normalizedAttributePath = N5URI.normalizeAttributePath(key);
 		JsonElement attributes;
 		if (cacheMeta()) {
-			attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON);
+			attributes = getCache().getAttributes(normalPathName, getAttributesKey());
 		} else {
 			attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName);
 		}
@@ -136,7 +136,7 @@ default boolean exists(final String pathName) {
 
 		final String normalPathName = N5URI.normalizeGroupPath(pathName);
 		if (cacheMeta())
-			return getCache().isGroup(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON);
+			return getCache().isGroup(normalPathName, getAttributesKey());
 		else {
 			return existsFromContainer(normalPathName, null);
 		}
@@ -180,7 +180,7 @@ default boolean datasetExists(final String pathName) throws N5IOException {
 
 		final String normalPathName = N5URI.normalizeGroupPath(pathName);
 		if (cacheMeta()) {
-			return getCache().isDataset(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON);
+			return getCache().isDataset(normalPathName, getAttributesKey());
 		}
 		return isDatasetFromContainer(normalPathName);
 	}
@@ -212,7 +212,7 @@ default JsonElement getAttributes(final String pathName) throws N5IOException {
 
 		/* If cached, return the cache */
 		if (cacheMeta()) {
-			return getCache().getAttributes(groupPath, N5KeyValueReader.ATTRIBUTES_JSON);
+			return getCache().getAttributes(groupPath, getAttributesKey());
 		} else {
 			return GsonKeyValueN5Reader.super.getAttributes(groupPath);
 		}

src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java

@@ -28,10 +28,6 @@
  */
 package org.janelia.saalfeldlab.n5;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.Arrays;
-
 import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
 
 import com.google.gson.Gson;
@@ -61,19 +57,11 @@ default void createGroup(final String path) throws N5Exception {
 		// avoid hitting the backend if this path is already a group according to the cache
 		// else if exists is true (then a dataset is present) so throw an exception to avoid
 		// overwriting / invalidating existing data
-		if (cacheMeta()) {
-			if (getCache().isGroup(normalPath, N5KeyValueReader.ATTRIBUTES_JSON))
-				return;
-			else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) {
-				throw new N5Exception("Can't make a group on existing path.");
-			}
-		}
+		if (groupExists(normalPath))
+			return;
+		else if (datasetExists(normalPath))
+			throw new N5Exception("Can't make a group on existing dataset.");
 
-		// N5Writer.super.createGroup(path);
-		/*
-		 * the lines below duplicate the single line above but would have to call
-		 * normalizeGroupPath again the below duplicates code, but avoids extra work
-		 */
 		getKeyValueAccess().createDirectories(absoluteGroupPath(normalPath));
 
 		if (cacheMeta()) {
@@ -88,8 +76,8 @@ else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) {
 			for (final String child : pathParts) {
 
 				final String childPath = parent.isEmpty() ? child : parent + "/" + child;
-				getCache().initializeNonemptyCache(childPath, N5KeyValueReader.ATTRIBUTES_JSON);
-				getCache().updateCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON);
+				getCache().initializeNonemptyCache(childPath, getAttributesKey());
+				getCache().updateCacheInfo(childPath, getAttributesKey());
 
 				// only add if the parent exists and has children cached already
 				if (parent != null && !child.isEmpty())
@@ -130,7 +118,7 @@ default void writeAndCacheAttributes(
 				nullRespectingAttributes = getGson().toJsonTree(attributes);
 			}
 			/* Update the cache, and write to the writer */
-			getCache().updateCacheInfo(normalGroupPath, N5KeyValueReader.ATTRIBUTES_JSON, nullRespectingAttributes);
+			getCache().updateCacheInfo(normalGroupPath, getAttributesKey(), nullRespectingAttributes);
 		}
 	}
 

src/main/java/org/janelia/saalfeldlab/n5/Compression.java

@@ -26,31 +26,6 @@
  * POSSIBILITY OF SUCH DAMAGE.
  * #L%
  */
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- *    this list of conditions and the following disclaimer in the documentation
- *    and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
 package org.janelia.saalfeldlab.n5;
 
 import java.io.Serializable;
@@ -61,8 +36,8 @@
 import java.lang.annotation.Target;
 
 import org.janelia.saalfeldlab.n5.codec.DataCodec;
-import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
 import org.janelia.saalfeldlab.n5.codec.CodecInfo;
+import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
 import org.scijava.annotations.Indexable;
 
 /**
@@ -74,11 +49,12 @@
  * serialization.
  * <p>
  * See also: an alternative method for serializing general {@link CodecInfo}s is
- * with the {@link NameConfigAdapter}.
+ * with the {@link NameConfigAdapter}. This interface remains for legacy
+ * (de)serialization.
  *
  * @author Stephan Saalfeld
  */
-public interface Compression extends Serializable, DataCodecInfo, DataCodec {
+public interface Compression extends Serializable, DataCodec, DataCodecInfo {
 
 	/**
 	 * Annotation for runtime discovery of compression schemes.
@@ -102,7 +78,6 @@ public interface Compression extends Serializable, DataCodecInfo, DataCodec {
 	@Target(ElementType.FIELD)
 	@interface CompressionParameter {}
 
-	@Override
 	default String getType() {
 
 		final CompressionType compressionType = getClass().getAnnotation(CompressionType.class);
@@ -116,4 +91,5 @@ default String getType() {
 	default DataCodec create() {
 		return this;
 	}
+
 }

src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java

@@ -26,31 +26,6 @@
  * POSSIBILITY OF SUCH DAMAGE.
  * #L%
  */
-/**
- * Copyright (c) 2017, Stephan Saalfeld
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * 1. Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright notice,
- *    this list of conditions and the following disclaimer in the documentation
- *    and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
 package org.janelia.saalfeldlab.n5;
 
 import java.lang.reflect.Constructor;
@@ -124,7 +99,7 @@ public static synchronized void update(final boolean override) {
 
 					newInstance.compressionConstructors.put(type, constructor);
 					newInstance.compressionParameters.put(type, parameters);
-				} catch (final ClassNotFoundException | NoSuchMethodException | ClassCastException
+				} catch (final NoClassDefFoundError | ClassNotFoundException | NoSuchMethodException | ClassCastException
 						| UnsatisfiedLinkError e) {
 					System.err.println("Compression '" + item.className() + "' could not be registered");
 				}