Migrate n5 4.0

pom.xml

@@ -4,12 +4,12 @@
 	<parent>
 		<groupId>org.scijava</groupId>
 		<artifactId>pom-scijava</artifactId>
-		<version>37.0.0</version>
+		<version>45.0.0</version>
 	</parent>
 
 	<groupId>org.janelia.saalfeldlab</groupId>
 	<artifactId>label-utilities-spark</artifactId>
-	<version>1.1.1-SNAPSHOT</version>
+	<version>1.2.0-SNAPSHOT</version>
 
 	<name>N5-Label-Multisets-Spark</name>
 	<description>Spark based tools for label data.</description>
@@ -99,15 +99,6 @@
 		<javadoc.skip>true</javadoc.skip>
 		<maven.javadoc.skip>${javadoc.skip}</maven.javadoc.skip>
 
-		<n5.version>3.3.1</n5.version>
-		<n5-aws-s3.version>4.2.2</n5-aws-s3.version>
-		<n5-google-cloud.version>4.1.2</n5-google-cloud.version>
-		<n5-universe.version>2.0.1</n5-universe.version>
-		<n5-hdf5.version>2.2.0</n5-hdf5.version>
-		<n5-zarr.version>1.4.0</n5-zarr.version>
-		<n5-imglib2.version>7.0.0</n5-imglib2.version>
-
-		<imglib2-label-multisets.version>0.14.0</imglib2-label-multisets.version>
 		<slf4j-api.version>1.7.36</slf4j-api.version>
 
 		<label-utilities.version>0.5.1</label-utilities.version>

src/main/java/org/janelia/saalfeldlab/label/spark/affinities/AverageAffinities.java

@@ -34,7 +34,7 @@
 import org.janelia.saalfeldlab.n5.DataBlock;
 import org.janelia.saalfeldlab.n5.DataType;
 import org.janelia.saalfeldlab.n5.DatasetAttributes;
-import org.janelia.saalfeldlab.n5.GzipCompression;
+import org.janelia.scicomp.n5.zstandard.ZstandardCompression;
 import org.janelia.saalfeldlab.n5.N5Reader;
 import org.janelia.saalfeldlab.n5.N5Writer;
 import org.janelia.saalfeldlab.n5.imglib2.N5Utils;
@@ -211,9 +211,9 @@ public static void run(String[] argv) throws IOException {
 		final DatasetAttributes inputAttributes = n5InSupplier.get().getDatasetAttributes(args.affinities);
 		final N5WriterSupplier n5OutSupplier = new N5WriterSupplier(args.outputContainer, args.prettyPrint, args.disbaleHtmlEscape);
 
-		n5OutSupplier.get().createDataset(args.averaged, ignoreLast(inputAttributes.getDimensions()), args.blockSize, DataType.FLOAT32, new GzipCompression());
+		n5OutSupplier.get().createDataset(args.averaged, ignoreLast(inputAttributes.getDimensions()), args.blockSize, DataType.FLOAT32, new ZstandardCompression());
 		n5InSupplier.get().listAttributes(args.affinities).forEach(ThrowingBiConsumer.unchecked((key, clazz) -> n5OutSupplier.get().setAttribute(args.averaged, key, n5InSupplier.get().getAttribute(args.affinities, key, clazz))));
-		n5OutSupplier.get().createDataset(args.averaged, ignoreLast(inputAttributes.getDimensions()), args.blockSize, DataType.FLOAT32, new GzipCompression());
+		n5OutSupplier.get().createDataset(args.averaged, ignoreLast(inputAttributes.getDimensions()), args.blockSize, DataType.FLOAT32, new ZstandardCompression());
 		n5OutSupplier.get().setAttribute(args.averaged, SUCCESS_KEY, false);
 
 		final Supplier<RandomAccessible<FloatType>> gliaMaskSupplier = args.gliaMask == null
@@ -389,7 +389,7 @@ private static void run(
 							success[i] = false;
 							try {
 								n5out.writeBlock(averaged, attributes, block);
-								final DataBlock<float[]> reloaded = (DataBlock<float[]>)n5out.readBlock(averaged, attributes, saveThisBlockAt);
+								final DataBlock<float[]> reloaded = n5out.readBlock(averaged, attributes, saveThisBlockAt);
 								success[i] = Arrays.equals(block.getData(), reloaded.getData());
 							} catch (Exception e) {
 								success[i] = false;

src/main/java/org/janelia/saalfeldlab/label/spark/affinities/MakePredictionMask.java

@@ -26,7 +26,7 @@
 import org.janelia.saalfeldlab.label.spark.N5Helpers;
 import org.janelia.saalfeldlab.n5.DataType;
 import org.janelia.saalfeldlab.n5.DatasetAttributes;
-import org.janelia.saalfeldlab.n5.GzipCompression;
+import org.janelia.scicomp.n5.zstandard.ZstandardCompression;
 import org.janelia.saalfeldlab.n5.N5Writer;
 import org.janelia.saalfeldlab.n5.imglib2.N5Utils;
 import org.slf4j.Logger;
@@ -231,7 +231,7 @@ public static void run(String[] argv) throws IOException {
 		final long[] validInputSizeInOutputCoordinates = convertAsLong(divide(inputSizeWorld, args.outputResolution), Math::floor);
 
 		final N5WriterSupplier n5out = new N5WriterSupplier(args.maskContainer, true, true);
-		n5out.get().createDataset(args.maskDataset, outputDatasetSize, args.blockSize(), DataType.UINT8, new GzipCompression());
+		n5out.get().createDataset(args.maskDataset, outputDatasetSize, args.blockSize(), DataType.UINT8, new ZstandardCompression());
 		n5out.get().setAttribute(args.maskDataset, NETWORK_SIZE_DIFF_KEY, networkSizeDiff);
 		n5out.get().setAttribute(args.maskDataset, "resolution", args.outputResolution);
 		n5out.get().setAttribute(args.maskDataset, "offset", args.outputOffset);
@@ -279,7 +279,7 @@ private static void run(
 						final long[] max = block._1()._2();
 						final Interval interval = new FinalInterval(min, max);
 						final RandomAccessible<UnsignedByteType> mask = inputMask.get();
-						final DatasetAttributes attributes = new DatasetAttributes(outputDatasetSize, blockSize, DataType.UINT8, new GzipCompression());
+						final DatasetAttributes attributes = new DatasetAttributes(outputDatasetSize, blockSize, DataType.UINT8, new ZstandardCompression());
 						final double[] minReal = LongStream.of(min).asDoubleStream().toArray();
 						final double[] maxReal = LongStream.of(max).asDoubleStream().toArray();
 						//						final Scale outputScale = new Scale(outputVoxelSize);

src/main/java/org/janelia/saalfeldlab/label/spark/affinities/SparkRain.java

@@ -49,7 +49,7 @@
 import org.janelia.saalfeldlab.n5.DataBlock;
 import org.janelia.saalfeldlab.n5.DataType;
 import org.janelia.saalfeldlab.n5.DatasetAttributes;
-import org.janelia.saalfeldlab.n5.GzipCompression;
+import org.janelia.scicomp.n5.zstandard.ZstandardCompression;
 import org.janelia.saalfeldlab.n5.LongArrayDataBlock;
 import org.janelia.saalfeldlab.n5.N5Reader;
 import org.janelia.saalfeldlab.n5.N5Writer;
@@ -317,11 +317,11 @@ public static void run(final String... argv) throws IOException {
 		attributes.put(OFFSET_KEY, offset);
 
 		final Map<String, DatasetAttributes> datasets = new HashMap<>();
-		Arrays.asList(uint64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT64, new GzipCompression())));
-		Arrays.asList(uint8Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT8, new GzipCompression())));
+		Arrays.asList(uint64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT64, new ZstandardCompression())));
+		Arrays.asList(uint8Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT8, new ZstandardCompression())));
 
 		if (args.smoothAffinitiesSigma > 0.0)
-			prepareOutputDataset(n5out.get(), args.smoothedAffinities, new DatasetAttributes(inputDims, IntStream.concat(IntStream.of(args.blockSize), IntStream.of(1)).toArray(), DataType.FLOAT32, new GzipCompression()), attributes);
+			prepareOutputDataset(n5out.get(), args.smoothedAffinities, new DatasetAttributes(inputDims, IntStream.concat(IntStream.of(args.blockSize), IntStream.of(1)).toArray(), DataType.FLOAT32, new ZstandardCompression()), attributes);
 
 		if (hasHalo) {
 			prepareOutputDatasets(
@@ -469,8 +469,8 @@ public static void run(
 					final ArrayImg<UnsignedLongType, LongArray> labels = ArrayImgs.unsignedLongs(parentsAndRoots.getFirst(), dims);
 					final Interval relevantInterval = Intervals.expand(labels, negativeHalo);
 
-					final DatasetAttributes croppedAttributes = new DatasetAttributes(outputDims, blockSize, DataType.UINT64, new GzipCompression());
-					final DatasetAttributes watershedAttributes = new DatasetAttributes(outputDims, watershedBlockSize, DataType.UINT64, new GzipCompression());
+					final DatasetAttributes croppedAttributes = new DatasetAttributes(outputDims, blockSize, DataType.UINT64, new ZstandardCompression());
+					final DatasetAttributes watershedAttributes = new DatasetAttributes(outputDims, watershedBlockSize, DataType.UINT64, new ZstandardCompression());
 
 					LOG.debug("Saving cropped watersheds to {}", hasHalo ? String.format(croppedDatasetPattern, watersheds) : watersheds);
 
@@ -543,7 +543,7 @@ public static void run(
 					final List<Point> seeds = Watersheds.collectSeeds(watershedSeedsMaskImg);
 					LOG.debug("Found watershed seeds {}", seeds);
 					final RandomAccessibleInterval<UnsignedByteType> watershedSeedsMaskImgUint8 = Converters.convert(watershedSeedsMaskImg, (src, tgt) -> tgt.set(src.get() ? 1 : 0), new UnsignedByteType());
-					final DatasetAttributes croppedWatershedSeedsAtributes = new DatasetAttributes(outputDims, blockSize, DataType.UINT8, new GzipCompression());
+					final DatasetAttributes croppedWatershedSeedsAtributes = new DatasetAttributes(outputDims, blockSize, DataType.UINT8, new ZstandardCompression());
 					N5Utils.saveBlock(Views.interval(watershedSeedsMaskImgUint8, relevantInterval), n5Writer, hasHalo ? String.format(croppedDatasetPattern, watershedSeeds) : watershedSeeds, croppedWatershedSeedsAtributes, blockOffset);
 					if (hasHalo) {
 						throw new UnsupportedOperationException("Need to implement halo support!");
@@ -679,7 +679,7 @@ private static void relabel(
 			final LongUnaryOperator idMapping) throws IOException {
 
 		final DatasetAttributes attributes = n5.getDatasetAttributes(dataset);
-		final LongArrayDataBlock block = ((LongArrayDataBlock)n5.readBlock(dataset, attributes, blockPos));
+		final DataBlock<long[]> block = n5.readBlock(dataset, attributes, blockPos);
 		final long[] data = block.getData();
 		for (int i = 0; i < data.length; ++i) {
 			data[i] = idMapping.applyAsLong(data[i]);

src/main/java/org/janelia/saalfeldlab/label/spark/convert/ConvertToLabelMultisetType.java

@@ -2,13 +2,11 @@
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
-import com.pivovarit.function.ThrowingSupplier;
 import net.imglib2.FinalInterval;
 import net.imglib2.Interval;
 import net.imglib2.RandomAccessibleInterval;
 import net.imglib2.algorithm.util.Grids;
 import net.imglib2.algorithm.util.Singleton;
-import net.imglib2.cache.img.CachedCellImg;
 import net.imglib2.converter.Converters;
 import net.imglib2.type.NativeType;
 import net.imglib2.type.label.FromIntegerTypeConverter;
@@ -21,13 +19,13 @@
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.Function0;
 import org.janelia.saalfeldlab.label.spark.N5Helpers;
 import org.janelia.saalfeldlab.n5.Compression;
 import org.janelia.saalfeldlab.n5.CompressionAdapter;
 import org.janelia.saalfeldlab.n5.DataType;
-import org.janelia.saalfeldlab.n5.DatasetAttributes;
 import org.janelia.saalfeldlab.n5.GsonUtils;
-import org.janelia.saalfeldlab.n5.GzipCompression;
+import org.janelia.scicomp.n5.zstandard.ZstandardCompression;
 import org.janelia.saalfeldlab.n5.N5Reader;
 import org.janelia.saalfeldlab.n5.N5Writer;
 import org.janelia.saalfeldlab.n5.imglib2.N5LabelMultisets;
@@ -45,7 +43,6 @@
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.net.URI;
-import java.net.URISyntaxException;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -95,9 +92,6 @@ static public class CommandLineParameters implements Callable<Void> {
 		@Option(names = {"--block-size", "-b"}, paramLabel = "BLOCK_SIZE", description = "Size of cells to use in the output N5 dataset. Defaults to 64. Either single integer value for isotropic block size or comma-seperated list of block size per dimension", split = ",")
 		private int[] blockSize;
 
-		@Option(names = {"--compression", "-c"}, paramLabel = "COMPRESSION", description = "Compression type to use in output N5 dataset")
-		public String compressionType = "{\"type\":\"gzip\",\"level\":-1}";
-
 		@Option(
 				names = {"--reverse-array-attributes"},
 				required = false,
@@ -113,10 +107,7 @@ public Void call() throws IOException {
 			final Gson gson = new GsonBuilder()
 					.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter())
 					.create();
-			final Compression compression = new GzipCompression();// .fromJson(
-			// compressionType,
-			// Compression.class
-			// );
+			final Compression compression = new ZstandardCompression();
 			final int nDim = N5Helpers.n5Reader(this.inputN5).getDatasetAttributes(this.inputDataset).getNumDimensions();
 			final int[] blockSize = this.blockSize.length < nDim ? IntStream.generate(() -> this.blockSize[0]).limit(nDim).toArray() : this.blockSize;
 
@@ -168,39 +159,52 @@ public static <I extends IntegerType<I> & NativeType<I>> void convertToLabelMult
 			final String outputGroupName,
 			final String outputDatasetName,
 			final Compression compression,
-			final boolean reverse) throws IOException {
+			final boolean reverse) {
+
+		final N5Reader reader = N5Helpers.n5Reader(inputGroup, blockSize);
+		final int[] inputBlockSize = reader.getDatasetAttributes(inputDataset).getBlockSize();
+		final Map<String, Object> attributesToCopy = resolveAttributesToCopy(reader, inputDataset, reverse);
 
-		final ThrowingSupplier<CachedCellImg<I, ?>, URISyntaxException> getCachedImg = () -> {
+		final Function0<RandomAccessibleInterval<I>> imgSupplier = () -> {
 			final URI inputGroupUri = StorageFormat.parseUri(inputGroup).getB();
 			final String inputImgCacheKey = new URIBuilder(inputGroupUri)
 					.setParameters(
 							new BasicNameValuePair("call", "convert-to-label-multiset-type"),
 							new BasicNameValuePair("dataset", inputDataset)
 					).toString();
-
 			return Singleton.get(inputImgCacheKey, () -> N5Helpers.openBounded(N5Helpers.n5Reader(inputGroup, blockSize), inputDataset));
 		};
 
-		final N5Reader reader = N5Helpers.n5Reader(inputGroup, blockSize);
-		final DatasetAttributes inputDataAttrs = reader.getDatasetAttributes(inputDataset);
-		final int[] inputBlockSize = inputDataAttrs.getBlockSize();
-		final RandomAccessibleInterval<I> img = getCachedImg.uncheck().get();
-		final Map<String, Class<?>> attributeNames;
-		if (reader instanceof ZarrKeyValueReader) {
-			attributeNames = Optional.of(reader)
-					.map(ZarrKeyValueReader.class::cast)
-					.map(it -> it.getZAttributes(inputDataset))
-					.map(GsonUtils::listAttributes)
-					.orElseGet(HashMap::new);
-		} else {
-			attributeNames = reader.listAttributes(inputDataset);
-			List.of(
-					LABEL_MULTISETTYPE_KEY,
-					DATA_TYPE_KEY,
-					COMPRESSION_KEY,
-					BLOCK_SIZE_KEY,
-					DIMENSIONS_KEY
-			).forEach(attributeNames::remove);
+		convertToLabelMultisetType(sc, imgSupplier, inputBlockSize, attributesToCopy, blockSize, outputGroupName, outputDatasetName, compression);
+	}
+
+	public static <I extends IntegerType<I> & NativeType<I>> void convertToLabelMultisetType(
+			final JavaSparkContext sc,
+			final Function0<RandomAccessibleInterval<I>> imgSupplier,
+			final int[] inputBlockSize,
+			final int[] blockSize,
+			final String outputGroupName,
+			final String outputDatasetName,
+			final Compression compression) {
+
+		convertToLabelMultisetType(sc, imgSupplier, inputBlockSize, new HashMap<>(), blockSize, outputGroupName, outputDatasetName, compression);
+	}
+
+	private static <I extends IntegerType<I> & NativeType<I>> void convertToLabelMultisetType(
+			final JavaSparkContext sc,
+			final Function0<RandomAccessibleInterval<I>> imgSupplier,
+			final int[] inputBlockSize,
+			final Map<String, Object> attributesToCopy,
+			final int[] blockSize,
+			final String outputGroupName,
+			final String outputDatasetName,
+			final Compression compression) {
+
+		final RandomAccessibleInterval<I> img;
+		try {
+			img = imgSupplier.call();
+		} catch (final Exception e) {
+			throw new RuntimeException("Unable to get source image", e);
 		}
 
 		final int nDim = img.numDimensions();
@@ -224,8 +228,8 @@ public static <I extends IntegerType<I> & NativeType<I>> void convertToLabelMult
 		}
 		writer.createDataset(outputDatasetName, dimensions, blockSize, DataType.UINT8, compression);
 		writer.setAttribute(outputDatasetName, LABEL_MULTISETTYPE_KEY, true);
-		for (final Entry<String, Class<?>> entry : attributeNames.entrySet())
-			writer.setAttribute(outputDatasetName, entry.getKey(), N5Helpers.reverseInplaceAndReturn(reader.getAttribute(inputDataset, entry.getKey(), entry.getValue()), reverse));
+		for (final Entry<String, Object> entry : attributesToCopy.entrySet())
+			writer.setAttribute(outputDatasetName, entry.getKey(), entry.getValue());
 
 		final int[] parallelizeBlockSize = new int[blockSize.length];
 		if (Intervals.numElements(blockSize) >= Intervals.numElements(inputBlockSize)) {
@@ -246,13 +250,7 @@ public static <I extends IntegerType<I> & NativeType<I>> void convertToLabelMult
 				.map(intervalMinMax -> {
 					final Interval interval = new FinalInterval(intervalMinMax._1(), intervalMinMax._2());
 
-					final URI uri = StorageFormat.parseUri(inputGroup).getB();
-					final String imgCacheKey = new URIBuilder(uri)
-							.setParameters(
-									new BasicNameValuePair("call", "convert-to-label-multiset-max-id"),
-									new BasicNameValuePair("dataset", inputDataset)
-							).toString();
-					final CachedCellImg<I, ?> source = Singleton.get(imgCacheKey, () -> N5Helpers.openBounded(N5Helpers.n5Reader(inputGroup, blockSize), inputDataset));
+					final RandomAccessibleInterval<I> source = imgSupplier.call();
 					final RandomAccessibleInterval<I> blockImg = Views.interval(source, interval);
 
 					final FromIntegerTypeConverter<I> converter = new FromIntegerTypeConverter<>();
@@ -280,4 +278,28 @@ public static <I extends IntegerType<I> & NativeType<I>> void convertToLabelMult
 		Singleton.clear();
 		writer.setAttribute(outputDatasetName, MAX_ID_KEY, maxId);
 	}
+
+	private static Map<String, Object> resolveAttributesToCopy(final N5Reader reader, final String inputDataset, final boolean reverse) {
+		final Map<String, Class<?>> attributeNames;
+		if (reader instanceof ZarrKeyValueReader) {
+			attributeNames = Optional.of(reader)
+					.map(ZarrKeyValueReader.class::cast)
+					.map(it -> it.getZAttributes(inputDataset))
+					.map(GsonUtils::listAttributes)
+					.orElseGet(HashMap::new);
+		} else {
+			attributeNames = reader.listAttributes(inputDataset);
+			List.of(
+					LABEL_MULTISETTYPE_KEY,
+					DATA_TYPE_KEY,
+					COMPRESSION_KEY,
+					BLOCK_SIZE_KEY,
+					DIMENSIONS_KEY
+			).forEach(attributeNames::remove);
+		}
+		final Map<String, Object> resolved = new HashMap<>();
+		for (final Entry<String, Class<?>> entry : attributeNames.entrySet())
+			resolved.put(entry.getKey(), N5Helpers.reverseInplaceAndReturn(reader.getAttribute(inputDataset, entry.getKey(), entry.getValue()), reverse));
+		return resolved;
+	}
 }

src/main/java/org/janelia/saalfeldlab/label/spark/downsample/SparkDownsampler.java

@@ -70,7 +70,7 @@ public static class CommandLineParameters implements Callable<Void> {
 		private int[] maxNumEntries;
 
 		@Option(names = {"--compression", "-c"}, paramLabel = "COMPRESSION", description = "Compression type to use in output N5 dataset")
-		public String compressionType = "{\"type\":\"gzip\",\"level\":\"-1\"}";
+		public String compressionType = "{\"type\":\"zstd\",\"level\":\"3\"}";
 
 		@Option(names = {"-h", "--help"}, usageHelp = true, description = "display a help message")
 		private boolean helpRequested;

src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/ExtractAndStoreLabelList.java

@@ -22,7 +22,7 @@
 import org.janelia.saalfeldlab.label.spark.N5Helpers;
 import org.janelia.saalfeldlab.n5.DataType;
 import org.janelia.saalfeldlab.n5.DatasetAttributes;
-import org.janelia.saalfeldlab.n5.GzipCompression;
+import org.janelia.scicomp.n5.zstandard.ZstandardCompression;
 import org.janelia.saalfeldlab.n5.LongArrayDataBlock;
 import org.janelia.saalfeldlab.n5.N5Reader;
 import org.janelia.saalfeldlab.n5.N5Writer;
@@ -167,7 +167,7 @@ private static <I extends IntegerType<I> & NativeType<I>> long callImpl(
 				grid.getImgDimensions(),
 				blockSize,
 				DataType.UINT64,
-				new GzipCompression());
+				new ZstandardCompression());
 		n5writer.writeBlock(outputDataset, attributes, block);
 		return maxVal;
 	}