diff --git a/pom.xml b/pom.xml index 6d0996d..b86489b 100644 --- a/pom.xml +++ b/pom.xml @@ -4,12 +4,12 @@ org.scijava pom-scijava - 37.0.0 + 45.0.0 org.janelia.saalfeldlab label-utilities-spark - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT N5-Label-Multisets-Spark Spark based tools for label data. @@ -99,15 +99,6 @@ true ${javadoc.skip} - 3.3.1 - 4.2.2 - 4.1.2 - 2.0.1 - 2.2.0 - 1.4.0 - 7.0.0 - - 0.14.0 1.7.36 0.5.1 diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/affinities/AverageAffinities.java b/src/main/java/org/janelia/saalfeldlab/label/spark/affinities/AverageAffinities.java index 2e7922f..3a1e8c0 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/affinities/AverageAffinities.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/affinities/AverageAffinities.java @@ -34,7 +34,7 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; @@ -211,9 +211,9 @@ public static void run(String[] argv) throws IOException { final DatasetAttributes inputAttributes = n5InSupplier.get().getDatasetAttributes(args.affinities); final N5WriterSupplier n5OutSupplier = new N5WriterSupplier(args.outputContainer, args.prettyPrint, args.disbaleHtmlEscape); - n5OutSupplier.get().createDataset(args.averaged, ignoreLast(inputAttributes.getDimensions()), args.blockSize, DataType.FLOAT32, new GzipCompression()); + n5OutSupplier.get().createDataset(args.averaged, ignoreLast(inputAttributes.getDimensions()), args.blockSize, DataType.FLOAT32, new ZstandardCompression()); n5InSupplier.get().listAttributes(args.affinities).forEach(ThrowingBiConsumer.unchecked((key, clazz) -> n5OutSupplier.get().setAttribute(args.averaged, key, n5InSupplier.get().getAttribute(args.affinities, key, clazz)))); - n5OutSupplier.get().createDataset(args.averaged, ignoreLast(inputAttributes.getDimensions()), args.blockSize, DataType.FLOAT32, new GzipCompression()); + n5OutSupplier.get().createDataset(args.averaged, ignoreLast(inputAttributes.getDimensions()), args.blockSize, DataType.FLOAT32, new ZstandardCompression()); n5OutSupplier.get().setAttribute(args.averaged, SUCCESS_KEY, false); final Supplier> gliaMaskSupplier = args.gliaMask == null @@ -389,7 +389,7 @@ private static void run( success[i] = false; try { n5out.writeBlock(averaged, attributes, block); - final DataBlock reloaded = (DataBlock)n5out.readBlock(averaged, attributes, saveThisBlockAt); + final DataBlock reloaded = n5out.readBlock(averaged, attributes, saveThisBlockAt); success[i] = Arrays.equals(block.getData(), reloaded.getData()); } catch (Exception e) { success[i] = false; diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/affinities/MakePredictionMask.java b/src/main/java/org/janelia/saalfeldlab/label/spark/affinities/MakePredictionMask.java index 14cbcd1..799c9bb 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/affinities/MakePredictionMask.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/affinities/MakePredictionMask.java @@ -26,7 +26,7 @@ import org.janelia.saalfeldlab.label.spark.N5Helpers; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.slf4j.Logger; @@ -231,7 +231,7 @@ public static void run(String[] argv) throws IOException { final long[] validInputSizeInOutputCoordinates = convertAsLong(divide(inputSizeWorld, args.outputResolution), Math::floor); final N5WriterSupplier n5out = new N5WriterSupplier(args.maskContainer, true, true); - n5out.get().createDataset(args.maskDataset, outputDatasetSize, args.blockSize(), DataType.UINT8, new GzipCompression()); + n5out.get().createDataset(args.maskDataset, outputDatasetSize, args.blockSize(), DataType.UINT8, new ZstandardCompression()); n5out.get().setAttribute(args.maskDataset, NETWORK_SIZE_DIFF_KEY, networkSizeDiff); n5out.get().setAttribute(args.maskDataset, "resolution", args.outputResolution); n5out.get().setAttribute(args.maskDataset, "offset", args.outputOffset); @@ -279,7 +279,7 @@ private static void run( final long[] max = block._1()._2(); final Interval interval = new FinalInterval(min, max); final RandomAccessible mask = inputMask.get(); - final DatasetAttributes attributes = new DatasetAttributes(outputDatasetSize, blockSize, DataType.UINT8, new GzipCompression()); + final DatasetAttributes attributes = new DatasetAttributes(outputDatasetSize, blockSize, DataType.UINT8, new ZstandardCompression()); final double[] minReal = LongStream.of(min).asDoubleStream().toArray(); final double[] maxReal = LongStream.of(max).asDoubleStream().toArray(); // final Scale outputScale = new Scale(outputVoxelSize); diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/affinities/SparkRain.java b/src/main/java/org/janelia/saalfeldlab/label/spark/affinities/SparkRain.java index 25f9a46..4342281 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/affinities/SparkRain.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/affinities/SparkRain.java @@ -49,7 +49,7 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; @@ -317,11 +317,11 @@ public static void run(final String... argv) throws IOException { attributes.put(OFFSET_KEY, offset); final Map datasets = new HashMap<>(); - Arrays.asList(uint64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT64, new GzipCompression()))); - Arrays.asList(uint8Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT8, new GzipCompression()))); + Arrays.asList(uint64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT64, new ZstandardCompression()))); + Arrays.asList(uint8Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT8, new ZstandardCompression()))); if (args.smoothAffinitiesSigma > 0.0) - prepareOutputDataset(n5out.get(), args.smoothedAffinities, new DatasetAttributes(inputDims, IntStream.concat(IntStream.of(args.blockSize), IntStream.of(1)).toArray(), DataType.FLOAT32, new GzipCompression()), attributes); + prepareOutputDataset(n5out.get(), args.smoothedAffinities, new DatasetAttributes(inputDims, IntStream.concat(IntStream.of(args.blockSize), IntStream.of(1)).toArray(), DataType.FLOAT32, new ZstandardCompression()), attributes); if (hasHalo) { prepareOutputDatasets( @@ -469,8 +469,8 @@ public static void run( final ArrayImg labels = ArrayImgs.unsignedLongs(parentsAndRoots.getFirst(), dims); final Interval relevantInterval = Intervals.expand(labels, negativeHalo); - final DatasetAttributes croppedAttributes = new DatasetAttributes(outputDims, blockSize, DataType.UINT64, new GzipCompression()); - final DatasetAttributes watershedAttributes = new DatasetAttributes(outputDims, watershedBlockSize, DataType.UINT64, new GzipCompression()); + final DatasetAttributes croppedAttributes = new DatasetAttributes(outputDims, blockSize, DataType.UINT64, new ZstandardCompression()); + final DatasetAttributes watershedAttributes = new DatasetAttributes(outputDims, watershedBlockSize, DataType.UINT64, new ZstandardCompression()); LOG.debug("Saving cropped watersheds to {}", hasHalo ? String.format(croppedDatasetPattern, watersheds) : watersheds); @@ -543,7 +543,7 @@ public static void run( final List seeds = Watersheds.collectSeeds(watershedSeedsMaskImg); LOG.debug("Found watershed seeds {}", seeds); final RandomAccessibleInterval watershedSeedsMaskImgUint8 = Converters.convert(watershedSeedsMaskImg, (src, tgt) -> tgt.set(src.get() ? 1 : 0), new UnsignedByteType()); - final DatasetAttributes croppedWatershedSeedsAtributes = new DatasetAttributes(outputDims, blockSize, DataType.UINT8, new GzipCompression()); + final DatasetAttributes croppedWatershedSeedsAtributes = new DatasetAttributes(outputDims, blockSize, DataType.UINT8, new ZstandardCompression()); N5Utils.saveBlock(Views.interval(watershedSeedsMaskImgUint8, relevantInterval), n5Writer, hasHalo ? String.format(croppedDatasetPattern, watershedSeeds) : watershedSeeds, croppedWatershedSeedsAtributes, blockOffset); if (hasHalo) { throw new UnsupportedOperationException("Need to implement halo support!"); @@ -679,7 +679,7 @@ private static void relabel( final LongUnaryOperator idMapping) throws IOException { final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - final LongArrayDataBlock block = ((LongArrayDataBlock)n5.readBlock(dataset, attributes, blockPos)); + final DataBlock block = n5.readBlock(dataset, attributes, blockPos); final long[] data = block.getData(); for (int i = 0; i < data.length; ++i) { data[i] = idMapping.applyAsLong(data[i]); diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/convert/ConvertToLabelMultisetType.java b/src/main/java/org/janelia/saalfeldlab/label/spark/convert/ConvertToLabelMultisetType.java index 7217a4a..5b4d374 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/convert/ConvertToLabelMultisetType.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/convert/ConvertToLabelMultisetType.java @@ -2,13 +2,11 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.pivovarit.function.ThrowingSupplier; import net.imglib2.FinalInterval; import net.imglib2.Interval; import net.imglib2.RandomAccessibleInterval; import net.imglib2.algorithm.util.Grids; import net.imglib2.algorithm.util.Singleton; -import net.imglib2.cache.img.CachedCellImg; import net.imglib2.converter.Converters; import net.imglib2.type.NativeType; import net.imglib2.type.label.FromIntegerTypeConverter; @@ -21,13 +19,13 @@ import org.apache.http.message.BasicNameValuePair; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.Function0; import org.janelia.saalfeldlab.label.spark.N5Helpers; import org.janelia.saalfeldlab.n5.Compression; import org.janelia.saalfeldlab.n5.CompressionAdapter; import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; import org.janelia.saalfeldlab.n5.GsonUtils; -import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.imglib2.N5LabelMultisets; @@ -45,7 +43,6 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.URI; -import java.net.URISyntaxException; import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; @@ -95,9 +92,6 @@ static public class CommandLineParameters implements Callable { @Option(names = {"--block-size", "-b"}, paramLabel = "BLOCK_SIZE", description = "Size of cells to use in the output N5 dataset. Defaults to 64. Either single integer value for isotropic block size or comma-seperated list of block size per dimension", split = ",") private int[] blockSize; - @Option(names = {"--compression", "-c"}, paramLabel = "COMPRESSION", description = "Compression type to use in output N5 dataset") - public String compressionType = "{\"type\":\"gzip\",\"level\":-1}"; - @Option( names = {"--reverse-array-attributes"}, required = false, @@ -113,10 +107,7 @@ public Void call() throws IOException { final Gson gson = new GsonBuilder() .registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()) .create(); - final Compression compression = new GzipCompression();// .fromJson( - // compressionType, - // Compression.class - // ); + final Compression compression = new ZstandardCompression(); final int nDim = N5Helpers.n5Reader(this.inputN5).getDatasetAttributes(this.inputDataset).getNumDimensions(); final int[] blockSize = this.blockSize.length < nDim ? IntStream.generate(() -> this.blockSize[0]).limit(nDim).toArray() : this.blockSize; @@ -168,39 +159,52 @@ public static & NativeType> void convertToLabelMult final String outputGroupName, final String outputDatasetName, final Compression compression, - final boolean reverse) throws IOException { + final boolean reverse) { + + final N5Reader reader = N5Helpers.n5Reader(inputGroup, blockSize); + final int[] inputBlockSize = reader.getDatasetAttributes(inputDataset).getBlockSize(); + final Map attributesToCopy = resolveAttributesToCopy(reader, inputDataset, reverse); - final ThrowingSupplier, URISyntaxException> getCachedImg = () -> { + final Function0> imgSupplier = () -> { final URI inputGroupUri = StorageFormat.parseUri(inputGroup).getB(); final String inputImgCacheKey = new URIBuilder(inputGroupUri) .setParameters( new BasicNameValuePair("call", "convert-to-label-multiset-type"), new BasicNameValuePair("dataset", inputDataset) ).toString(); - return Singleton.get(inputImgCacheKey, () -> N5Helpers.openBounded(N5Helpers.n5Reader(inputGroup, blockSize), inputDataset)); }; - final N5Reader reader = N5Helpers.n5Reader(inputGroup, blockSize); - final DatasetAttributes inputDataAttrs = reader.getDatasetAttributes(inputDataset); - final int[] inputBlockSize = inputDataAttrs.getBlockSize(); - final RandomAccessibleInterval img = getCachedImg.uncheck().get(); - final Map> attributeNames; - if (reader instanceof ZarrKeyValueReader) { - attributeNames = Optional.of(reader) - .map(ZarrKeyValueReader.class::cast) - .map(it -> it.getZAttributes(inputDataset)) - .map(GsonUtils::listAttributes) - .orElseGet(HashMap::new); - } else { - attributeNames = reader.listAttributes(inputDataset); - List.of( - LABEL_MULTISETTYPE_KEY, - DATA_TYPE_KEY, - COMPRESSION_KEY, - BLOCK_SIZE_KEY, - DIMENSIONS_KEY - ).forEach(attributeNames::remove); + convertToLabelMultisetType(sc, imgSupplier, inputBlockSize, attributesToCopy, blockSize, outputGroupName, outputDatasetName, compression); + } + + public static & NativeType> void convertToLabelMultisetType( + final JavaSparkContext sc, + final Function0> imgSupplier, + final int[] inputBlockSize, + final int[] blockSize, + final String outputGroupName, + final String outputDatasetName, + final Compression compression) { + + convertToLabelMultisetType(sc, imgSupplier, inputBlockSize, new HashMap<>(), blockSize, outputGroupName, outputDatasetName, compression); + } + + private static & NativeType> void convertToLabelMultisetType( + final JavaSparkContext sc, + final Function0> imgSupplier, + final int[] inputBlockSize, + final Map attributesToCopy, + final int[] blockSize, + final String outputGroupName, + final String outputDatasetName, + final Compression compression) { + + final RandomAccessibleInterval img; + try { + img = imgSupplier.call(); + } catch (final Exception e) { + throw new RuntimeException("Unable to get source image", e); } final int nDim = img.numDimensions(); @@ -224,8 +228,8 @@ public static & NativeType> void convertToLabelMult } writer.createDataset(outputDatasetName, dimensions, blockSize, DataType.UINT8, compression); writer.setAttribute(outputDatasetName, LABEL_MULTISETTYPE_KEY, true); - for (final Entry> entry : attributeNames.entrySet()) - writer.setAttribute(outputDatasetName, entry.getKey(), N5Helpers.reverseInplaceAndReturn(reader.getAttribute(inputDataset, entry.getKey(), entry.getValue()), reverse)); + for (final Entry entry : attributesToCopy.entrySet()) + writer.setAttribute(outputDatasetName, entry.getKey(), entry.getValue()); final int[] parallelizeBlockSize = new int[blockSize.length]; if (Intervals.numElements(blockSize) >= Intervals.numElements(inputBlockSize)) { @@ -246,13 +250,7 @@ public static & NativeType> void convertToLabelMult .map(intervalMinMax -> { final Interval interval = new FinalInterval(intervalMinMax._1(), intervalMinMax._2()); - final URI uri = StorageFormat.parseUri(inputGroup).getB(); - final String imgCacheKey = new URIBuilder(uri) - .setParameters( - new BasicNameValuePair("call", "convert-to-label-multiset-max-id"), - new BasicNameValuePair("dataset", inputDataset) - ).toString(); - final CachedCellImg source = Singleton.get(imgCacheKey, () -> N5Helpers.openBounded(N5Helpers.n5Reader(inputGroup, blockSize), inputDataset)); + final RandomAccessibleInterval source = imgSupplier.call(); final RandomAccessibleInterval blockImg = Views.interval(source, interval); final FromIntegerTypeConverter converter = new FromIntegerTypeConverter<>(); @@ -280,4 +278,28 @@ public static & NativeType> void convertToLabelMult Singleton.clear(); writer.setAttribute(outputDatasetName, MAX_ID_KEY, maxId); } + + private static Map resolveAttributesToCopy(final N5Reader reader, final String inputDataset, final boolean reverse) { + final Map> attributeNames; + if (reader instanceof ZarrKeyValueReader) { + attributeNames = Optional.of(reader) + .map(ZarrKeyValueReader.class::cast) + .map(it -> it.getZAttributes(inputDataset)) + .map(GsonUtils::listAttributes) + .orElseGet(HashMap::new); + } else { + attributeNames = reader.listAttributes(inputDataset); + List.of( + LABEL_MULTISETTYPE_KEY, + DATA_TYPE_KEY, + COMPRESSION_KEY, + BLOCK_SIZE_KEY, + DIMENSIONS_KEY + ).forEach(attributeNames::remove); + } + final Map resolved = new HashMap<>(); + for (final Entry> entry : attributeNames.entrySet()) + resolved.put(entry.getKey(), N5Helpers.reverseInplaceAndReturn(reader.getAttribute(inputDataset, entry.getKey(), entry.getValue()), reverse)); + return resolved; + } } diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/downsample/SparkDownsampler.java b/src/main/java/org/janelia/saalfeldlab/label/spark/downsample/SparkDownsampler.java index cf7478e..fb95660 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/downsample/SparkDownsampler.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/downsample/SparkDownsampler.java @@ -70,7 +70,7 @@ public static class CommandLineParameters implements Callable { private int[] maxNumEntries; @Option(names = {"--compression", "-c"}, paramLabel = "COMPRESSION", description = "Compression type to use in output N5 dataset") - public String compressionType = "{\"type\":\"gzip\",\"level\":\"-1\"}"; + public String compressionType = "{\"type\":\"zstd\",\"level\":\"3\"}"; @Option(names = {"-h", "--help"}, usageHelp = true, description = "display a help message") private boolean helpRequested; diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/ExtractAndStoreLabelList.java b/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/ExtractAndStoreLabelList.java index 12e4339..bd08cd1 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/ExtractAndStoreLabelList.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/ExtractAndStoreLabelList.java @@ -22,7 +22,7 @@ import org.janelia.saalfeldlab.label.spark.N5Helpers; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; @@ -167,7 +167,7 @@ private static & NativeType> long callImpl( grid.getImgDimensions(), blockSize, DataType.UINT64, - new GzipCompression()); + new ZstandardCompression()); n5writer.writeBlock(outputDataset, attributes, block); return maxVal; } diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/ExtractUniqueLabelsPerBlock.java b/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/ExtractUniqueLabelsPerBlock.java index f071cbd..f1d634e 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/ExtractUniqueLabelsPerBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/ExtractUniqueLabelsPerBlock.java @@ -12,7 +12,7 @@ import org.janelia.saalfeldlab.label.spark.exception.InvalidN5Container; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.universe.N5Factory; @@ -126,7 +126,7 @@ public static long extractUniqueLabels( } final N5Writer writer = n5Writer(outputN5, blockSize); - final DatasetAttributes outputAttributes = new DatasetAttributes(dims, blockSize, DataType.UINT64, new GzipCompression()); + final DatasetAttributes outputAttributes = new DatasetAttributes(dims, blockSize, DataType.UINT64, new ZstandardCompression()); writer.createDataset(outputDataset, outputAttributes); final List> intervals = Grids diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/LabelToBlockMapping.java b/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/LabelToBlockMapping.java index 81a351d..1425061 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/LabelToBlockMapping.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/LabelToBlockMapping.java @@ -17,12 +17,8 @@ import org.janelia.saalfeldlab.labels.blocks.LabelBlockLookup; import org.janelia.saalfeldlab.labels.blocks.LabelBlockLookupAdapter; import org.janelia.saalfeldlab.labels.blocks.n5.LabelBlockLookupFromN5Relative; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; -import org.janelia.saalfeldlab.n5.LongArrayDataBlock; -import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.*; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.janelia.saalfeldlab.n5.universe.StorageFormat; import org.slf4j.Logger; @@ -130,8 +126,7 @@ public static void createMappingWithMultiscaleCheckN5( final String[] sortedScaleDirs = N5Helpers.listAndSortScaleDatasets(reader, inputDataset); for (int level = 0; level < sortedScaleDirs.length; ++level) { final String scaleDataset = sortedScaleDirs[level]; - // writer.createDataset( enclosingGroup + "/" + scaleDataset, new long[] { Long.MAX_VALUE }, new int[] { stepSize }, DataType.INT8, new GzipCompression() ); - writer.createDataset(String.format(enclosingGroup + "/" + pattern, level), new long[]{Long.MAX_VALUE}, new int[]{stepSize}, DataType.INT8, new GzipCompression()); + writer.createDataset(String.format(enclosingGroup + "/" + pattern, level), new long[]{Long.MAX_VALUE}, new int[]{stepSize}, DataType.INT8, new ZstandardCompression()); LOG.info("Creating mapping for scale dataset {} in group {} of n5 container {} at target {}", scaleDataset, inputDataset, inputN5, enclosingGroup); createMappingN5( sc, @@ -196,7 +191,8 @@ public static void createMappingN5( new BasicNameValuePair("call", "label-block-mapping-create-mapping-n5") ).toString(); final N5Reader n5reader = Singleton.get(readerCacheKey, () -> N5Helpers.n5Reader(inputN5, N5Helpers.DEFAULT_BLOCK_SIZE)); - final LongArrayDataBlock block = (LongArrayDataBlock)n5reader.readBlock(inputDataset, new DatasetAttributes(dims, blockSize, DataType.UINT64, new GzipCompression()), blockPos); + DatasetAttributes inputDatasetAttributes = n5reader.getDatasetAttributes(inputDataset); + final DataBlock block = n5reader.readBlock(inputDataset, inputDatasetAttributes, blockPos); return new Tuple2<>(minMax, block.getData()); }) .flatMapToPair(input -> Arrays @@ -262,7 +258,8 @@ public static void createMapping( new BasicNameValuePair("call", "label-block-mapping-create-mapping") ).toString(); final N5Reader n5reader = Singleton.get(readerCacheKey, () -> N5Helpers.n5Reader(inputN5, N5Helpers.DEFAULT_BLOCK_SIZE)); - final LongArrayDataBlock block = (LongArrayDataBlock)n5reader.readBlock(inputDataset, new DatasetAttributes(dims, blockSize, DataType.UINT64, new GzipCompression()), blockPos); + DatasetAttributes inputDatasetAttributes = n5reader.getDatasetAttributes(inputDataset); + final DataBlock block = n5reader.readBlock(inputDataset, inputDatasetAttributes, blockPos); return new Tuple2<>(minMax, block.getData()); }) .flatMapToPair(input -> Arrays diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/downsample/LabelListDownsampleFunction.java b/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/downsample/LabelListDownsampleFunction.java index 27007fa..309952e 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/downsample/LabelListDownsampleFunction.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/downsample/LabelListDownsampleFunction.java @@ -9,10 +9,7 @@ import org.apache.http.message.BasicNameValuePair; import org.apache.spark.api.java.function.VoidFunction; import org.janelia.saalfeldlab.label.spark.N5Helpers; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.LongArrayDataBlock; -import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.*; import org.janelia.saalfeldlab.n5.universe.StorageFormat; import java.net.URI; @@ -82,7 +79,7 @@ public void call(final Interval interval) throws Exception { final int[] bs = attr.getBlockSize(); for (final long[] cellPos : cellPositions) { Arrays.setAll(cellPos, d -> cellPos[d] / bs[d]); - final LongArrayDataBlock source = (LongArrayDataBlock)reader.readBlock(inputDatasetName, attr, cellPos); + final DataBlock source = reader.readBlock(inputDatasetName, attr, cellPos); containedLabels.addAll(source.getData()); } diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/downsample/LabelListDownsampler.java b/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/downsample/LabelListDownsampler.java index 6735a08..9537a96 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/downsample/LabelListDownsampler.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/uniquelabels/downsample/LabelListDownsampler.java @@ -12,7 +12,7 @@ import org.janelia.saalfeldlab.n5.CompressionAdapter; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; import org.slf4j.Logger; @@ -185,7 +185,7 @@ public static void downsample(final JavaSparkContext sc, .ofNullable(reader.getAttribute(readDatasetName, DOWNSAMPLING_FACTORS_KEY, double[].class)) .orElse(DoubleStream.generate(() -> 1.0).limit(nDim).toArray()); final double[] accumulatedDownsamplingFactor = IntStream.range(0, nDim).mapToDouble(d -> previousDownsamplingFactor[d] * downsampleFactor[d]).toArray(); - writer.createDataset(outputDatasetName, downsampledDimensions, blockSize, DataType.UINT64, new GzipCompression()); + writer.createDataset(outputDatasetName, downsampledDimensions, blockSize, DataType.UINT64, new ZstandardCompression()); writer.setAttribute(outputDatasetName, DOWNSAMPLING_FACTORS_KEY, accumulatedDownsamplingFactor); sc.parallelize(positions) diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkMutexWatersheds.java b/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkMutexWatersheds.java index 8c8f685..4f8d70b 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkMutexWatersheds.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkMutexWatersheds.java @@ -38,7 +38,7 @@ import org.janelia.saalfeldlab.label.spark.N5Helpers; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; @@ -374,9 +374,9 @@ G extends RealType> void runMutexWatersheds( final N5Writer n5Out = outputContainer.get(); - n5Out.createDataset(mutexWatershedDataset, outputSize, blockSize, DataType.UINT64, new GzipCompression()); - n5Out.createDataset(mutexWatershedRelabeledDataset, outputSize, blockSize, DataType.UINT64, new GzipCompression()); - n5Out.createDataset(mutexWatershedMergedDataset, outputSize, blockSize, DataType.UINT64, new GzipCompression()); + n5Out.createDataset(mutexWatershedDataset, outputSize, blockSize, DataType.UINT64, new ZstandardCompression()); + n5Out.createDataset(mutexWatershedRelabeledDataset, outputSize, blockSize, DataType.UINT64, new ZstandardCompression()); + n5Out.createDataset(mutexWatershedMergedDataset, outputSize, blockSize, DataType.UINT64, new ZstandardCompression()); n5Out.setAttribute(mutexWatershedMergedDataset, "completedSuccessfully", false); final double primitiveThreshold = threshold == null ? -1.0 : threshold; @@ -508,7 +508,7 @@ G extends RealType> void runMutexWatersheds( final long[] blockOffset = new long[min.length]; Arrays.setAll(blockOffset, d -> min[d] / blockSize[d]); - final DatasetAttributes attributes = new DatasetAttributes(outputSize, blockSize, DataType.UINT64, new GzipCompression()); + final DatasetAttributes attributes = new DatasetAttributes(outputSize, blockSize, DataType.UINT64, new ZstandardCompression()); @@ -578,7 +578,7 @@ G extends RealType> void runMutexWatersheds( final long[] blockOffset = new long[bwo.min.length]; Arrays.setAll(blockOffset, d -> bwo.min[d] / blockSize[d]); - final DatasetAttributes attributes = new DatasetAttributes(outputSize, blockSize, DataType.UINT64, new GzipCompression()); + final DatasetAttributes attributes = new DatasetAttributes(outputSize, blockSize, DataType.UINT64, new ZstandardCompression()); N5Utils.saveBlock(relabeled, writer, mutexWatershedRelabeledDataset, attributes, blockOffset); }); @@ -689,7 +689,7 @@ G extends RealType> void runMutexWatersheds( final long[] blockOffset = new long[bwo.min.length]; Arrays.setAll(blockOffset, d -> bwo.min[d] / blockSize[d]); - final DatasetAttributes attributes = new DatasetAttributes(outputSize, blockSize, DataType.UINT64, new GzipCompression()); + final DatasetAttributes attributes = new DatasetAttributes(outputSize, blockSize, DataType.UINT64, new ZstandardCompression()); N5Utils.saveBlock(remapped, writer, mutexWatershedMergedDataset, attributes, blockOffset); }); Singleton.clear(); diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatersheds.java b/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatersheds.java index 7b59111..b02bcee 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatersheds.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatersheds.java @@ -53,7 +53,7 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.LongArrayDataBlock; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5Writer; @@ -227,8 +227,8 @@ public static void run(final String... argv) throws IOException { attributes.put(OFFSET_KEY, offset); final Map datasets = new HashMap<>(); - Arrays.asList(uint64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT64, new GzipCompression()))); - Arrays.asList(uint8Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT8, new GzipCompression()))); + Arrays.asList(uint64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT64, new ZstandardCompression()))); + Arrays.asList(uint8Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT8, new ZstandardCompression()))); if (hasHalo) { prepareOutputDatasets( @@ -339,8 +339,8 @@ public static void run( final Interval relevantInterval = Intervals.expand(labels, negativeHalo); - final DatasetAttributes croppedAttributes = new DatasetAttributes(outputDims, blockSize, DataType.UINT64, new GzipCompression()); - final DatasetAttributes watershedAttributes = new DatasetAttributes(outputDims, watershedBlockSize, DataType.UINT64, new GzipCompression()); + final DatasetAttributes croppedAttributes = new DatasetAttributes(outputDims, blockSize, DataType.UINT64, new ZstandardCompression()); + final DatasetAttributes watershedAttributes = new DatasetAttributes(outputDims, watershedBlockSize, DataType.UINT64, new ZstandardCompression()); // TODO do seeded watersheds LOG.debug("Found watershed seeds {}", seeds); @@ -585,7 +585,7 @@ private static void relabel( final LongUnaryOperator idMapping) throws IOException { final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - final LongArrayDataBlock block = ((LongArrayDataBlock)n5.readBlock(dataset, attributes, blockPos)); + final DataBlock block = n5.readBlock(dataset, attributes, blockPos); final long[] data = block.getData(); for (int i = 0; i < data.length; ++i) { data[i] = idMapping.applyAsLong(data[i]); diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatershedsOnDistanceTransform.java b/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatershedsOnDistanceTransform.java index 1bacdca..6b47e7e 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatershedsOnDistanceTransform.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatershedsOnDistanceTransform.java @@ -56,12 +56,8 @@ import org.apache.spark.api.java.function.PairFunction; import org.janelia.saalfeldlab.label.spark.N5Helpers; import org.janelia.saalfeldlab.label.spark.Version; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; -import org.janelia.saalfeldlab.n5.LongArrayDataBlock; -import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.*; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.slf4j.Logger; @@ -229,8 +225,8 @@ public static void run(final String... argv) throws IOException { attributes.put(OFFSET_KEY, offset); final Map datasets = new HashMap<>(); - Arrays.asList(uint64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT64, new GzipCompression()))); - Arrays.asList(float64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.FLOAT64, new GzipCompression()))); + Arrays.asList(uint64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT64, new ZstandardCompression()))); + Arrays.asList(float64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.FLOAT64, new ZstandardCompression()))); prepareOutputDatasets( n5out.get(), @@ -328,7 +324,7 @@ public static void run( } } - final Function attributes = dt -> new DatasetAttributes(outputDims, blockSize, dt, new GzipCompression()); + final Function attributes = dt -> new DatasetAttributes(outputDims, blockSize, dt, new ZstandardCompression()); LOG.debug("Saving relief"); // Save relief @@ -546,7 +542,7 @@ private static void relabel( final LongUnaryOperator idMapping) throws IOException { final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - final LongArrayDataBlock block = ((LongArrayDataBlock)n5.readBlock(dataset, attributes, blockPos)); + final DataBlock block = n5.readBlock(dataset, attributes, blockPos); final long[] data = block.getData(); for (int i = 0; i < data.length; ++i) { data[i] = idMapping.applyAsLong(data[i]); diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatershedsOnDistanceTransformOfSampledFunction.java b/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatershedsOnDistanceTransformOfSampledFunction.java index d25d286..5635b9f 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatershedsOnDistanceTransformOfSampledFunction.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatershedsOnDistanceTransformOfSampledFunction.java @@ -56,12 +56,8 @@ import org.apache.spark.api.java.function.PairFunction; import org.janelia.saalfeldlab.label.spark.N5Helpers; import org.janelia.saalfeldlab.label.spark.Version; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; -import org.janelia.saalfeldlab.n5.LongArrayDataBlock; -import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.*; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.slf4j.Logger; @@ -233,8 +229,8 @@ public static void run(final String... argv) throws IOException { attributes.put(OFFSET_KEY, offset); final Map datasets = new HashMap<>(); - Arrays.asList(uint64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT64, new GzipCompression()))); - Arrays.asList(float64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.FLOAT64, new GzipCompression()))); + Arrays.asList(uint64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT64, new ZstandardCompression()))); + Arrays.asList(float64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.FLOAT64, new ZstandardCompression()))); LOG.info("uint64 datasets: {}", (Object)uint64Datasets); LOG.info("float64 datasets: {}", (Object)float64Datasets); LOG.info("Datasets: {}", datasets); @@ -339,7 +335,7 @@ public static void run( } } - final Function attributes = dt -> new DatasetAttributes(outputDims, blockSize, dt, new GzipCompression()); + final Function attributes = dt -> new DatasetAttributes(outputDims, blockSize, dt, new ZstandardCompression()); LOG.debug("Saving relief"); // Save relief @@ -578,7 +574,7 @@ private static void relabel( final LongUnaryOperator idMapping) throws IOException { final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - final LongArrayDataBlock block = ((LongArrayDataBlock)n5.readBlock(dataset, attributes, blockPos)); + final DataBlock block = n5.readBlock(dataset, attributes, blockPos); final long[] data = block.getData(); for (int i = 0; i < data.length; ++i) { data[i] = idMapping.applyAsLong(data[i]); diff --git a/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatershedsOnDistanceTransformOfSampledFunctionSeedOnlyOnEdge.java b/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatershedsOnDistanceTransformOfSampledFunctionSeedOnlyOnEdge.java index 9f89b81..5082a65 100644 --- a/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatershedsOnDistanceTransformOfSampledFunctionSeedOnlyOnEdge.java +++ b/src/main/java/org/janelia/saalfeldlab/label/spark/watersheds/SparkWatershedsOnDistanceTransformOfSampledFunctionSeedOnlyOnEdge.java @@ -56,12 +56,8 @@ import org.apache.spark.api.java.function.PairFunction; import org.janelia.saalfeldlab.label.spark.N5Helpers; import org.janelia.saalfeldlab.label.spark.Version; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; -import org.janelia.saalfeldlab.n5.LongArrayDataBlock; -import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.*; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.slf4j.Logger; @@ -231,8 +227,8 @@ public static void run(final String... argv) throws IOException { attributes.put(OFFSET_KEY, offset); final Map datasets = new HashMap<>(); - Arrays.asList(uint64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT64, new GzipCompression()))); - Arrays.asList(float64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.FLOAT64, new GzipCompression()))); + Arrays.asList(uint64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.UINT64, new ZstandardCompression()))); + Arrays.asList(float64Datasets).forEach(ds -> datasets.put(ds, new DatasetAttributes(outputDims, args.blockSize, DataType.FLOAT64, new ZstandardCompression()))); LOG.info("uint64 datasets: {}", (Object)uint64Datasets); LOG.info("float64 datasets: {}", (Object)float64Datasets); LOG.info("Datasets: {}", datasets); @@ -363,7 +359,7 @@ public static void run( } } - final Function attributes = dt -> new DatasetAttributes(outputDims, blockSize, dt, new GzipCompression()); + final Function attributes = dt -> new DatasetAttributes(outputDims, blockSize, dt, new ZstandardCompression()); LOG.debug("Saving relief"); // Save relief @@ -602,7 +598,7 @@ private static void relabel( final LongUnaryOperator idMapping) throws IOException { final DatasetAttributes attributes = n5.getDatasetAttributes(dataset); - final LongArrayDataBlock block = ((LongArrayDataBlock)n5.readBlock(dataset, attributes, blockPos)); + final DataBlock block = n5.readBlock(dataset, attributes, blockPos); final long[] data = block.getData(); for (int i = 0; i < data.length; ++i) { data[i] = idMapping.applyAsLong(data[i]); diff --git a/src/test/java/org/janelia/saalfeldlab/label/spark/UniqueLabelsAndMappingTest.java b/src/test/java/org/janelia/saalfeldlab/label/spark/UniqueLabelsAndMappingTest.java index 6d532d5..5b12ca2 100644 --- a/src/test/java/org/janelia/saalfeldlab/label/spark/UniqueLabelsAndMappingTest.java +++ b/src/test/java/org/janelia/saalfeldlab/label/spark/UniqueLabelsAndMappingTest.java @@ -17,12 +17,7 @@ import org.janelia.saalfeldlab.label.spark.exception.InvalidN5Container; import org.janelia.saalfeldlab.label.spark.uniquelabels.ExtractUniqueLabelsPerBlock; import org.janelia.saalfeldlab.label.spark.uniquelabels.LabelToBlockMapping; -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.LongArrayDataBlock; -import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.*; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.junit.After; import org.junit.Assert; @@ -131,7 +126,7 @@ public void test() throws InvalidDataType, IOException, InvalidN5Container, Inva final long[] blockPosition = block.clone(); Arrays.setAll(blockPosition, d -> blockPosition[d] / blockSize[d]); - final LongArrayDataBlock blockData = ((LongArrayDataBlock)n5.readBlock(uniqueLabelDataset, uniqueLabelAttributes, blockPosition)); + final DataBlock blockData = n5.readBlock(uniqueLabelDataset, uniqueLabelAttributes, blockPosition); final long[] sortedContents = blockData.getData().clone(); Arrays.sort(sortedContents); diff --git a/src/test/java/org/janelia/saalfeldlab/label/spark/affinities/MakeEmptyMask.java b/src/test/java/org/janelia/saalfeldlab/label/spark/affinities/MakeEmptyMask.java index bf77452..f67de3b 100644 --- a/src/test/java/org/janelia/saalfeldlab/label/spark/affinities/MakeEmptyMask.java +++ b/src/test/java/org/janelia/saalfeldlab/label/spark/affinities/MakeEmptyMask.java @@ -8,9 +8,9 @@ import org.janelia.saalfeldlab.label.spark.N5Helpers; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.GzipCompression; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; +import org.janelia.scicomp.n5.zstandard.ZstandardCompression; import java.io.IOException; import java.util.Arrays; @@ -32,7 +32,7 @@ public static void main(String[] args) throws IOException, ExecutionException, I rawAttributes.getDimensions(), rawAttributes.getBlockSize(), DataType.UINT8, - new GzipCompression()); + new ZstandardCompression()); container.createDataset(maskPath, maskAttributes); container.setAttribute(maskPath, "value_range", new double[]{0.0, 1.0});