Skip to content

Commit 7eff156

Browse files
cmhulbertcmhulbert
authored
Merge pull request #44 from saalfeldlab/migrate-n5-4.0
Migrate n5 4.0
2 parents a9352e1 + 24f2603 commit 7eff156

11 files changed

Lines changed: 436 additions & 190 deletions

File tree

‎pom.xml‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,13 @@
55
<parent>
66
<groupId>org.scijava</groupId>
77
<artifactId>pom-scijava</artifactId>
8-
<version>37.0.0</version>
8+
<version>45.0.0</version>
99
<relativePath />
1010
</parent>
1111

1212
<groupId>org.janelia.saalfeldlab</groupId>
1313
<artifactId>n5-spark</artifactId>
14-
<version>4.0.1-SNAPSHOT</version>
14+
<version>4.1.0-SNAPSHOT</version>
1515

1616
<name>N5 Spark</name>
1717
<description>N5 Spark-based processing tools</description>
@@ -103,12 +103,6 @@
103103

104104
<!-- NB: Deploy releases to the SciJava Maven repository. -->
105105
<releaseProfiles>sign,deploy-to-scijava</releaseProfiles>
106-
<args4j.version>2.33</args4j.version>
107-
<n5.version>3.2.0</n5.version>
108-
<n5-imglib2.version>7.0.0</n5-imglib2.version>
109-
<n5-ij.version>4.1.1</n5-ij.version>
110-
<bigdataviewer-core.version>10.4.14</bigdataviewer-core.version>
111-
112106
<kryo.version>4.0.2</kryo.version>
113107
<alphanumeric-comparator.version>1.4.1</alphanumeric-comparator.version>
114108
<spark.version>3.5.4</spark.version>
@@ -125,7 +119,6 @@
125119
<dependency>
126120
<groupId>args4j</groupId>
127121
<artifactId>args4j</artifactId>
128-
<version>${args4j.version}</version>
129122
</dependency>
130123
<dependency>
131124
<groupId>net.imglib2</groupId>
@@ -146,22 +139,27 @@
146139
<dependency>
147140
<groupId>sc.fiji</groupId>
148141
<artifactId>bigdataviewer-core</artifactId>
149-
<version>${bigdataviewer-core.version}</version>
150142
</dependency>
151143
<dependency>
152144
<groupId>org.janelia.saalfeldlab</groupId>
153145
<artifactId>n5</artifactId>
154-
<version>${n5.version}</version>
155146
</dependency>
156147
<dependency>
157148
<groupId>org.janelia.saalfeldlab</groupId>
158149
<artifactId>n5-imglib2</artifactId>
159-
<version>${n5-imglib2.version}</version>
150+
</dependency>
151+
<!-- pin the v3 (sharding) zarr stack, overriding the older versions pulled transitively by n5-ij -->
152+
<dependency>
153+
<groupId>org.janelia.saalfeldlab</groupId>
154+
<artifactId>n5-zarr</artifactId>
155+
</dependency>
156+
<dependency>
157+
<groupId>org.janelia.saalfeldlab</groupId>
158+
<artifactId>n5-universe</artifactId>
160159
</dependency>
161160
<dependency>
162161
<groupId>org.janelia.saalfeldlab</groupId>
163162
<artifactId>n5-ij</artifactId>
164-
<version>${n5-ij.version}</version>
165163
<exclusions>
166164
<exclusion>
167165
<groupId>net.imagej</groupId>

‎src/main/java/org/janelia/saalfeldlab/n5/spark/N5MaxIntensityProjectionSpark.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import org.janelia.saalfeldlab.n5.DatasetAttributes;
5656
import org.janelia.saalfeldlab.n5.N5FSReader;
5757
import org.janelia.saalfeldlab.n5.N5Reader;
58+
import org.janelia.saalfeldlab.n5.imglib2.N5Utils;
5859
import org.janelia.saalfeldlab.n5.spark.supplier.N5ReaderSupplier;
5960
import org.janelia.saalfeldlab.n5.spark.util.CmdUtils;
6061
import org.janelia.saalfeldlab.n5.spark.util.N5SparkUtils;
@@ -219,7 +220,7 @@ public static <T extends NativeType<T> & RealType<T>> void createMaxIntensityPro
219220
new BasicNameValuePair("dataset", datasetPath),
220221
new BasicNameValuePair("call", "create-max-intensity-projection")
221222
).toString();
222-
final CachedCellImg<T, ?> cellImg = Singleton.get(readerCacheKey, () -> N5SparkUtils.openWithLoaderCache(n5, datasetPath, new SoftRefLoaderCache<>()));
223+
final CachedCellImg<T, ?> cellImg = Singleton.get(readerCacheKey, () -> (CachedCellImg<T, ?>)N5Utils.open(n5, datasetPath));
223224
final long[] cellGridDimensions = cellImg.getCellGrid().getGridDimensions();
224225
numCells = Intervals.numElements(cellGridDimensions);
225226
type = Util.getTypeFromInterval(cellImg).createVariable();
@@ -249,7 +250,7 @@ public static <T extends NativeType<T> & RealType<T>> void createMaxIntensityPro
249250
new BasicNameValuePair("dataset", datasetPath),
250251
new BasicNameValuePair("call", "create-max-intensity-projection")
251252
).toString();
252-
final CachedCellImg<T, ?> cellImg = Singleton.get(readerCacheKey, () -> N5SparkUtils.openWithLoaderCache(n5, datasetPath, new SoftRefLoaderCache<>()));
253+
final CachedCellImg<T, ?> cellImg = Singleton.get(readerCacheKey, () -> (CachedCellImg<T, ?>)N5Utils.open(n5, datasetPath));
253254
final RandomAccess<T> cellImgRandomAccess = cellImg.randomAccess();
254255

255256
final long[] cellMin = new long[dim], cellMax = new long[dim];

‎src/main/java/org/janelia/saalfeldlab/n5/spark/N5ToSliceTiffSpark.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import org.janelia.saalfeldlab.n5.DatasetAttributes;
5959
import org.janelia.saalfeldlab.n5.N5FSReader;
6060
import org.janelia.saalfeldlab.n5.N5Reader;
61+
import org.janelia.saalfeldlab.n5.imglib2.N5Utils;
6162
import org.janelia.saalfeldlab.n5.spark.supplier.N5ReaderSupplier;
6263
import org.janelia.saalfeldlab.n5.spark.util.N5SparkUtils;
6364
import org.janelia.saalfeldlab.n5.spark.util.SliceDimension;
@@ -261,7 +262,7 @@ public static < T extends NativeType< T > & RealType< T >, U extends NativeType<
261262
new BasicNameValuePair("dataset", datasetPath),
262263
new BasicNameValuePair("call", "create-max-intensity-projection")
263264
).toString();
264-
final CachedCellImg<T, ?> cellImg = Singleton.get(readerCacheKey, () -> N5SparkUtils.openWithLoaderCache(n5, datasetPath, new SoftRefLoaderCache<>()));
265+
final CachedCellImg<T, ?> cellImg = Singleton.get(readerCacheKey, () -> (CachedCellImg<T, ?>)N5Utils.open(n5, datasetPath));
265266

266267
final CellGrid cellGrid = cellImg.getCellGrid();
267268
final long[] slicePos = new long[ cellImg.numDimensions() ], cellPos = new long[ cellImg.numDimensions() ];

‎src/main/java/org/janelia/saalfeldlab/n5/spark/SliceTiffToN5Spark.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ public static < T extends NativeType< T > > void convert(
223223
new BasicNameValuePair("call", "slice-tiff-to-n5-spark")
224224
).toString();
225225

226-
final RandomAccessibleInterval< T > tmpImg = Singleton.get(tmpReaderCacheKey, () -> N5Utils.open( n5Local, tmpDataset ));
226+
final RandomAccessibleInterval< T > tmpImg = Singleton.get(tmpReaderCacheKey, () -> (RandomAccessibleInterval< T > )N5Utils.open( n5Local, tmpDataset ));
227227

228228
final Interval interval = N5SparkUtils.toInterval( minMaxTuple );
229229
final RandomAccessibleInterval< T > tmpImgCrop = Views.offsetInterval( tmpImg, interval );

‎src/main/java/org/janelia/saalfeldlab/n5/spark/downsample/N5DownsamplerSpark.java‎

Lines changed: 74 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
*/
2929
package org.janelia.saalfeldlab.n5.spark.downsample;
3030

31-
import bdv.export.Downsample;
31+
import org.janelia.saalfeldlab.n5.spark.util.Downsample;
3232
import net.imglib2.FinalInterval;
3333
import net.imglib2.Interval;
3434
import net.imglib2.RandomAccessibleInterval;
@@ -47,6 +47,8 @@
4747
import org.janelia.saalfeldlab.n5.DatasetAttributes;
4848
import org.janelia.saalfeldlab.n5.N5FSWriter;
4949
import org.janelia.saalfeldlab.n5.N5Writer;
50+
import org.janelia.saalfeldlab.n5.zarr.v3.ZarrV3DatasetAttributes;
51+
import org.janelia.saalfeldlab.n5.zarr.v3.ZarrV3KeyValueWriter;
5052
import org.janelia.saalfeldlab.n5.imglib2.N5Utils;
5153
import org.janelia.saalfeldlab.n5.spark.supplier.N5WriterSupplier;
5254
import org.janelia.saalfeldlab.n5.spark.util.CmdUtils;
@@ -176,6 +178,42 @@ public static <T extends NativeType<T> & RealType<T>> void downsample(
176178
final int[] blockSize,
177179
final boolean overwriteExisting) throws IOException {
178180

181+
downsample(
182+
sparkContext,
183+
n5Supplier,
184+
inputDatasetPath,
185+
outputDatasetPath,
186+
downsamplingFactors,
187+
blockSize,
188+
null,
189+
overwriteExisting);
190+
}
191+
192+
/**
193+
* Downsamples the given input dataset with respect to the given downsampling factors, storing the output blocks in
194+
* shards of {@code chunksPerShard} chunks per axis. A {@code null} {@code chunksPerShard} writes an unsharded dataset
195+
* (identical to the other overloads). Sharding requires a zarr3 ({@link ZarrV3KeyValueWriter}) writer.
196+
*
197+
* @param sparkContext
198+
* @param n5Supplier
199+
* @param inputDatasetPath
200+
* @param outputDatasetPath
201+
* @param downsamplingFactors
202+
* @param blockSize
203+
* @param chunksPerShard number of chunks per shard per axis, or {@code null} for an unsharded output
204+
* @param overwriteExisting
205+
* @throws IOException
206+
*/
207+
public static <T extends NativeType<T> & RealType<T>> void downsample(
208+
final JavaSparkContext sparkContext,
209+
final N5WriterSupplier n5Supplier,
210+
final String inputDatasetPath,
211+
final String outputDatasetPath,
212+
final int[] downsamplingFactors,
213+
final int[] blockSize,
214+
final int[] chunksPerShard,
215+
final boolean overwriteExisting) throws IOException {
216+
179217
final N5Writer n5 = n5Supplier.get();
180218
if (!n5.datasetExists(inputDatasetPath))
181219
throw new IllegalArgumentException("Input N5 dataset " + inputDatasetPath + " does not exist");
@@ -206,13 +244,26 @@ public static <T extends NativeType<T> & RealType<T>> void downsample(
206244
}
207245
}
208246

209-
n5.createDataset(
210-
outputDatasetPath,
211-
outputDimensions,
212-
outputBlockSize,
213-
inputAttributes.getDataType(),
214-
inputAttributes.getCompression()
215-
);
247+
if (chunksPerShard != null) {
248+
if (!(n5 instanceof ZarrV3KeyValueWriter))
249+
throw new IllegalArgumentException("Sharded downsampling requires a ZarrV3 writer");
250+
final int[] shardSize = new int[dim];
251+
for (int d = 0; d < dim; ++d)
252+
shardSize[d] = chunksPerShard[d] * outputBlockSize[d];
253+
254+
n5.createDataset(
255+
outputDatasetPath,
256+
new ZarrV3DatasetAttributes(outputDimensions, shardSize, outputBlockSize, inputAttributes.getDataType(), inputAttributes.getCompression())
257+
);
258+
} else {
259+
n5.createDataset(
260+
outputDatasetPath,
261+
outputDimensions,
262+
outputBlockSize,
263+
inputAttributes.getDataType(),
264+
inputAttributes.getCompression()
265+
);
266+
}
216267

217268
// set the downsampling factors attribute
218269
final int[] inputAbsoluteDownsamplingFactors = n5.getAttribute(inputDatasetPath, DOWNSAMPLING_FACTORS_ATTRIBUTE_KEY, int[].class);
@@ -221,14 +272,17 @@ public static <T extends NativeType<T> & RealType<T>> void downsample(
221272
outputAbsoluteDownsamplingFactors[d] = downsamplingFactors[d] * (inputAbsoluteDownsamplingFactors != null ? inputAbsoluteDownsamplingFactors[d] : 1);
222273
n5.setAttribute(outputDatasetPath, DOWNSAMPLING_FACTORS_ATTRIBUTE_KEY, outputAbsoluteDownsamplingFactors);
223274

224-
final CellGrid outputCellGrid = new CellGrid(outputDimensions, outputBlockSize);
275+
/* the parallel-write unit is DatasetAttributes#getBlockSize: the shard for a sharded dataset, the chunk otherwise */
276+
final int[] writeBlockSize = n5.getDatasetAttributes(outputDatasetPath).getBlockSize();
277+
278+
final CellGrid outputCellGrid = new CellGrid(outputDimensions, writeBlockSize);
225279
final long numDownsampledBlocks = Intervals.numElements(outputCellGrid.getGridDimensions());
226280
final List<Long> blockIndexes = LongStream.range(0, numDownsampledBlocks).boxed().collect(Collectors.toList());
227281

228282
sparkContext
229283
.parallelize(blockIndexes, Math.min(blockIndexes.size(), MAX_PARTITIONS))
230284
.foreach(blockIndex -> {
231-
final CellGrid cellGrid = new CellGrid(outputDimensions, outputBlockSize);
285+
final CellGrid cellGrid = new CellGrid(outputDimensions, writeBlockSize);
232286
final long[] blockGridPosition = new long[cellGrid.numDimensions()];
233287
cellGrid.getCellGridPositionFlat(blockIndex, blockGridPosition);
234288

@@ -261,7 +315,7 @@ public static <T extends NativeType<T> & RealType<T>> void downsample(
261315
new BasicNameValuePair("call", "n5-downsample-spark")
262316
).toString();
263317

264-
final RandomAccessibleInterval< T > source = Singleton.get(imgCacheKey, () -> N5Utils.open( n5Writer, inputDatasetPath ));
318+
final RandomAccessibleInterval< T > source = Singleton.get(imgCacheKey, () -> (RandomAccessibleInterval< T > )N5Utils.open( n5Writer, inputDatasetPath ));
265319
final RandomAccessibleInterval<T> sourceBlock = Views.offsetInterval(source, sourceInterval);
266320

267321
/* test if empty */
@@ -279,12 +333,19 @@ public static <T extends NativeType<T> & RealType<T>> void downsample(
279333
final RandomAccessibleInterval<T> targetBlock = new ArrayImgFactory<>(defaultValue).create(targetInterval);
280334
Downsample.downsample(sourceBlock, targetBlock, downsamplingFactors);
281335

336+
final DatasetAttributes outputAttributes = n5Writer.getDatasetAttributes(outputDatasetPath);
337+
final int[] shardBlockSize = outputAttributes.getBlockSize();
338+
final int[] innerChunkSize = outputAttributes.getChunkSize();
339+
final long[] chunkGridOffset = new long[dim];
340+
for (int d = 0; d < dim; ++d)
341+
chunkGridOffset[d] = blockGridPosition[d] * (shardBlockSize[d] / innerChunkSize[d]);
342+
282343
if (overwriteExisting) {
283344
// Empty blocks will not be written out. Delete blocks to avoid remnant blocks if overwriting.
284-
N5Utils.deleteBlock(targetBlock, n5Writer, outputDatasetPath, blockGridPosition);
345+
N5Utils.deleteBlock(targetBlock, n5Writer, outputDatasetPath, chunkGridOffset);
285346
}
286347

287-
N5Utils.saveNonEmptyBlock(targetBlock, n5Writer, outputDatasetPath, blockGridPosition, defaultValue);
348+
N5Utils.saveNonEmptyBlock(targetBlock, n5Writer, outputDatasetPath, outputAttributes, chunkGridOffset, defaultValue);
288349
});
289350
}
290351

0 commit comments

Comments
 (0)