Skip to content

Commit fd22d3b

Browse files
cmhulbertcmhulbert
committed
feat: add --scale to to-scalar for pyramid
build: bump pom-scijava 45.0.0
1 parent 3e65b81 commit fd22d3b

5 files changed

Lines changed: 173 additions & 71 deletions

File tree

‎pom.xml‎

Lines changed: 2 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<parent>
44
<groupId>org.scijava</groupId>
55
<artifactId>pom-scijava</artifactId>
6-
<version>44.0.0</version>
6+
<version>45.0.0</version>
77
</parent>
88

99
<groupId>org.janelia.saalfeldlab</groupId>
@@ -43,26 +43,12 @@
4343
<kotlin.version>1.9.24</kotlin.version>
4444
<kotlin.compiler.incremental>true</kotlin.compiler.incremental>
4545

46-
<n5-universe.version>3.0.1</n5-universe.version>
47-
<n5.version>4.0.1</n5.version>
48-
<n5-aws-s3.version>5.0.0</n5-aws-s3.version>
49-
<n5-blosc.version>2.0.0</n5-blosc.version>
50-
<n5-google-cloud.version>6.0.0</n5-google-cloud.version>
51-
<n5-hdf5.version>3.0.0</n5-hdf5.version>
52-
<n5-imglib2.version>8.0.0</n5-imglib2.version>
53-
<n5-zarr.version>2.0.1</n5-zarr.version>
54-
<n5-zstandard.version>2.0.0</n5-zstandard.version>
55-
56-
57-
<imglib2-label-multisets.version>0.15.2</imglib2-label-multisets.version>
5846
<label-utilities-spark.version>1.2.0-SNAPSHOT</label-utilities-spark.version>
5947
<n5-spark.version>4.1.0-SNAPSHOT</n5-spark.version>
6048
<spark.version>3.5.4</spark.version>
6149
<spark-core_2.12.version>${spark.version}</spark-core_2.12.version>
6250
<jackson.version>2.15.4</jackson.version>
6351

64-
<junit-jupiter.version>5.11.2</junit-jupiter.version>
65-
6652
<!-- For JPackage -->
6753
<app.version>1.2.0</app.version>
6854
<app.name>paintera-convert</app.name>
@@ -238,19 +224,16 @@
238224
<dependency>
239225
<groupId>org.junit.jupiter</groupId>
240226
<artifactId>junit-jupiter-api</artifactId>
241-
<version>${junit-jupiter.version}</version>
242227
<scope>test</scope>
243228
</dependency>
244229
<dependency>
245230
<groupId>org.junit.jupiter</groupId>
246231
<artifactId>junit-jupiter-engine</artifactId>
247-
<version>${junit-jupiter.version}</version>
248232
<scope>test</scope>
249233
</dependency>
250234
<dependency>
251235
<groupId>org.junit.jupiter</groupId>
252236
<artifactId>junit-jupiter-params</artifactId>
253-
<version>${junit-jupiter.version}</version>
254237
<scope>test</scope>
255238
</dependency>
256239
<dependency>
@@ -312,23 +295,18 @@
312295
<artifactId>maven-dependency-plugin</artifactId>
313296
<executions>
314297
<execution>
315-
<!-- This copies the jar files declared by the Maven build into a single directory.
316-
This makes it easy to assemble the final installer package -->
317298
<id>copy-dependencies</id>
318299
<phase>package</phase>
319300
<goals>
320301
<goal>copy-dependencies</goal>
321302
</goals>
322303
<configuration>
323-
<!-- Include only compile stage dependencies (no test dependencies) -->
324-
<includeScope>compile</includeScope>
304+
<includeScope>runtime</includeScope>
325305
</configuration>
326306
</execution>
327307
</executions>
328308
</plugin>
329309
<plugin>
330-
<!-- Put the application jar generated by the project into the dependency folder - the default folder
331-
for the maven-dependency-plugin output. -->
332310
<groupId>org.apache.maven.plugins</groupId>
333311
<artifactId>maven-jar-plugin</artifactId>
334312
<configuration>
@@ -352,7 +330,6 @@
352330
<artifactId>jtoolprovider-plugin</artifactId>
353331
<version>1.0.34</version>
354332
<executions>
355-
<!-- This command runs jlink, generating a custom JVM by combining this project with the JDK modules. -->
356333
<execution>
357334
<id>jlink</id>
358335
<phase>package</phase>
@@ -372,20 +349,13 @@
372349
</args>
373350
</configuration>
374351
</execution>
375-
<!-- The command that actually generates the installer. The options for each platform
376-
are found the /src/packaging directory. These files are copied into the target directory by
377-
the Maven resources plugin, which does the filtering to replace the original values with
378-
the actual Maven properties.
379-
https://github.com/wiverson/jtoolprovider-plugin/blob/main/java-tool-doc.md
380-
-->
381352
<execution>
382353
<id>jpackage</id>
383354
<phase>install</phase>
384355
<goals>
385356
<goal>java-tool</goal>
386357
</goals>
387358
<configuration>
388-
<!--suppress UnresolvedMavenProperty -->
389359
<mainJar>${project.build.finalName}.jar</mainJar>
390360
<toolName>jpackage</toolName>
391361
<dest>${project.build.directory}/installer-${matrix.os}</dest>

‎src/main/kotlin/org/janelia/saalfeldlab/conversion/ExtractHighestResolutionLabelDataset.kt‎

Lines changed: 50 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,9 @@ import com.pivovarit.function.ThrowingConsumer
44
import gnu.trove.map.TLongLongMap
55
import gnu.trove.map.hash.TLongLongHashMap
66
import io.github.oshai.kotlinlogging.KotlinLogging
7-
import net.imglib2.FinalInterval
87
import net.imglib2.Interval
98
import net.imglib2.RandomAccessibleInterval
109
import net.imglib2.algorithm.util.Grids
11-
import net.imglib2.img.array.ArrayImgs
12-
import net.imglib2.loops.LoopBuilder
1310
import net.imglib2.algorithm.util.Singleton
1411
import net.imglib2.algorithm.util.Singleton.ThrowingSupplier
1512
import net.imglib2.converter.Converter
@@ -36,7 +33,9 @@ import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.OmeNgffMetadataPars
3633
import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter
3734
import org.janelia.saalfeldlab.n5.zarr.v3.ZarrV3DatasetAttributes
3835
import org.janelia.saalfeldlab.n5.zarr.v3.ZarrV3KeyValueWriter
36+
import org.janelia.saalfeldlab.n5.spark.downsample.N5LabelDownsamplerSpark
3937
import org.janelia.saalfeldlab.n5.spark.supplier.N5ReaderSupplier
38+
import org.janelia.saalfeldlab.n5.spark.util.ShardedBlocks
4039
import org.janelia.saalfeldlab.n5.spark.supplier.N5WriterSupplier
4140
import org.janelia.scicomp.n5.zstandard.ZstandardCompression
4241
import picocli.CommandLine
@@ -70,9 +69,11 @@ object ExtractHighestResolutionLabelDataset {
7069
considerFragmentSegmentAssignment: Boolean,
7170
assignment: TLongLongMap,
7271
xyzUnit: Array<String>,
73-
chunksPerShard: IntArray? = null
72+
chunksPerShard: IntArray? = null,
73+
scales: Array<IntArray> = emptyArray(),
74+
downsampleBlockSizes: Array<IntArray> = emptyArray()
7475
) {
75-
extract(sc, n5in, n5out, datasetIn, datasetOut, blockSizeOut, considerFragmentSegmentAssignment, assignment, xyzUnit, chunksPerShard)
76+
extract(sc, n5in, n5out, datasetIn, datasetOut, blockSizeOut, considerFragmentSegmentAssignment, assignment, xyzUnit, chunksPerShard, scales, downsampleBlockSizes)
7677
}
7778

7879
@JvmStatic
@@ -87,7 +88,9 @@ object ExtractHighestResolutionLabelDataset {
8788
considerFragmentSegmentAssignment: Boolean,
8889
assignment: TLongLongMap,
8990
xyzUnit: Array<String>,
90-
chunksPerShard: IntArray? = null
91+
chunksPerShard: IntArray? = null,
92+
scales: Array<IntArray> = emptyArray(),
93+
downsampleBlockSizes: Array<IntArray> = emptyArray()
9194
) where IN : NativeType<IN>?, IN : IntegerType<IN>? {
9295
extract<IN, UnsignedLongType>(
9396
sc,
@@ -103,7 +106,9 @@ object ExtractHighestResolutionLabelDataset {
103106
considerFragmentSegmentAssignment,
104107
assignment,
105108
xyzUnit,
106-
chunksPerShard
109+
chunksPerShard,
110+
scales,
111+
downsampleBlockSizes
107112
)
108113
}
109114

@@ -120,7 +125,9 @@ object ExtractHighestResolutionLabelDataset {
120125
considerFragmentSegmentAssignment: Boolean,
121126
assignment: TLongLongMap,
122127
xyzUnit: Array<String>,
123-
chunksPerShard: IntArray? = null
128+
chunksPerShard: IntArray? = null,
129+
scales: Array<IntArray> = emptyArray(),
130+
downsampleBlockSizes: Array<IntArray> = emptyArray()
124131
) where IN : NativeType<IN>?, IN : IntegerType<IN>?, OUT : NativeType<OUT>?, OUT : IntegerType<OUT>? {
125132
val n5InLocal = n5in.get()
126133
if (!n5InLocal.exists(datasetIn)) {
@@ -140,7 +147,7 @@ object ExtractHighestResolutionLabelDataset {
140147
}
141148
extract(
142149
sc, n5in, n5out, "$datasetIn/data", datasetOut, blockSizeOut, outputTypeSupplier, updatedAdditionalEntries,
143-
considerFragmentSegmentAssignment, assignment, xyzUnit, chunksPerShard
150+
considerFragmentSegmentAssignment, assignment, xyzUnit, chunksPerShard, scales, downsampleBlockSizes
144151
)
145152
return
146153
} catch (e: NoValidDatasetException) {
@@ -150,7 +157,7 @@ object ExtractHighestResolutionLabelDataset {
150157
try {
151158
extract(
152159
sc, n5in, n5out, "$datasetIn/s0", datasetOut, blockSizeOut, outputTypeSupplier, additionalAttributes,
153-
considerFragmentSegmentAssignment, assignment, xyzUnit, chunksPerShard
160+
considerFragmentSegmentAssignment, assignment, xyzUnit, chunksPerShard, scales, downsampleBlockSizes
154161
)
155162
return
156163
} catch (e: NoValidDatasetException) {
@@ -186,24 +193,33 @@ object ExtractHighestResolutionLabelDataset {
186193
val keys = assignment.keys()
187194
val values = assignment.values()
188195

189-
val resolution = n5InLocal.getAttribute(datasetIn, "resolution", DoubleArray::class.java)
190-
val offset = n5InLocal.getAttribute(datasetIn, "offset", DoubleArray::class.java)
191-
/* resolution/offset become the OME-NGFF scale/translation transforms on the multiscale group */
196+
val resolution = n5InLocal.getAttribute(datasetIn, "resolution", DoubleArray::class.java) ?: DoubleArray(dimensions.size) { 1.0 }
197+
val offset = n5InLocal.getAttribute(datasetIn, "offset", DoubleArray::class.java) ?: DoubleArray(dimensions.size) { 0.0 }
198+
/* resolution/offset become the OME-NGFF scale/translation transforms; one dataset per pyramid level, with the
199+
* per-level scale accumulating the relative downsampling factors */
192200
val axes = arrayOf(
193201
Axis(Axis.SPACE, "x", xyzUnit[0], false),
194202
Axis(Axis.SPACE, "y", xyzUnit[1], false),
195203
Axis(Axis.SPACE, "z", xyzUnit[2], false)
196204
)
205+
val scalePaths = Array(scales.size + 1) { "s$it" }
206+
val levelResolutions = arrayListOf(resolution.copyOf())
207+
val levelTranslations = arrayListOf(offset.copyOf())
208+
val cumulativeFactor = DoubleArray(dimensions.size) { 1.0 }
209+
scales.forEach { factor ->
210+
for (d in cumulativeFactor.indices) cumulativeFactor[d] = cumulativeFactor[d] * factor[d]
211+
levelResolutions.add(DoubleArray(dimensions.size) { resolution[it] * cumulativeFactor[it] })
212+
levelTranslations.add(offset.copyOf())
213+
}
197214
val metadata = OmeNgffMetadata.buildForWriting(
198215
dimensions.size,
199216
datasetOut,
200217
ngffVersion,
201218
axes,
202-
arrayOf("s0"),
203-
arrayOf(resolution ?: DoubleArray(dimensions.size) { 1.0 }),
204-
arrayOf(offset ?: DoubleArray(dimensions.size) { 0.0 })
219+
scalePaths,
220+
levelResolutions.toTypedArray(),
221+
levelTranslations.toTypedArray()
205222
)
206-
OmeNgffMetadataParser(outWriter).writeMetadata(metadata, outWriter, datasetOut)
207223

208224
n5InLocal.getAttribute(datasetIn, "maxId", Long::class.javaPrimitiveType)?.let { maxId ->
209225
outWriter.setAttribute(dataDataset, "maxId", maxId)
@@ -267,29 +283,11 @@ object ExtractHighestResolutionLabelDataset {
267283
val attributes = writer.getDatasetAttributes(dataDataset)
268284

269285
val fillValue = (attributes as? ZarrV3DatasetAttributes)?.run { ByteBuffer.wrap(fillBytes).long } ?: 0L
270-
var blockIsEmpty = true
271286
if (attributes.isSharded) {
272-
/* materialize a full shard-sized block (0-padded past the edge) and write it at the shard grid position.*/
273-
val shardBlockSize = attributes.blockSize
274-
val data = LongArray(shardBlockSize.fold(1) { acc, s -> acc * s })
275-
val shardImg = ArrayImgs.unsignedLongs(data, *shardBlockSize.map { it.toLong() }.toLongArray())
276-
LoopBuilder.setImages(
277-
Views.zeroMin(converted),
278-
Views.interval(shardImg, FinalInterval(*Intervals.dimensionsAsLongArray(converted)))
279-
).forEachPixel { source, target ->
280-
val value = source!!.integerLong
281-
target.setInteger(value)
282-
/* set blockIsEmpty = false first value that is not the fill value. */
283-
if (blockIsEmpty && value != fillValue)
284-
blockIsEmpty = false
285-
}
286-
if (!blockIsEmpty)
287-
writer.writeBlock(
288-
dataDataset,
289-
attributes,
290-
LongArrayDataBlock(shardBlockSize, blockWithPosition._2(), data)
291-
)
287+
val fillType = outputTypeSupplier.get()!!.also { it.setInteger(fillValue) }
288+
ShardedBlocks.saveNonEmptyShard(writer, dataDataset, attributes, blockWithPosition._2(), converted, fillType)
292289
} else {
290+
var blockIsEmpty = true
293291
val blockDims = Intervals.dimensionsAsIntArray(converted)
294292
val data = LongArray(Intervals.numElements(converted).toInt())
295293
val cursor = Views.flatIterable(converted).cursor()
@@ -308,6 +306,20 @@ object ExtractHighestResolutionLabelDataset {
308306
)
309307
}
310308
}
309+
310+
/* build the downsampled pyramid s1..sN from s0; each level is sharded when chunksPerShard is set */
311+
var previousLevel = dataDataset
312+
scales.forEachIndexed { level, factor ->
313+
val nextLevel = "$datasetOut/s${level + 1}"
314+
N5LabelDownsamplerSpark.downsampleLabel<UnsignedLongType>(
315+
sc, n5out, previousLevel, nextLevel, factor,
316+
downsampleBlockSizes.getOrElse(level) { blockSize }, chunksPerShard, false
317+
)
318+
previousLevel = nextLevel
319+
}
320+
321+
/* write the multiscale metadata once every level exists */
322+
OmeNgffMetadataParser(outWriter).writeMetadata(metadata, outWriter, datasetOut)
311323
}
312324

313325
private fun <IN : IntegerType<IN>?, OUT : IntegerType<OUT>?> getAppropriateConverter(map: TLongLongMap?): Converter<IN, OUT> {

‎src/main/kotlin/org/janelia/saalfeldlab/conversion/to/scalar/ToScalar.kt‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import gnu.trove.map.hash.TLongLongHashMap
55
import org.apache.spark.api.java.JavaSparkContext
66
import org.janelia.saalfeldlab.conversion.*
77
import org.janelia.saalfeldlab.conversion.to.newSparkConf
8+
import org.janelia.saalfeldlab.conversion.to.paintera.SpatialIntArray
89
import org.janelia.saalfeldlab.n5.universe.StorageFormat
910
import picocli.CommandLine
1011
import java.io.IOException
@@ -69,6 +70,28 @@ class ToScalar : Callable<Int> {
6970
)
7071
private var chunksPerShard: IntArray? = null
7172

73+
@CommandLine.Option(
74+
names = ["--scale"],
75+
arity = "1..*",
76+
split = "\\s",
77+
converter = [SpatialIntArray.Converter::class],
78+
paramLabel = SpatialIntArray.PARAM_LABEL,
79+
description = [
80+
"Relative downsampling factors for each level in the format x,y,z, where x,y,z are integers. Single integers u are interpreted as u,u,u.",
81+
]
82+
)
83+
private var _scales: Array<SpatialIntArray>? = null
84+
85+
@CommandLine.Option(
86+
names = ["--downsample-block-sizes"],
87+
arity = "1..*",
88+
split = "\\s",
89+
converter = [SpatialIntArray.Converter::class],
90+
paramLabel = SpatialIntArray.PARAM_LABEL,
91+
description = ["Output block size per downsampled level; defaults to --block-size for every level."]
92+
)
93+
private var _downsampleBlockSizes: Array<SpatialIntArray>? = null
94+
7295
@CommandLine.Option(
7396
names = ["--xyz-unit"],
7497
required = true,
@@ -148,6 +171,11 @@ class ToScalar : Callable<Int> {
148171
}
149172
}
150173

174+
val scales = _scales?.map { it.array }?.toTypedArray() ?: emptyArray()
175+
val downsampleBlockSizes = _downsampleBlockSizes?.map { it.array }?.toTypedArray()
176+
?.also { if (it.size != scales.size) throw InvalidBlockSize(blockSize, "--downsample-block-sizes must have one entry per --scale level (${scales.size}), but got ${it.size}") }
177+
?: Array(scales.size) { blockSize }
178+
151179
extract(
152180
inputContainer,
153181
outputFormat,
@@ -157,6 +185,8 @@ class ToScalar : Callable<Int> {
157185
blockSize,
158186
chunksPerShard,
159187
xyzUnit,
188+
scales,
189+
downsampleBlockSizes,
160190
considerFragmentSegmentAssignment,
161191
assignment,
162192
sparkMaster
@@ -181,6 +211,8 @@ class ToScalar : Callable<Int> {
181211
blockSize: IntArray,
182212
chunksPerShard: IntArray?,
183213
xyzUnit: Array<String>,
214+
scales: Array<IntArray>,
215+
downsampleBlockSizes: Array<IntArray>,
184216
considerFragmentSegmentAssignment: Boolean,
185217
assignment: TLongLongMap,
186218
sparkMaster: String?
@@ -199,7 +231,9 @@ class ToScalar : Callable<Int> {
199231
considerFragmentSegmentAssignment,
200232
assignment,
201233
xyzUnit,
202-
chunksPerShard
234+
chunksPerShard,
235+
scales,
236+
downsampleBlockSizes
203237
)
204238
}
205239

0 commit comments

Comments
 (0)