Merge pull request #63 from saalfeldlab/1.3.1

1.3.1

Author: cmhulbert

.fileglancer/runnables.yaml

@@ -69,25 +69,31 @@ runnables:
         name: Scale factors
         type: string
         required: true
-        default: "2,2,2 2,2,2"
-        description: >-
-          Relative downsampling factor per level as whitespace-separated x,y,z triples
-          (a single integer u means u,u,u). Example: `2,2,1 2,2,1 2`.
+        default: "2 2"
+        description: |
+          Relative downsampling factor per level as whitespace-separated x,y,z triples (a single integer u means u,u,u). 
+            Example: `2,2,1 2,2,1 2`.
       - flag: --block-size
         name: Block size
         type: string
-        default: "64,64,64"
+        default: "32,32,32"
       - flag: --reverse-array-attributes
         name: Reverse array attributes
+        hidden: true
         type: boolean
         default: false
-        description: "Reverse resolution/offset arrays, i.e. [x,y,z] -> [z,y,x]. NOTE: N5-Zarr interop generally handles this automatically, especially for Zarr3. Only Reverse if you are sure it's necessary."
+        description: |
+          Reverse resolution/offset arrays, i.e. [x,y,z] -> [z,y,x].
+          NOTE: N5-Zarr interop generally handles this automatically, especially for Zarr3.
+          Only Reverse if you are sure it's necessary.
       - flag: --winner-takes-all-downsampling
         name: Winner-takes-all downsampling
         hidden: true
         type: boolean
         default: false
-        description: "Use a scalar label type with winner-takes-all downsampling. If 'false' or omitted, data will be LabelMultisetType"
+        description: |
+          In most cases this should be left "false" or omitted, which defaults the conversion to LabelMultisetType, which is useful for Paintera.
+          If true, uses a scalar label type with winner-takes-all downsampling.
       - flag: --overwrite-existing
         name: Overwrite existing
         hidden: true
@@ -145,12 +151,13 @@ runnables:
           - ZARR3
         required: false
         default: ZARR
-        description: >-
+        description: |
           Output storage format of the generated dataset. Explicit support for N5, ZARR2, and ZARR3.
           ZARR without the version will default to ZARR3 if no container exists, but if a container
-          exists, it will keep the version of the container. All zarr containers contain OME-Zarr
-          metadata, and ZARR3 supports sharding. If left unset, the format is taken from the output
-          container's path, else inferred on write.
+          exists, it will keep the version of the container. 
+          All zarr containers contain OME-Zarr metadata.
+          ZARR3 supports sharding.
+          If not provided a best-attempt is made to guess the format from the path, existing contain, or lastly a fallback to ZARR3.
       - flag: --scale
         name: Scale factors
         type: string

src/main/kotlin/org/janelia/saalfeldlab/conversion/DatasetInfo.kt

@@ -14,10 +14,16 @@ data class DatasetInfo(
 	val inputContainer: String,
 	val inputDataset: String,
 	val outputContainer: URI,
-	val outputFormat: StorageFormat? = null,
+	/* Paintera datasets are currently N5-only; the label multiset type and the adjacent index dataset are varlen */
+	val outputFormat: StorageFormat = StorageFormat.N5,
 	val outputGroup: String = inputDataset
 ) : Serializable {
 
+	/* label-utilities-spark opens containers from a plain string; qualify it with the
+	 * storage scheme so it cannot guess a different format than the one we write with */
+	val outputContainerUri: String
+		get() = "${outputFormat.name.lowercase()}:$outputContainer"
+
 	val type: String
 		get() {
 			val attrs = attributes

src/main/kotlin/org/janelia/saalfeldlab/conversion/to/paintera/DatasetConverterLabel.kt

@@ -141,8 +141,8 @@ private fun handleLabelDataset(
 
 		val maxId = ExtractUniqueLabelsPerBlock.extractUniqueLabels(
 			sc,
-			info.outputContainer.toString(),
-			info.outputContainer.toString(),
+			info.outputContainerUri,
+			info.outputContainerUri,
 			originalResolutionOutputDataset,
 			Paths.get(uniqueLabelsGroup, "s0").toString()
 		)
@@ -153,7 +153,7 @@ private fun handleLabelDataset(
 		if (scales.isNotEmpty())
 		// TODO refactor this to be nicer
 		{
-			LabelListDownsampler.donwsampleMultiscale(sc, info.outputContainer.toString(), uniqueLabelsGroup, scales, downsampleBlockSizes)
+			LabelListDownsampler.donwsampleMultiscale(sc, info.outputContainerUri, uniqueLabelsGroup, scales, downsampleBlockSizes)
 		}
 	} else {
 		// TODO pass compression and reverse array as parameters
@@ -167,7 +167,7 @@ private fun handleLabelDataset(
 				slicedSupplier,
 				initialBlockSize,
 				initialBlockSize,
-				info.outputContainer.toString(),
+				info.outputContainerUri,
 				originalResolutionOutputDataset,
 				ZstandardCompression()
 			)
@@ -177,7 +177,7 @@ private fun handleLabelDataset(
 				info.inputContainer,
 				info.inputDataset,
 				initialBlockSize,
-				info.outputContainer.toString(),
+				info.outputContainerUri,
 				originalResolutionOutputDataset,
 				ZstandardCompression(),
 				reverse
@@ -189,33 +189,33 @@ private fun handleLabelDataset(
 
 		ExtractUniqueLabelsPerBlock.extractUniqueLabels(
 			sc,
-			info.outputContainer.toString(),
-			info.outputContainer.toString(),
+			info.outputContainerUri,
+			info.outputContainerUri,
 			originalResolutionOutputDataset,
 			"$uniqueLabelsGroup/s0"
 		)
 		LabelListDownsampler.addMultiScaleTag(writer, uniqueLabelsGroup)
 
 		if (scales.isNotEmpty()) {
 			// TODO pass compression as parameter
-			SparkDownsampler.downsampleMultiscale(sc, info.outputContainer.toString(), dataGroup, scales, downsampleBlockSizes, maxNumEntriesArray, ZstandardCompression())
-			LabelListDownsampler.donwsampleMultiscale(sc, info.outputContainer.toString(), uniqueLabelsGroup, scales, downsampleBlockSizes)
+			SparkDownsampler.downsampleMultiscale(sc, info.outputContainerUri, dataGroup, scales, downsampleBlockSizes, maxNumEntriesArray, ZstandardCompression())
+			LabelListDownsampler.donwsampleMultiscale(sc, info.outputContainerUri, uniqueLabelsGroup, scales, downsampleBlockSizes)
 		}
 	}
 
 	if (labelBlockLookupN5BlockSize != null) {
 		LabelToBlockMapping.createMappingWithMultiscaleCheckN5(
 			sc,
-			info.outputContainer.toString(),
+			info.outputContainerUri,
 			uniqueLabelsGroup,
-			info.outputContainer.toString(),
+			info.outputContainerUri,
 			info.outputGroup,
 			labelBlockMappingGroupBasename,
 			labelBlockLookupN5BlockSize
 		)
 
 	} else {
-		LabelToBlockMapping.createMappingWithMultiscaleCheck(sc, info.outputContainer.toString(), uniqueLabelsGroup, labelBlockMappingGroupDirectory)
+		LabelToBlockMapping.createMappingWithMultiscaleCheck(sc, info.outputContainerUri, uniqueLabelsGroup, labelBlockMappingGroupDirectory)
 	}
 	/* write the group-level lookup metadata */
 	writer.setAttribute(info.outputGroup, LABEL_BLOCK_LOOKUP_KEY, LabelBlockLookupFromN5Relative("$labelBlockMappingGroupBasename/s%d"))

src/main/kotlin/org/janelia/saalfeldlab/conversion/to/paintera/ToPainteraData.kt

@@ -75,12 +75,19 @@ class ToPainteraData {
 		@CommandLine.Option(names = ["--output-format"], required = false, defaultValue = "", paramLabel = "OUTPUT_FORMAT")
 		var _outputFormat: String = ""
 
-		val outputFormat: StorageFormat?
-            get() = runCatching {
+		/* the format the user asked for, via --output-format or a scheme/extension on the container; null if unspecified */
+		val requestedOutputFormat: StorageFormat?
+			get() = runCatching {
 				StorageFormat.valueOf(_outputFormat)
 			}.getOrNull()
 				?: StorageFormat.parseUri(_outputContainer).a
 
+		/* Only N5 supports Paintera datasets right now, as both the label multiset type
+		 * and the adjacent index dataset(s) are varlen; an ambiguous container path would
+		 * otherwise be created in the N5Factory default format */
+		val outputFormat: StorageFormat
+			get() = StorageFormat.N5
+
 		@CommandLine.Option(
 			names = ["--spark-master"],
 			required = false,
@@ -102,13 +109,11 @@ class ToPainteraData {
 			if (helpRequested)
 				return 0
 
-			/* Only N5 Format supports Paintera datasets right now, as both the
-			* label multiset type and the adjacent index dataset are varlen  */
-			val resolvedOutputFormat = outputFormat ?: StorageFormat.guessStorageFromUri(outputContainer)
-			if (resolvedOutputFormat != null && resolvedOutputFormat != StorageFormat.N5) {
+			val requested = requestedOutputFormat
+			if (requested != null && requested != StorageFormat.N5) {
 				val error = InvalidOutputContainer(
 					_outputContainer,
-					"to-paintera only supports output-format=N5, but got `$resolvedOutputFormat'"
+					"to-paintera only supports output-format=N5, but got `$requested'"
 				)
 				LOG.error { error.message }
 				return error.exitCode

src/test/kotlin/org/janelia/saalfeldlab/conversion/PainteraConvertTest.kt

@@ -337,6 +337,39 @@ class PainteraConvertTest {
             })
     }
 
+    /* an output container without a recognizable extension used to be created in the N5Factory
+     * default format (zarr3), which cannot hold the varlen label multiset blocks */
+    @Test
+    fun `ambiguous output container should be n5`() {
+        val inputPath = "${Files.createTempDirectory("no-ext-in")}.n5"
+        N5Utils.save(LABELS, createWriter(inputPath), LABEL_SOURCE_DATASET, blockSize, RawCompression())
+        val painteraPath = Files.createTempDirectory("no-ext-out").resolve("data").toString()
+        val target = "volumes/labels"
+        System.setProperty("spark.master", "local[1]")
+        main(
+            arrayOf(
+                "to-paintera",
+                "--container=$inputPath",
+                "--output-container=$painteraPath",
+                "-d", LABEL_SOURCE_DATASET,
+                "--type=label",
+                "--target-dataset=$target",
+                "--block-size=2,2,2"
+            )
+        )
+
+        assertTrue(Files.exists(java.nio.file.Paths.get(painteraPath, "attributes.json")))
+        val painteraN5 = createWriter("n5:$painteraPath")
+        assertEquals(DataType.UINT8, painteraN5.getDatasetAttributes("$target/data/s0").dataType)
+
+        /* the unique-label extraction reads the multiset back; it threw NegativeArraySizeException on zarr3 */
+        assertTrue(painteraN5.datasetExists("$target/unique-labels/s0"))
+        LoopBuilder.setImages(LABELS, N5LabelMultisets.openLabelMultiset(painteraN5, "$target/data/s0"))
+            .forEachPixel(BiConsumer { e: UnsignedLongType, a: LabelMultisetType ->
+                assertTrue(a.entrySet().size == 1 && a.entrySet().iterator().next().element.id() == e.get())
+            })
+    }
+
     @Test
     fun `to-paintera only supports n5 output format`() {
         val inputPath = "${Files.createTempDirectory("guard-in")}.n5"

startup-scripts/flintstone-paintera-convert.sh

@@ -91,15 +91,40 @@ fi
 LIB_DIR="${LIB_DIR:-$PROJECT_ROOT/target/dependency}"
 
 find_main_jar() {
-    ls "$LIB_DIR"/paintera-conversion-helper-*.jar 2>/dev/null \
-        | grep -vE '\-(sources|javadoc)\.jar$' | head -1 || true
+    local jar
+    for jar in "$LIB_DIR"/paintera-conversion-helper-*.jar; do
+        [[ -f "$jar" ]] || continue
+        case "$jar" in
+            *-sources.jar|*-javadoc.jar|*-tests.jar) continue ;;
+        esac
+        printf '%s\n' "$jar"
+        return 0
+    done
 }
 
+# a bundled Spark jar means a prior local build populated the dir with compile-scope Spark; prepend
+# `clean` to the build so it starts fresh with -Pspark-provided, since a bundled Spark would conflict
+# with the cluster's SPARK_HOME
+has_bundled_spark() {
+    local jar
+    for jar in "$LIB_DIR"/spark-core_2.12-*.jar; do
+        [[ -f "$jar" ]] && return 0
+    done
+    return 1
+}
+clean_first=0
+if has_bundled_spark; then
+    echo "cleaning a prior local (Spark-bundled) build before the cluster build" 1>&2
+    clean_first=1
+fi
+
 MAIN_JAR="${MAIN_JAR:-$(find_main_jar)}"
 
-if [[ ! -f "$MAIN_JAR" ]]; then
+if [[ "$clean_first" -eq 1 || ! -f "$MAIN_JAR" ]]; then
     export MAVEN_OPTS="-XX:ActiveProcessorCount=4 -XX:MaxRAMPercentage=25 ${MAVEN_OPTS:-}"
-    "$PROJECT_ROOT/mvnw" package -DskipTests -Pspark-provided ${MAVEN_ARGS:-}
+    goals=(package)
+    [[ "$clean_first" -eq 1 ]] && goals=(clean package)
+    "$PROJECT_ROOT/mvnw" "${goals[@]}" -DskipTests -Pspark-provided ${MAVEN_ARGS:-}
     MAIN_JAR="$(find_main_jar)"
 fi
 

startup-scripts/paintera-convert.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+# paintera-convert wrapper script.
+#
+# This script builds the jar once, and explicitly excludes the default `spark-provided` profile (`-P=-spark-provided`).
+# This ensure Spark is bundled onto the classpath instead of being supplied by a cluster SPARK_HOME.
+# paintera-convert then runs on a local Spark master (local[*] by default).
+#
+# All arguments are passed directly to paintera-convert.
+#
+# usage: local-paintera-convert.sh <paintera-convert args>
+#   e.g. local-paintera-convert.sh to-paintera --container=in.zarr -d labels/s0 \
+#          --output-container=out.n5 --target-dataset=labels --type=label \
+#          --scale 2,2,2 2,2,2 --block-size=32,32,32
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+CLASS="${CLASS:-org.janelia.saalfeldlab.conversion.PainteraConvert}"
+LIB_DIR="${LIB_DIR:-$PROJECT_ROOT/target/dependency}"
+
+if [[ "$#" -eq 0 ]]; then
+    echo "usage: $(basename "$0") <paintera-convert args>" 1>&2
+    echo "   e.g. $(basename "$0") to-scalar -i in.n5 -I labels -o out.zarr --xyz-unit nm --scale 2,2,2" 1>&2
+    exit 1
+fi
+
+find_main_jar() {
+    local jar
+    for jar in "$LIB_DIR"/paintera-conversion-helper-*.jar; do
+        [[ -f "$jar" ]] || continue
+        case "$jar" in
+            *-sources.jar|*-javadoc.jar|*-tests.jar) continue ;;
+        esac
+        printf '%s\n' "$jar"
+        return 0
+    done
+}
+
+has_bundled_spark() {
+    local jar
+    for jar in "$LIB_DIR"/spark-core_2.12-*.jar; do
+        [[ -f "$jar" ]] && return 0
+    done
+    return 1
+}
+
+MAIN_JAR="$(find_main_jar)"
+
+# build once; also (re)build if a previous cluster build left the lib dir without a bundled Spark
+if [[ ! -f "$MAIN_JAR" ]] || ! has_bundled_spark; then
+    export MAVEN_OPTS="-XX:ActiveProcessorCount=4 -XX:MaxRAMPercentage=25 ${MAVEN_OPTS:-}"
+    ( cd "$PROJECT_ROOT" && ./mvnw package -DskipTests -P=-spark-provided ${MAVEN_ARGS:-} )
+    MAIN_JAR="$(find_main_jar)"
+fi
+
+if [[ ! -f "$MAIN_JAR" ]]; then
+    echo "error: main jar not found in $LIB_DIR after build" 1>&2
+    exit 1
+fi
+
+JAVACMD="java"
+[[ -n "${JAVA_HOME:-}" && -x "$JAVA_HOME/bin/java" ]] && JAVACMD="$JAVA_HOME/bin/java"
+
+JVM_ARGS="${JVM_ARGS:---add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED -XX:SoftRefLRUPolicyMSPerMB=1}"
+
+exec "$JAVACMD" $JVM_ARGS -cp "$LIB_DIR/*" "$CLASS" "$@"