22
33import com .google .gson .Gson ;
44import com .google .gson .GsonBuilder ;
5- import com .pivovarit .function .ThrowingSupplier ;
65import net .imglib2 .FinalInterval ;
76import net .imglib2 .Interval ;
87import net .imglib2 .RandomAccessibleInterval ;
98import net .imglib2 .algorithm .util .Grids ;
109import net .imglib2 .algorithm .util .Singleton ;
11- import net .imglib2 .cache .img .CachedCellImg ;
1210import net .imglib2 .converter .Converters ;
1311import net .imglib2 .type .NativeType ;
1412import net .imglib2 .type .label .FromIntegerTypeConverter ;
2119import org .apache .http .message .BasicNameValuePair ;
2220import org .apache .spark .SparkConf ;
2321import org .apache .spark .api .java .JavaSparkContext ;
22+ import org .apache .spark .api .java .function .Function0 ;
2423import org .janelia .saalfeldlab .label .spark .N5Helpers ;
2524import org .janelia .saalfeldlab .n5 .Compression ;
2625import org .janelia .saalfeldlab .n5 .CompressionAdapter ;
2726import org .janelia .saalfeldlab .n5 .DataType ;
28- import org .janelia .saalfeldlab .n5 .DatasetAttributes ;
2927import org .janelia .saalfeldlab .n5 .GsonUtils ;
30- import org .janelia .saalfeldlab .n5 .GzipCompression ;
28+ import org .janelia .scicomp .n5 .zstandard . ZstandardCompression ;
3129import org .janelia .saalfeldlab .n5 .N5Reader ;
3230import org .janelia .saalfeldlab .n5 .N5Writer ;
3331import org .janelia .saalfeldlab .n5 .imglib2 .N5LabelMultisets ;
4543import java .io .IOException ;
4644import java .lang .invoke .MethodHandles ;
4745import java .net .URI ;
48- import java .net .URISyntaxException ;
4946import java .util .Arrays ;
5047import java .util .Comparator ;
5148import java .util .HashMap ;
@@ -95,9 +92,6 @@ static public class CommandLineParameters implements Callable<Void> {
9592 @ Option (names = {"--block-size" , "-b" }, paramLabel = "BLOCK_SIZE" , description = "Size of cells to use in the output N5 dataset. Defaults to 64. Either single integer value for isotropic block size or comma-seperated list of block size per dimension" , split = "," )
9693 private int [] blockSize ;
9794
98- @ Option (names = {"--compression" , "-c" }, paramLabel = "COMPRESSION" , description = "Compression type to use in output N5 dataset" )
99- public String compressionType = "{\" type\" :\" gzip\" ,\" level\" :-1}" ;
100-
10195 @ Option (
10296 names = {"--reverse-array-attributes" },
10397 required = false ,
@@ -113,10 +107,7 @@ public Void call() throws IOException {
113107 final Gson gson = new GsonBuilder ()
114108 .registerTypeHierarchyAdapter (Compression .class , CompressionAdapter .getJsonAdapter ())
115109 .create ();
116- final Compression compression = new GzipCompression ();// .fromJson(
117- // compressionType,
118- // Compression.class
119- // );
110+ final Compression compression = new ZstandardCompression ();
120111 final int nDim = N5Helpers .n5Reader (this .inputN5 ).getDatasetAttributes (this .inputDataset ).getNumDimensions ();
121112 final int [] blockSize = this .blockSize .length < nDim ? IntStream .generate (() -> this .blockSize [0 ]).limit (nDim ).toArray () : this .blockSize ;
122113
@@ -168,39 +159,52 @@ public static <I extends IntegerType<I> & NativeType<I>> void convertToLabelMult
168159 final String outputGroupName ,
169160 final String outputDatasetName ,
170161 final Compression compression ,
171- final boolean reverse ) throws IOException {
162+ final boolean reverse ) {
163+
164+ final N5Reader reader = N5Helpers .n5Reader (inputGroup , blockSize );
165+ final int [] inputBlockSize = reader .getDatasetAttributes (inputDataset ).getBlockSize ();
166+ final Map <String , Object > attributesToCopy = resolveAttributesToCopy (reader , inputDataset , reverse );
172167
173- final ThrowingSupplier < CachedCellImg < I , ?>, URISyntaxException > getCachedImg = () -> {
168+ final Function0 < RandomAccessibleInterval < I >> imgSupplier = () -> {
174169 final URI inputGroupUri = StorageFormat .parseUri (inputGroup ).getB ();
175170 final String inputImgCacheKey = new URIBuilder (inputGroupUri )
176171 .setParameters (
177172 new BasicNameValuePair ("call" , "convert-to-label-multiset-type" ),
178173 new BasicNameValuePair ("dataset" , inputDataset )
179174 ).toString ();
180-
181175 return Singleton .get (inputImgCacheKey , () -> N5Helpers .openBounded (N5Helpers .n5Reader (inputGroup , blockSize ), inputDataset ));
182176 };
183177
184- final N5Reader reader = N5Helpers .n5Reader (inputGroup , blockSize );
185- final DatasetAttributes inputDataAttrs = reader .getDatasetAttributes (inputDataset );
186- final int [] inputBlockSize = inputDataAttrs .getBlockSize ();
187- final RandomAccessibleInterval <I > img = getCachedImg .uncheck ().get ();
188- final Map <String , Class <?>> attributeNames ;
189- if (reader instanceof ZarrKeyValueReader ) {
190- attributeNames = Optional .of (reader )
191- .map (ZarrKeyValueReader .class ::cast )
192- .map (it -> it .getZAttributes (inputDataset ))
193- .map (GsonUtils ::listAttributes )
194- .orElseGet (HashMap ::new );
195- } else {
196- attributeNames = reader .listAttributes (inputDataset );
197- List .of (
198- LABEL_MULTISETTYPE_KEY ,
199- DATA_TYPE_KEY ,
200- COMPRESSION_KEY ,
201- BLOCK_SIZE_KEY ,
202- DIMENSIONS_KEY
203- ).forEach (attributeNames ::remove );
178+ convertToLabelMultisetType (sc , imgSupplier , inputBlockSize , attributesToCopy , blockSize , outputGroupName , outputDatasetName , compression );
179+ }
180+
181+ public static <I extends IntegerType <I > & NativeType <I >> void convertToLabelMultisetType (
182+ final JavaSparkContext sc ,
183+ final Function0 <RandomAccessibleInterval <I >> imgSupplier ,
184+ final int [] inputBlockSize ,
185+ final int [] blockSize ,
186+ final String outputGroupName ,
187+ final String outputDatasetName ,
188+ final Compression compression ) {
189+
190+ convertToLabelMultisetType (sc , imgSupplier , inputBlockSize , new HashMap <>(), blockSize , outputGroupName , outputDatasetName , compression );
191+ }
192+
193+ private static <I extends IntegerType <I > & NativeType <I >> void convertToLabelMultisetType (
194+ final JavaSparkContext sc ,
195+ final Function0 <RandomAccessibleInterval <I >> imgSupplier ,
196+ final int [] inputBlockSize ,
197+ final Map <String , Object > attributesToCopy ,
198+ final int [] blockSize ,
199+ final String outputGroupName ,
200+ final String outputDatasetName ,
201+ final Compression compression ) {
202+
203+ final RandomAccessibleInterval <I > img ;
204+ try {
205+ img = imgSupplier .call ();
206+ } catch (final Exception e ) {
207+ throw new RuntimeException ("Unable to get source image" , e );
204208 }
205209
206210 final int nDim = img .numDimensions ();
@@ -224,8 +228,8 @@ public static <I extends IntegerType<I> & NativeType<I>> void convertToLabelMult
224228 }
225229 writer .createDataset (outputDatasetName , dimensions , blockSize , DataType .UINT8 , compression );
226230 writer .setAttribute (outputDatasetName , LABEL_MULTISETTYPE_KEY , true );
227- for (final Entry <String , Class <?>> entry : attributeNames .entrySet ())
228- writer .setAttribute (outputDatasetName , entry .getKey (), N5Helpers . reverseInplaceAndReturn ( reader . getAttribute ( inputDataset , entry .getKey (), entry . getValue ()), reverse ));
231+ for (final Entry <String , Object > entry : attributesToCopy .entrySet ())
232+ writer .setAttribute (outputDatasetName , entry .getKey (), entry .getValue ());
229233
230234 final int [] parallelizeBlockSize = new int [blockSize .length ];
231235 if (Intervals .numElements (blockSize ) >= Intervals .numElements (inputBlockSize )) {
@@ -246,13 +250,7 @@ public static <I extends IntegerType<I> & NativeType<I>> void convertToLabelMult
246250 .map (intervalMinMax -> {
247251 final Interval interval = new FinalInterval (intervalMinMax ._1 (), intervalMinMax ._2 ());
248252
249- final URI uri = StorageFormat .parseUri (inputGroup ).getB ();
250- final String imgCacheKey = new URIBuilder (uri )
251- .setParameters (
252- new BasicNameValuePair ("call" , "convert-to-label-multiset-max-id" ),
253- new BasicNameValuePair ("dataset" , inputDataset )
254- ).toString ();
255- final CachedCellImg <I , ?> source = Singleton .get (imgCacheKey , () -> N5Helpers .openBounded (N5Helpers .n5Reader (inputGroup , blockSize ), inputDataset ));
253+ final RandomAccessibleInterval <I > source = imgSupplier .call ();
256254 final RandomAccessibleInterval <I > blockImg = Views .interval (source , interval );
257255
258256 final FromIntegerTypeConverter <I > converter = new FromIntegerTypeConverter <>();
@@ -280,4 +278,28 @@ public static <I extends IntegerType<I> & NativeType<I>> void convertToLabelMult
280278 Singleton .clear ();
281279 writer .setAttribute (outputDatasetName , MAX_ID_KEY , maxId );
282280 }
281+
282+ private static Map <String , Object > resolveAttributesToCopy (final N5Reader reader , final String inputDataset , final boolean reverse ) {
283+ final Map <String , Class <?>> attributeNames ;
284+ if (reader instanceof ZarrKeyValueReader ) {
285+ attributeNames = Optional .of (reader )
286+ .map (ZarrKeyValueReader .class ::cast )
287+ .map (it -> it .getZAttributes (inputDataset ))
288+ .map (GsonUtils ::listAttributes )
289+ .orElseGet (HashMap ::new );
290+ } else {
291+ attributeNames = reader .listAttributes (inputDataset );
292+ List .of (
293+ LABEL_MULTISETTYPE_KEY ,
294+ DATA_TYPE_KEY ,
295+ COMPRESSION_KEY ,
296+ BLOCK_SIZE_KEY ,
297+ DIMENSIONS_KEY
298+ ).forEach (attributeNames ::remove );
299+ }
300+ final Map <String , Object > resolved = new HashMap <>();
301+ for (final Entry <String , Class <?>> entry : attributeNames .entrySet ())
302+ resolved .put (entry .getKey (), N5Helpers .reverseInplaceAndReturn (reader .getAttribute (inputDataset , entry .getKey (), entry .getValue ()), reverse ));
303+ return resolved ;
304+ }
283305}
0 commit comments