Skip to content

Commit 866afa3

Browse files
cmhulbertcmhulbert
authored
Merge pull request #50 from saalfeldlab/ambiguousZarr
Ambiguous zarr
2 parents 223972a + ce31d21 commit 866afa3

11 files changed

Lines changed: 276 additions & 78 deletions

File tree

‎pom.xml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -111,13 +111,13 @@
111111
<!-- NB: Deploy releases to the SciJava Maven repository. -->
112112
<releaseProfiles>sign,deploy-to-scijava</releaseProfiles>
113113

114-
<n5.version>4.0.0-alpha-11</n5.version>
114+
<n5.version>4.0.0-alpha-12</n5.version>
115115
<n5-aws-s3.version>4.4.0-alpha-9</n5-aws-s3.version>
116-
<n5-google-cloud.version>5.2.0-alpha-7</n5-google-cloud.version>
117-
<n5-zarr.version>2.0.0-alpha-7</n5-zarr.version>
118-
<n5-hdf5.version>2.3.0-alpha-6</n5-hdf5.version>
119116
<n5-blosc.version>2.0.0-alpha-4</n5-blosc.version>
120-
<n5-imglib2.version>7.1.0-alpha-7</n5-imglib2.version>
117+
<n5-google-cloud.version>5.2.0-alpha-7</n5-google-cloud.version>
118+
<n5-hdf5.version>2.3.0-alpha-7</n5-hdf5.version>
119+
<n5-imglib2.version>7.1.0-alpha-8</n5-imglib2.version>
120+
<n5-zarr.version>2.0.0-alpha-8</n5-zarr.version>
121121
<n5-zstandard.version>2.0.0-alpha-4</n5-zstandard.version>
122122

123123
<jackson-jq.version>1.0.0-preview.20191208</jackson-jq.version>

‎src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java‎

Lines changed: 153 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess;
3636
import org.janelia.saalfeldlab.n5.KeyValueAccess;
3737
import org.janelia.saalfeldlab.n5.N5Exception;
38+
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
3839
import org.janelia.saalfeldlab.n5.N5KeyValueReader;
3940
import org.janelia.saalfeldlab.n5.N5KeyValueWriter;
4041
import org.janelia.saalfeldlab.n5.N5Reader;
@@ -58,12 +59,12 @@
5859
import java.net.URISyntaxException;
5960
import java.nio.file.Paths;
6061
import java.util.Arrays;
62+
import java.util.List;
6163
import java.util.function.BiFunction;
6264
import java.util.function.Consumer;
6365
import java.util.regex.Pattern;
6466

65-
import static org.janelia.saalfeldlab.n5.universe.StorageFormat.getStorageFromNestedScheme;
66-
import static org.janelia.saalfeldlab.n5.universe.StorageFormat.guessStorageFromKeys;
67+
import static org.janelia.saalfeldlab.n5.universe.StorageFormat.*;
6768

6869
/**
6970
* Factory for various N5 readers and writers. Implementation-specific
@@ -244,7 +245,7 @@ public N5Reader openFSReader(final String path) {
244245
@Deprecated
245246
public N5Reader openZarrReader(final String path) {
246247

247-
return openN5ContainerWithStorageFormat(StorageFormat.ZARR2, path, this::openReader);
248+
return openN5ContainerWithStorageFormat(StorageFormat.ZARR, path, this::openReader);
248249
}
249250

250251
/**
@@ -274,7 +275,10 @@ public N5Reader openHDF5Reader(final String path) {
274275
* @param uri uri to the google cloud object
275276
* @return the N5Reader
276277
* @throws URISyntaxException if uri is malformed
278+
*
279+
* @deprecated use {@link N5Factory#openReader(KeyValueAccessBackend, URI)} instead
277280
*/
281+
@Deprecated
278282
public N5Reader openGoogleCloudReader(final String uri) throws URISyntaxException {
279283

280284
return openN5ContainerWithBackend(KeyValueAccessBackend.GOOGLE_CLOUD, uri, true, this::openReader);
@@ -286,7 +290,10 @@ public N5Reader openGoogleCloudReader(final String uri) throws URISyntaxExceptio
286290
* @param uri uri to the amazon s3 object
287291
* @return the N5Reader
288292
* @throws URISyntaxException if uri is malformed
293+
*
294+
* @deprecated use {@link N5Factory#openReader(KeyValueAccessBackend, URI)} instead
289295
*/
296+
@Deprecated
290297
public N5Reader openAWSS3Reader(final String uri) throws URISyntaxException {
291298

292299
return openN5ContainerWithBackend(KeyValueAccessBackend.AWS, uri, true, this::openReader);
@@ -298,12 +305,23 @@ public N5Reader openAWSS3Reader(final String uri) throws URISyntaxException {
298305
* @param uri uri to the N5Reader
299306
* @return the N5Reader
300307
* @throws URISyntaxException if uri is malformed
308+
*
309+
* @deprecated use {@link N5Factory#openReader(KeyValueAccessBackend, URI)} instead
301310
*/
311+
@Deprecated
302312
public N5Reader openFileSystemReader(final String uri) throws URISyntaxException {
303313

304314
return openN5ContainerWithBackend(KeyValueAccessBackend.FILE, uri, true, this::openReader);
305315
}
306316

317+
StorageFormat[] orderedStorageFormats() {
318+
319+
return Arrays.stream(StorageFormat.values()).sorted((l, r) -> {
320+
if (l == preferredStorageFormat) return -1;
321+
return l.compareTo(r);
322+
}).toArray(StorageFormat[]::new);
323+
}
324+
307325
/**
308326
* Open an {@link N5Reader} at the given URL, with best attempts to infer
309327
* the appropriate {@link KeyValueAccessBackend} and {@link StorageFormat}.
@@ -353,14 +371,39 @@ public N5Reader openReader(final StorageFormat format, final URI uri) {
353371
return openN5Container(format, uri, true, this::openReader);
354372
}
355373

356-
StorageFormat[] orderedStorageFormats() {
357-
358-
return Arrays.stream(StorageFormat.values()).sorted((l, r) -> {
359-
if (l == preferredStorageFormat) return -1;
360-
return l.compareTo(r);
361-
}).toArray(StorageFormat[]::new);
374+
/**
375+
* Open and N5Reader at the given URI given the specified {@code KeyValueAccessBackend}.
376+
*
377+
* @param backend key-value access
378+
* @param uri location of reader
379+
* @return the N5Reader
380+
*/
381+
public N5Reader openReader(final KeyValueAccessBackend backend, final String uri) {
382+
final Pair<StorageFormat, URI> storageAndUri = StorageFormat.parseUri(uri);
383+
final StorageFormat format = storageAndUri.getA();
384+
final URI asUri = storageAndUri.getB();
385+
final boolean inferredStorageFormat = format != null && getStorageFromNestedScheme(uri).getA() == null;
386+
final KeyValueAccess kva = backend.apply(asUri, this, true);
387+
if (inferredStorageFormat) {
388+
final StorageFormat inferredFromKeys = guessStorageFromKeys(asUri, kva);
389+
final StorageFormat inferredFormat = inferredFromKeys != null ? inferredFromKeys : format;
390+
return openReader(inferredFormat, kva, asUri);
391+
}
392+
return openReader(format, kva, asUri);
362393
}
363394

395+
/**
396+
* Open and N5Reader at the given URI given the specified {@code KeyValueAccessBackend}.
397+
*
398+
* @param backend key-value access
399+
* @param uri location of reader
400+
* @return the N5Reader
401+
*/
402+
public N5Reader openReader(final KeyValueAccessBackend backend, final URI uri) {
403+
final KeyValueAccess kva = backend.apply(uri, this, true);
404+
final StorageFormat inferredFromKeys = guessStorageFromKeys(uri, kva);
405+
return openReader(inferredFromKeys, kva, uri);
406+
}
364407

365408

366409
/**
@@ -371,8 +414,7 @@ StorageFormat[] orderedStorageFormats() {
371414
* @param location root URI of the resulting N5Reader
372415
* @return the N5Reader
373416
*/
374-
private N5Reader openReader(@Nullable final StorageFormat storage, @Nullable final KeyValueAccess access, URI location) {
375-
417+
public N5Reader openReader(@Nullable final StorageFormat storage, final KeyValueAccess access, URI location) {
376418

377419
if (storage == null) {
378420
for (final StorageFormat format : orderedStorageFormats()) {
@@ -389,17 +431,37 @@ private N5Reader openReader(@Nullable final StorageFormat storage, @Nullable fin
389431
switch (storage) {
390432
case N5:
391433
return new N5KeyValueReader(access, containerPath, gsonBuilder, cacheAttributes);
392-
case ZARR:
434+
case ZARR3:
393435
return new ZarrV3KeyValueReader(access,containerPath, gsonBuilder, cacheAttributes);
394436
case ZARR2:
395437
return new ZarrKeyValueReader(access, containerPath, gsonBuilder, zarrMapN5DatasetAttributes, zarrMergeAttributes, cacheAttributes);
438+
case ZARR:
439+
return newGenericZarrReader(access, location);
396440
case HDF5:
397441
return new N5HDF5Reader(containerPath, hdf5OverrideBlockSize, gsonBuilder, hdf5DefaultBlockSize);
398442
}
399443
return null;
400444
}
401445
}
402446

447+
/**
448+
* Open a zarr as N5Reader at the given {@code access} and {@code location}.
449+
* Will prefer returning the newest version of zarr that is found at the location.
450+
*
451+
* @param access to the key-value access backend
452+
* @param location of the zarr container
453+
* @return an N5Reader
454+
*/
455+
private N5Reader newGenericZarrReader(final KeyValueAccess access, final URI location) {
456+
for (StorageFormat zarrFormat : Arrays.asList(ZARR3, ZARR2)) {
457+
try {
458+
return openReader(zarrFormat, access, location);
459+
} catch (Exception ignored) {
460+
}
461+
}
462+
throw new N5IOException("Unable to open Zarr reader at " + location.toString() + " as N5Reader");
463+
}
464+
403465
/**
404466
* Open an {@link N5Writer} for N5 Container.
405467
* <p>
@@ -432,7 +494,7 @@ public N5Writer openFSWriter(final String path) {
432494
@Deprecated
433495
public N5Writer openZarrWriter(final String path) {
434496

435-
return openN5ContainerWithStorageFormat(StorageFormat.ZARR2, path, this::openWriter);
497+
return openN5ContainerWithStorageFormat(StorageFormat.ZARR, path, this::openWriter);
436498
}
437499

438500
/**
@@ -462,7 +524,10 @@ public N5Writer openHDF5Writer(final String path) {
462524
* @param uri uri to the google cloud object
463525
* @return the N5GoogleCloudStorageWriter
464526
* @throws URISyntaxException if uri is malformed
527+
*
528+
* @deprecated use {@link #openWriter(KeyValueAccessBackend, String)} instead.
465529
*/
530+
@Deprecated
466531
public N5Writer openGoogleCloudWriter(final String uri) throws URISyntaxException {
467532

468533
return openN5ContainerWithBackend(KeyValueAccessBackend.GOOGLE_CLOUD, uri, false, this::openWriter);
@@ -474,7 +539,11 @@ public N5Writer openGoogleCloudWriter(final String uri) throws URISyntaxExceptio
474539
* @param uri uri to the s3 object
475540
* @return the N5Writer
476541
* @throws URISyntaxException if the URI is malformed
542+
*
543+
*
544+
* @deprecated use {@link #openWriter(KeyValueAccessBackend, String)} instead.
477545
*/
546+
@Deprecated()
478547
public N5Writer openAWSS3Writer(final String uri) throws URISyntaxException {
479548

480549
return openN5ContainerWithBackend(KeyValueAccessBackend.AWS, uri, false, this::openWriter);
@@ -496,12 +565,47 @@ public N5Writer openWriter(final String uri) {
496565
final KeyValueAccess kva = getKeyValueAccess(asUri, false);
497566
final StorageFormat inferredFromKeys = guessStorageFromKeys(asUri, kva);
498567
final StorageFormat inferredFormat = inferredFromKeys != null ? inferredFromKeys : format;
499-
return openWriter(inferredFormat, asUri);
568+
return openWriter(inferredFormat, kva, asUri);
500569
}
501570

502571
return openWriter(format, asUri);
503572
}
504573

574+
/**
575+
* Create or Open an N5Writer as the given uri and backend.
576+
*
577+
* @param backend key-value access
578+
* @param uri location of writer
579+
* @return the N5Writer
580+
*/
581+
public N5Writer openWriter(final KeyValueAccessBackend backend, final URI uri) {
582+
final KeyValueAccess kva = backend.apply(uri, this, false);
583+
final StorageFormat inferredFromKeys = guessStorageFromKeys(uri, kva);
584+
return openWriter(inferredFromKeys, kva, uri);
585+
}
586+
587+
/**
588+
* Create or Open an N5Writer as the given uri and backend.
589+
*
590+
* @param backend key-value access
591+
* @param uri location of writer
592+
* @return the N5Writer
593+
*/
594+
public N5Writer openWriter(final KeyValueAccessBackend backend, final String uri) {
595+
final Pair<StorageFormat, URI> storageAndUri = StorageFormat.parseUri(uri);
596+
final StorageFormat format = storageAndUri.getA();
597+
final URI asUri = storageAndUri.getB();
598+
final KeyValueAccess kva = backend.apply(asUri, this, false);
599+
final boolean inferredStorageFormat = format != null && getStorageFromNestedScheme(uri).getA() == null;
600+
if (inferredStorageFormat) {
601+
final StorageFormat inferredFromKeys = guessStorageFromKeys(asUri, kva);
602+
final StorageFormat inferredFormat = inferredFromKeys != null ? inferredFromKeys : format;
603+
return openWriter(inferredFormat, kva, asUri);
604+
}
605+
606+
return openWriter(format, kva, asUri);
607+
}
608+
505609
/**
506610
* Open an {@link N5Writer} at the given URL, with best attempts to infer
507611
* the appropriate {@link KeyValueAccessBackend}.
@@ -538,7 +642,7 @@ public N5Writer openWriter(final StorageFormat format, final URI uri) {
538642
* @param location root location of the resulting N5Writer
539643
* @return the N5Writer
540644
*/
541-
public N5Writer openWriter(@Nullable final StorageFormat storage, @Nullable final KeyValueAccess access, final URI location) {
645+
public N5Writer openWriter(@Nullable final StorageFormat storage, final KeyValueAccess access, final URI location) {
542646

543647
if (storage == null) {
544648
for (final StorageFormat format : orderedStorageFormats()) {
@@ -553,12 +657,14 @@ public N5Writer openWriter(@Nullable final StorageFormat storage, @Nullable fina
553657

554658
final String containerLocation = location.toString();
555659
switch (storage) {
556-
case ZARR:
660+
case ZARR3:
557661
final ZarrV3KeyValueWriter writer = new ZarrV3KeyValueWriter(access, containerLocation, gsonBuilder, cacheAttributes);
558662
writer.setDimensionSeparator(zarrDimensionSeparator);
559663
return writer;
560664
case ZARR2:
561665
return new ZarrKeyValueWriter(access, containerLocation, gsonBuilder, zarrMapN5DatasetAttributes, zarrMergeAttributes, zarrDimensionSeparator, cacheAttributes);
666+
case ZARR:
667+
return newGenericZarrWriter(access, location);
562668
case N5:
563669
return new N5KeyValueWriter(access, containerLocation, gsonBuilder, cacheAttributes);
564670
case HDF5:
@@ -568,6 +674,37 @@ public N5Writer openWriter(@Nullable final StorageFormat storage, @Nullable fina
568674
return null;
569675
}
570676

677+
/**
678+
* Try to get a zarr writer at the given location and access.
679+
* If a container exists at the location, load that version as a writer if possible.
680+
* If no container exists, create a new writer with the newest zarr version.
681+
*
682+
* @param access to the key-value backend
683+
* @param location of the zarr writer
684+
* @return the zarr writer
685+
*/
686+
private N5Writer newGenericZarrWriter(final KeyValueAccess access, final URI location) {
687+
List<StorageFormat> zarrFormats = Arrays.asList(ZARR3, ZARR2);
688+
for (StorageFormat zarrFormat : zarrFormats) {
689+
/* we dont care about the read, but we do want to prefer a writer over a container that
690+
* exists, rather than creating a new writer; the only way to check is to see if
691+
* we can get a valid reader. If we can, try the writer. */
692+
try (N5Reader ignore = openReader(zarrFormat, access, location)) {
693+
return openWriter(zarrFormat, access, location);
694+
} catch (Exception ignored) {
695+
}
696+
}
697+
/* However, if we have no valid readers, then try and return the first valid (created) writer */
698+
for (StorageFormat zarrFormat : zarrFormats) {
699+
try {
700+
return openWriter(zarrFormat, access, location);
701+
} catch (Exception ignored) {
702+
}
703+
}
704+
705+
throw new N5IOException("Unable to open Zarr writer at " + location.toString() + " as N5Reader");
706+
}
707+
571708
private <T extends N5Reader> T openN5ContainerWithStorageFormat(
572709
final StorageFormat format,
573710
final String uri,

0 commit comments

Comments
 (0)