Skip to content

Ambiguous zarr #50

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cmhulbert merged 9 commits into from
Apr 7, 2026
Merged

Ambiguous zarr #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
ddcfc0b
chore: depend on snapshot versions
bogovicj Mar 24, 2026
6bf6d0b
feat: refactoring for TranslatedN5Reader/Writer
bogovicj Mar 24, 2026
6b77553
feat: add methods for openReader/Writer with KeyValueAccessBackend
cmhulbert Mar 19, 2026
9b92ca9
feat: introduce ZARR3 StorageFormat, repurpose ZARR3
cmhulbert Mar 19, 2026
05b3b6f
fix: test and imports for ZARR3 StorageFormat
cmhulbert Mar 20, 2026
3393f6d
fix: N5FactoryWithCache override should be latest possible openRead/W…
cmhulbert Mar 20, 2026
2b0cc0d
test: add test for ambiguous StorageFormat.ZARR behavior
cmhulbert Mar 20, 2026
0fc05ad
fix: don't annotate Nullable on KVA (we don't allow null param on the…
cmhulbert Apr 7, 2026
ce31d21
chore: bump n5 dep versions
cmhulbert Apr 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,13 +111,13 @@
<!-- NB: Deploy releases to the SciJava Maven repository. -->
<releaseProfiles>sign,deploy-to-scijava</releaseProfiles>

<n5.version>4.0.0-alpha-11</n5.version>
<n5.version>4.0.0-alpha-12</n5.version>
<n5-aws-s3.version>4.4.0-alpha-9</n5-aws-s3.version>
<n5-google-cloud.version>5.2.0-alpha-7</n5-google-cloud.version>
<n5-zarr.version>2.0.0-alpha-7</n5-zarr.version>
<n5-hdf5.version>2.3.0-alpha-6</n5-hdf5.version>
<n5-blosc.version>2.0.0-alpha-4</n5-blosc.version>
<n5-imglib2.version>7.1.0-alpha-7</n5-imglib2.version>
<n5-google-cloud.version>5.2.0-alpha-7</n5-google-cloud.version>
<n5-hdf5.version>2.3.0-alpha-7</n5-hdf5.version>
<n5-imglib2.version>7.1.0-alpha-8</n5-imglib2.version>
<n5-zarr.version>2.0.0-alpha-8</n5-zarr.version>
<n5-zstandard.version>2.0.0-alpha-4</n5-zstandard.version>

<jackson-jq.version>1.0.0-preview.20191208</jackson-jq.version>
Expand Down
169 changes: 153 additions & 16 deletions src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess;
import org.janelia.saalfeldlab.n5.KeyValueAccess;
import org.janelia.saalfeldlab.n5.N5Exception;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
import org.janelia.saalfeldlab.n5.N5KeyValueReader;
import org.janelia.saalfeldlab.n5.N5KeyValueWriter;
import org.janelia.saalfeldlab.n5.N5Reader;
Expand All @@ -58,12 +59,12 @@
import java.net.URISyntaxException;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.regex.Pattern;

import static org.janelia.saalfeldlab.n5.universe.StorageFormat.getStorageFromNestedScheme;
import static org.janelia.saalfeldlab.n5.universe.StorageFormat.guessStorageFromKeys;
import static org.janelia.saalfeldlab.n5.universe.StorageFormat.*;

/**
* Factory for various N5 readers and writers. Implementation-specific
Expand Down Expand Up @@ -244,7 +245,7 @@ public N5Reader openFSReader(final String path) {
@Deprecated
public N5Reader openZarrReader(final String path) {

return openN5ContainerWithStorageFormat(StorageFormat.ZARR2, path, this::openReader);
return openN5ContainerWithStorageFormat(StorageFormat.ZARR, path, this::openReader);
}

/**
Expand Down Expand Up @@ -274,7 +275,10 @@ public N5Reader openHDF5Reader(final String path) {
* @param uri uri to the google cloud object
* @return the N5Reader
* @throws URISyntaxException if uri is malformed
*
* @deprecated use {@link N5Factory#openReader(KeyValueAccessBackend, URI)} instead
*/
@Deprecated
public N5Reader openGoogleCloudReader(final String uri) throws URISyntaxException {

return openN5ContainerWithBackend(KeyValueAccessBackend.GOOGLE_CLOUD, uri, true, this::openReader);
Expand All @@ -286,7 +290,10 @@ public N5Reader openGoogleCloudReader(final String uri) throws URISyntaxExceptio
* @param uri uri to the amazon s3 object
* @return the N5Reader
* @throws URISyntaxException if uri is malformed
*
* @deprecated use {@link N5Factory#openReader(KeyValueAccessBackend, URI)} instead
*/
@Deprecated
public N5Reader openAWSS3Reader(final String uri) throws URISyntaxException {

return openN5ContainerWithBackend(KeyValueAccessBackend.AWS, uri, true, this::openReader);
Expand All @@ -298,12 +305,23 @@ public N5Reader openAWSS3Reader(final String uri) throws URISyntaxException {
* @param uri uri to the N5Reader
* @return the N5Reader
* @throws URISyntaxException if uri is malformed
*
* @deprecated use {@link N5Factory#openReader(KeyValueAccessBackend, URI)} instead
*/
@Deprecated
public N5Reader openFileSystemReader(final String uri) throws URISyntaxException {

return openN5ContainerWithBackend(KeyValueAccessBackend.FILE, uri, true, this::openReader);
}

StorageFormat[] orderedStorageFormats() {

return Arrays.stream(StorageFormat.values()).sorted((l, r) -> {
if (l == preferredStorageFormat) return -1;
return l.compareTo(r);
}).toArray(StorageFormat[]::new);
}

/**
* Open an {@link N5Reader} at the given URL, with best attempts to infer
* the appropriate {@link KeyValueAccessBackend} and {@link StorageFormat}.
Expand Down Expand Up @@ -353,14 +371,39 @@ public N5Reader openReader(final StorageFormat format, final URI uri) {
return openN5Container(format, uri, true, this::openReader);
}

StorageFormat[] orderedStorageFormats() {

return Arrays.stream(StorageFormat.values()).sorted((l, r) -> {
if (l == preferredStorageFormat) return -1;
return l.compareTo(r);
}).toArray(StorageFormat[]::new);
/**
* Open and N5Reader at the given URI given the specified {@code KeyValueAccessBackend}.
*
* @param backend key-value access
* @param uri location of reader
* @return the N5Reader
*/
public N5Reader openReader(final KeyValueAccessBackend backend, final String uri) {
final Pair<StorageFormat, URI> storageAndUri = StorageFormat.parseUri(uri);
final StorageFormat format = storageAndUri.getA();
final URI asUri = storageAndUri.getB();
final boolean inferredStorageFormat = format != null && getStorageFromNestedScheme(uri).getA() == null;
final KeyValueAccess kva = backend.apply(asUri, this, true);
if (inferredStorageFormat) {
final StorageFormat inferredFromKeys = guessStorageFromKeys(asUri, kva);
final StorageFormat inferredFormat = inferredFromKeys != null ? inferredFromKeys : format;
return openReader(inferredFormat, kva, asUri);
}
return openReader(format, kva, asUri);
}

/**
* Open and N5Reader at the given URI given the specified {@code KeyValueAccessBackend}.
*
* @param backend key-value access
* @param uri location of reader
* @return the N5Reader
*/
public N5Reader openReader(final KeyValueAccessBackend backend, final URI uri) {
final KeyValueAccess kva = backend.apply(uri, this, true);
final StorageFormat inferredFromKeys = guessStorageFromKeys(uri, kva);
return openReader(inferredFromKeys, kva, uri);
}


/**
Expand All @@ -371,8 +414,7 @@ StorageFormat[] orderedStorageFormats() {
* @param location root URI of the resulting N5Reader
* @return the N5Reader
*/
private N5Reader openReader(@Nullable final StorageFormat storage, @Nullable final KeyValueAccess access, URI location) {

public N5Reader openReader(@Nullable final StorageFormat storage, final KeyValueAccess access, URI location) {

if (storage == null) {
for (final StorageFormat format : orderedStorageFormats()) {
Expand All @@ -389,17 +431,37 @@ private N5Reader openReader(@Nullable final StorageFormat storage, @Nullable fin
switch (storage) {
case N5:
return new N5KeyValueReader(access, containerPath, gsonBuilder, cacheAttributes);
case ZARR:
case ZARR3:
return new ZarrV3KeyValueReader(access,containerPath, gsonBuilder, cacheAttributes);
case ZARR2:
return new ZarrKeyValueReader(access, containerPath, gsonBuilder, zarrMapN5DatasetAttributes, zarrMergeAttributes, cacheAttributes);
case ZARR:
return newGenericZarrReader(access, location);
case HDF5:
return new N5HDF5Reader(containerPath, hdf5OverrideBlockSize, gsonBuilder, hdf5DefaultBlockSize);
}
return null;
}
}

/**
* Open a zarr as N5Reader at the given {@code access} and {@code location}.
* Will prefer returning the newest version of zarr that is found at the location.
*
* @param access to the key-value access backend
* @param location of the zarr container
* @return an N5Reader
*/
private N5Reader newGenericZarrReader(final KeyValueAccess access, final URI location) {
for (StorageFormat zarrFormat : Arrays.asList(ZARR3, ZARR2)) {
try {
return openReader(zarrFormat, access, location);
} catch (Exception ignored) {
}
}
throw new N5IOException("Unable to open Zarr reader at " + location.toString() + " as N5Reader");
}

/**
* Open an {@link N5Writer} for N5 Container.
* <p>
Expand Down Expand Up @@ -432,7 +494,7 @@ public N5Writer openFSWriter(final String path) {
@Deprecated
public N5Writer openZarrWriter(final String path) {

return openN5ContainerWithStorageFormat(StorageFormat.ZARR2, path, this::openWriter);
return openN5ContainerWithStorageFormat(StorageFormat.ZARR, path, this::openWriter);
}

/**
Expand Down Expand Up @@ -462,7 +524,10 @@ public N5Writer openHDF5Writer(final String path) {
* @param uri uri to the google cloud object
* @return the N5GoogleCloudStorageWriter
* @throws URISyntaxException if uri is malformed
*
* @deprecated use {@link #openWriter(KeyValueAccessBackend, String)} instead.
*/
@Deprecated
public N5Writer openGoogleCloudWriter(final String uri) throws URISyntaxException {

return openN5ContainerWithBackend(KeyValueAccessBackend.GOOGLE_CLOUD, uri, false, this::openWriter);
Expand All @@ -474,7 +539,11 @@ public N5Writer openGoogleCloudWriter(final String uri) throws URISyntaxExceptio
* @param uri uri to the s3 object
* @return the N5Writer
* @throws URISyntaxException if the URI is malformed
*
*
* @deprecated use {@link #openWriter(KeyValueAccessBackend, String)} instead.
*/
@Deprecated()
public N5Writer openAWSS3Writer(final String uri) throws URISyntaxException {

return openN5ContainerWithBackend(KeyValueAccessBackend.AWS, uri, false, this::openWriter);
Expand All @@ -496,12 +565,47 @@ public N5Writer openWriter(final String uri) {
final KeyValueAccess kva = getKeyValueAccess(asUri, false);
final StorageFormat inferredFromKeys = guessStorageFromKeys(asUri, kva);
final StorageFormat inferredFormat = inferredFromKeys != null ? inferredFromKeys : format;
return openWriter(inferredFormat, asUri);
return openWriter(inferredFormat, kva, asUri);
}

return openWriter(format, asUri);
}

/**
* Create or Open an N5Writer as the given uri and backend.
*
* @param backend key-value access
* @param uri location of writer
* @return the N5Writer
*/
public N5Writer openWriter(final KeyValueAccessBackend backend, final URI uri) {
final KeyValueAccess kva = backend.apply(uri, this, false);
final StorageFormat inferredFromKeys = guessStorageFromKeys(uri, kva);
return openWriter(inferredFromKeys, kva, uri);
}

/**
* Create or Open an N5Writer as the given uri and backend.
*
* @param backend key-value access
* @param uri location of writer
* @return the N5Writer
*/
public N5Writer openWriter(final KeyValueAccessBackend backend, final String uri) {
final Pair<StorageFormat, URI> storageAndUri = StorageFormat.parseUri(uri);
final StorageFormat format = storageAndUri.getA();
final URI asUri = storageAndUri.getB();
final KeyValueAccess kva = backend.apply(asUri, this, false);
final boolean inferredStorageFormat = format != null && getStorageFromNestedScheme(uri).getA() == null;
if (inferredStorageFormat) {
final StorageFormat inferredFromKeys = guessStorageFromKeys(asUri, kva);
final StorageFormat inferredFormat = inferredFromKeys != null ? inferredFromKeys : format;
return openWriter(inferredFormat, kva, asUri);
}

return openWriter(format, kva, asUri);
}

/**
* Open an {@link N5Writer} at the given URL, with best attempts to infer
* the appropriate {@link KeyValueAccessBackend}.
Expand Down Expand Up @@ -538,7 +642,7 @@ public N5Writer openWriter(final StorageFormat format, final URI uri) {
* @param location root location of the resulting N5Writer
* @return the N5Writer
*/
public N5Writer openWriter(@Nullable final StorageFormat storage, @Nullable final KeyValueAccess access, final URI location) {
public N5Writer openWriter(@Nullable final StorageFormat storage, final KeyValueAccess access, final URI location) {

if (storage == null) {
for (final StorageFormat format : orderedStorageFormats()) {
Expand All @@ -553,12 +657,14 @@ public N5Writer openWriter(@Nullable final StorageFormat storage, @Nullable fina

final String containerLocation = location.toString();
switch (storage) {
case ZARR:
case ZARR3:
final ZarrV3KeyValueWriter writer = new ZarrV3KeyValueWriter(access, containerLocation, gsonBuilder, cacheAttributes);
writer.setDimensionSeparator(zarrDimensionSeparator);
return writer;
case ZARR2:
return new ZarrKeyValueWriter(access, containerLocation, gsonBuilder, zarrMapN5DatasetAttributes, zarrMergeAttributes, zarrDimensionSeparator, cacheAttributes);
case ZARR:
return newGenericZarrWriter(access, location);
case N5:
return new N5KeyValueWriter(access, containerLocation, gsonBuilder, cacheAttributes);
case HDF5:
Expand All @@ -568,6 +674,37 @@ public N5Writer openWriter(@Nullable final StorageFormat storage, @Nullable fina
return null;
}

/**
* Try to get a zarr writer at the given location and access.
* If a container exists at the location, load that version as a writer if possible.
* If no container exists, create a new writer with the newest zarr version.
*
* @param access to the key-value backend
* @param location of the zarr writer
* @return the zarr writer
*/
private N5Writer newGenericZarrWriter(final KeyValueAccess access, final URI location) {
List<StorageFormat> zarrFormats = Arrays.asList(ZARR3, ZARR2);
for (StorageFormat zarrFormat : zarrFormats) {
/* we dont care about the read, but we do want to prefer a writer over a container that
* exists, rather than creating a new writer; the only way to check is to see if
* we can get a valid reader. If we can, try the writer. */
try (N5Reader ignore = openReader(zarrFormat, access, location)) {
return openWriter(zarrFormat, access, location);
} catch (Exception ignored) {
}
}
/* However, if we have no valid readers, then try and return the first valid (created) writer */
for (StorageFormat zarrFormat : zarrFormats) {
try {
return openWriter(zarrFormat, access, location);
} catch (Exception ignored) {
}
}

throw new N5IOException("Unable to open Zarr writer at " + location.toString() + " as N5Reader");
}

private <T extends N5Reader> T openN5ContainerWithStorageFormat(
final StorageFormat format,
final String uri,
Expand Down
Loading
Loading