From 75772b8720a76595dc6f5fdaf39e67b78cf19c78 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 20 Aug 2025 15:36:28 -0400 Subject: [PATCH 1/5] feat: DatasetAttributes.getCodecs * downstream implementations (zarr) need access to the bytesCodecs --- .../java/org/janelia/saalfeldlab/n5/DatasetAttributes.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 181efdf7b..f32bf6b7a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -32,7 +32,6 @@ import java.util.Arrays; import java.util.HashMap; -import org.janelia.saalfeldlab.n5.codec.Codec; import org.janelia.saalfeldlab.n5.codec.ArrayCodec; import org.janelia.saalfeldlab.n5.codec.BytesCodec; import org.janelia.saalfeldlab.n5.codec.DataBlockSerializer; @@ -140,6 +139,11 @@ public ArrayCodec getArrayCodec() { return arrayCodec; } + public BytesCodec[] getCodecs() { + + return byteCodecs; + } + @SuppressWarnings("unchecked") DataBlockSerializer getDataBlockSerializer() { return (DataBlockSerializer) dataBlockSerializer; From 7915ed364405b4e8db79d4fb58d01ce827d7884b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 20 Aug 2025 15:37:46 -0400 Subject: [PATCH 2/5] feat: getAttributesKey method * enables zarr to re-use the implementation here re: attributes --- .../saalfeldlab/n5/CachedGsonKeyValueN5Reader.java | 12 ++++++------ .../saalfeldlab/n5/CachedGsonKeyValueN5Writer.java | 8 ++++---- .../janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java | 2 +- .../org/janelia/saalfeldlab/n5/GsonN5Reader.java | 7 +++++++ .../org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 6 ++++++ .../saalfeldlab/n5/http/HttpReaderFsWriter.java | 5 +++++ 6 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index 6f7e7c0a8..3f0db47bf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -73,7 +73,7 @@ default DatasetAttributes getDatasetAttributes(final String pathName) { return null; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPath, N5KeyValueReader.ATTRIBUTES_JSON); + attributes = getCache().getAttributes(normalPath, getAttributesKey()); } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPath); } @@ -99,7 +99,7 @@ default T getAttribute( final JsonElement attributes; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + attributes = getCache().getAttributes(normalPathName, getAttributesKey()); } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName); } @@ -120,7 +120,7 @@ default T getAttribute( final String normalizedAttributePath = N5URI.normalizeAttributePath(key); JsonElement attributes; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + attributes = getCache().getAttributes(normalPathName, getAttributesKey()); } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName); } @@ -136,7 +136,7 @@ default boolean exists(final String pathName) { final String normalPathName = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) - return getCache().isGroup(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + return getCache().isGroup(normalPathName, getAttributesKey()); else { return existsFromContainer(normalPathName, null); } @@ -180,7 +180,7 @@ default boolean datasetExists(final String pathName) throws N5IOException { final String normalPathName = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) { - return getCache().isDataset(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); + return getCache().isDataset(normalPathName, getAttributesKey()); } return isDatasetFromContainer(normalPathName); } @@ -212,7 +212,7 @@ default JsonElement getAttributes(final String pathName) throws N5IOException { /* If cached, return the cache */ if (cacheMeta()) { - return getCache().getAttributes(groupPath, N5KeyValueReader.ATTRIBUTES_JSON); + return getCache().getAttributes(groupPath, getAttributesKey()); } else { return GsonKeyValueN5Reader.super.getAttributes(groupPath); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index 5e6c85d15..e327463c0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -62,9 +62,9 @@ default void createGroup(final String path) throws N5Exception { // else if exists is true (then a dataset is present) so throw an exception to avoid // overwriting / invalidating existing data if (cacheMeta()) { - if (getCache().isGroup(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) + if (getCache().isGroup(normalPath, getAttributesKey())) return; - else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { + else if (getCache().exists(normalPath, getAttributesKey())) { throw new N5Exception("Can't make a group on existing path."); } } @@ -88,8 +88,8 @@ else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { for (final String child : pathParts) { final String childPath = parent.isEmpty() ? child : parent + "/" + child; - getCache().initializeNonemptyCache(childPath, N5KeyValueReader.ATTRIBUTES_JSON); - getCache().updateCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON); + getCache().initializeNonemptyCache(childPath, getAttributesKey()); + getCache().updateCacheInfo(childPath, getAttributesKey()); // only add if the parent exists and has children cached already if (parent != null && !child.isEmpty()) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index a742cc299..e19fecb98 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -165,6 +165,6 @@ default String absoluteGroupPath(final String normalGroupPath) { */ default String absoluteAttributesPath(final String normalPath) { - return getKeyValueAccess().compose(getURI(), normalPath, N5KeyValueReader.ATTRIBUTES_JSON); + return getKeyValueAccess().compose(getURI(), normalPath, getAttributesKey()); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index a0aab927a..d678a26f2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -43,6 +43,13 @@ public interface GsonN5Reader extends N5Reader { Gson getGson(); + /** + * Get the key for the {@link KeyValueAccess}, that is used for storing attributes. + * + * @return the attributes key + */ + String getAttributesKey(); + @Override default Map> listAttributes(final String pathName) throws N5Exception { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index e86b712ae..22731edd0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -165,6 +165,12 @@ public Gson getGson() { return gson; } + @Override + public String getAttributesKey() { + + return ATTRIBUTES_JSON; + } + @Override public KeyValueAccess getKeyValueAccess() { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index 68edb9d3f..52c560fff 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -273,4 +273,9 @@ public HttpRead writer.writeSerializedBlock(object, datasetPath, datasetAttributes, gridPosition); } + + @Override + public String getAttributesKey() { + return writer.getAttributesKey(); + } } From 51e510ddf9d77f04d6cedc898fcbf1e9a93a99ae Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 26 Aug 2025 16:47:33 -0400 Subject: [PATCH 3/5] fix: remove attribute should use setAttributes * not writeAttributes * this change allows zarr v3 to re-use this method --- .../java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index f7bc044a6..ba79c847b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -218,7 +218,7 @@ default T removeAttribute(final String pathName, final String key, final Cla throw new N5Exception.N5ClassCastException(e); } if (obj != null) { - writeAttributes(normalPath, attributes); + setAttributes(normalPath, attributes); } return obj; } From bce0138f64cd4fe495d05326b6d5d933acbdb03a Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 26 Aug 2025 16:49:57 -0400 Subject: [PATCH 4/5] RawBytes type should be "bytes" * so that it can be used for zarr v3 --- .../org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java index ca34e7a9e..47486058c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/RawBytesArrayCodec.java @@ -11,7 +11,7 @@ public class RawBytesArrayCodec implements ArrayCodec { private static final long serialVersionUID = 3282569607795127005L; - public static final String TYPE = "rawbytes"; + public static final String TYPE = "bytes"; @NameConfig.Parameter(value = "endian", optional = true) private final ByteOrder byteOrder; From 7b9dee43f54c5941eb2805f00ff5134f3f48fc60 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 26 Aug 2025 16:50:44 -0400 Subject: [PATCH 5/5] feat: make registerGson an instance method * so that it can be overrided by ZarrV3Reader * remove static registerGson in GsonUtils --- .../java/org/janelia/saalfeldlab/n5/GsonUtils.java | 9 --------- .../org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 12 +++++++++++- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index e017a7c25..a683cafde 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -80,15 +80,6 @@ */ public interface GsonUtils { - static Gson registerGson(final GsonBuilder gsonBuilder) { - - gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); - gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); - gsonBuilder.disableHtmlEscaping(); - return gsonBuilder.create(); - } - /** * Reads the attributes json from a given {@link Reader}. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 22731edd0..b1bc01d36 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -33,6 +33,7 @@ import com.google.gson.JsonElement; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; +import org.janelia.saalfeldlab.n5.codec.Codec; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -127,7 +128,7 @@ protected N5KeyValueReader( throws N5Exception { this.keyValueAccess = keyValueAccess; - this.gson = GsonUtils.registerGson(gsonBuilder); + this.gson = registerGson(gsonBuilder).create(); this.cacheMeta = cacheMeta; this.cache = newCache(); @@ -159,6 +160,15 @@ private boolean inferExistence(String path) { return attributes != null || exists(path); } + protected GsonBuilder registerGson(final GsonBuilder gsonBuilder) { + + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Codec.class, NameConfigAdapter.getJsonAdapter(Codec.class)); + gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.disableHtmlEscaping(); + return gsonBuilder; + } + @Override public Gson getGson() {