Merge pull request #12 from scalableminds/fix-http-store

Author: normanrz

.github/workflows/ci.yml

@@ -19,17 +19,17 @@ jobs:
         shell: bash
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '11'
           distribution: 'temurin'
           cache: maven
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 
@@ -56,7 +56,7 @@ jobs:
       - name: Assemble JAR
         run: mvn package -DskipTests
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
-          name: jar
+          name: jar-${{ runner.os }}
           path: target/*.jar

.github/workflows/deploy.yml

@@ -7,10 +7,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Java 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '11'
           distribution: 'adopt'
@@ -32,7 +32,7 @@ jobs:
       # Begin copy from ci.yml. Refactor?
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 

README.md

@@ -17,7 +17,8 @@ Group hierarchy = Group.open(
     new HttpStore("https://static.webknossos.org/data/zarr_v3")
         .resolve("l4_sample")
 );
-Array array = hierarchy.get("color").get("1");
+Group color = (Group) hierarchy.get("color");
+Array array = (Array) color.get("1");
 ucar.ma2.Array outArray = array.read(
     new long[]{0, 3073, 3073, 513}, // offset
     new int[]{1, 64, 64, 64} // shape
@@ -31,11 +32,12 @@ Array array = Array.create(
         .withChunkShape(1, 1024, 1024, 1024)
         .withFillValue(0)
         .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc()))
-        .build();
+        .build()
 );
+ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1024, 1024, 1024});
 array.write(
     new long[]{0, 0, 0, 0}, // offset
-    ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1024, 1024, 1024})
+    data
 );
 ```
 ## Development Start-Guide

src/main/java/dev/zarr/zarrjava/store/HttpStore.java

@@ -79,7 +79,7 @@ public ByteBuffer get(String[] keys, long start, long end) {
       throw new IllegalArgumentException("Argument 'start' needs to be non-negative.");
     }
     Request request = new Request.Builder().url(resolveKeys(keys)).header(
-        "Range", String.format("Bytes=%d-%d", start, end + 1)).build();
+        "Range", String.format("Bytes=%d-%d", start, end - 1)).build();
     return get(request);
   }
 

src/main/java/dev/zarr/zarrjava/v3/Array.java

@@ -142,6 +142,11 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx
     if (shape.length != metadata.ndim()) {
       throw new IllegalArgumentException("'shape' needs to have rank '" + metadata.ndim() + "'.");
     }
+    for (int dimIdx = 0; dimIdx < metadata.ndim(); dimIdx++) {
+      if (offset[dimIdx] < 0 || offset[dimIdx] + shape[dimIdx] > metadata.shape[dimIdx]) {
+        throw new ZarrException("Requested data is outside of the array's domain.");
+      }
+    }
 
     final int[] chunkShape = metadata.chunkShape();
     if (IndexingUtils.isSingleFullChunk(offset, shape, chunkShape)) {
@@ -168,7 +173,9 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx
 
                 final String[] chunkKeys = metadata.chunkKeyEncoding.encodeChunkKey(chunkCoords);
                 final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
-
+                if (!chunkHandle.exists()) {
+                  return;
+                }
                 if (codecPipeline.supportsPartialDecode()) {
                   final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
                       Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);

src/test/java/dev/zarr/zarrjava/ZarrTest.java

@@ -6,12 +6,10 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.luben.zstd.Zstd;
 import com.github.luben.zstd.ZstdCompressCtx;
-import dev.zarr.zarrjava.store.FilesystemStore;
-import dev.zarr.zarrjava.store.HttpStore;
-import dev.zarr.zarrjava.store.S3Store;
-import dev.zarr.zarrjava.store.StoreHandle;
+import dev.zarr.zarrjava.store.*;
 import dev.zarr.zarrjava.utils.MultiArrayUtils;
 import dev.zarr.zarrjava.v3.*;
+import dev.zarr.zarrjava.v3.codec.Codec;
 import dev.zarr.zarrjava.v3.codec.CodecBuilder;
 import dev.zarr.zarrjava.v3.codec.core.BytesCodec;
 import dev.zarr.zarrjava.v3.codec.core.TransposeCodec;
@@ -65,13 +63,15 @@ public static void clearTestoutputFolder() throws IOException {
     @CsvSource({
             "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9",
             "gzip,0", "gzip,5",
-            "zstd,0_true", "zstd,5_true","zstd,0_false", "zstd,5_false",
+            "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false",
             "bytes,BIG", "bytes,LITTLE",
             "transpose,_",
             "sharding,start", "sharding,end",
             "sharding_nested,_",
             "crc32c,_",
-    })    public void testReadFromZarrita(String codec, String codecParam) throws IOException, ZarrException, InterruptedException {
+    })
+
+    public void testReadFromZarrita(String codec, String codecParam) throws IOException, ZarrException, InterruptedException {
         String command = pythonPath();
         ProcessBuilder pb = new ProcessBuilder(command, PYTHON_TEST_PATH.resolve("zarrita_write.py").toString(), codec, codecParam, TESTOUTPUT.toString());
         Process process = pb.start();
@@ -142,7 +142,7 @@ public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException
     @CsvSource({
             "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9",
             "gzip,0", "gzip,5",
-            "zstd,0_true", "zstd,5_true","zstd,0_false", "zstd,5_false",
+            "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false",
             "bytes,BIG", "bytes,LITTLE",
             "transpose,_",
             "sharding,start", "sharding,end",
@@ -233,12 +233,12 @@ public void testWriteReadWithZarrita(String codec, String codecParam) throws Exc
         assert exitCode == 0;
     }
 
-    static Stream<Function<CodecBuilder, CodecBuilder>> invalidCodecBuilder(){
+    static Stream<Function<CodecBuilder, CodecBuilder>> invalidCodecBuilder() {
         return Stream.of(
             c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE),
             c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE),
-            c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1,0}),
-            c -> c.withTranspose(new int[]{1,0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1,0})
+            c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}),
+            c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0})
         );
     }
 
@@ -247,9 +247,9 @@ static Stream<Function<CodecBuilder, CodecBuilder>> invalidCodecBuilder(){
     public void testCheckInvalidCodecConfiguration(Function<CodecBuilder, CodecBuilder> codecBuilder) throws Exception {
         StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode()));
         ArrayMetadataBuilder builder = Array.metadataBuilder()
-            .withShape(new long[] {4, 4})
+            .withShape(new long[]{4, 4})
             .withDataType(DataType.UINT32)
-            .withChunkShape(new int[]{2,2})
+            .withChunkShape(new int[]{2, 2})
             .withCodecs(codecBuilder);
 
         assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build()));
@@ -287,7 +287,7 @@ static Stream<int[]> invalidChunkSizes() {
     @ParameterizedTest
     @MethodSource("invalidChunkSizes")
     public void testCheckInvalidChunkDimensions(int[] chunkSize) {
-        long[] shape = new long[] {4, 4};
+        long[] shape = new long[]{4, 4};
 
         StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_chunksize");
         ArrayMetadataBuilder builder = Array.metadataBuilder()
@@ -312,8 +312,8 @@ static Stream<int[]> invalidShardSizes() {
     @ParameterizedTest
     @MethodSource("invalidShardSizes")
     public void testCheckShardingBounds(int[] shardSize) throws Exception {
-        long[] shape = new long[] {10, 10};
-        int[] innerChunkSize = new int[] {2, 2};
+        long[] shape = new long[]{10, 10};
+        int[] innerChunkSize = new int[]{2, 2};
 
         ArrayMetadataBuilder builder = Array.metadataBuilder()
             .withShape(shape)
@@ -371,8 +371,8 @@ static Stream<int[]> invalidTransposeOrder() {
         return Stream.of(
             new int[]{1, 0, 0},
             new int[]{1, 2, 3},
-            new int[]{1,2,3,0},
-            new int[]{1,2}
+            new int[]{1, 2, 3, 0},
+            new int[]{1, 2}
         );
     }
 
@@ -392,6 +392,7 @@ public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exceptio
         ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt);
         assertThrows(ZarrException.class, () -> transposeCodec.encode(testData));
     }
+
     @Test
     public void testFileSystemStores() throws IOException, ZarrException {
         FilesystemStore fsStore = new FilesystemStore(TESTDATA);
@@ -564,4 +565,70 @@ public void testV2() throws IOException {
     }
 
 
+    @Test
+    public void testReadme1() throws IOException, ZarrException {
+        Group hierarchy = Group.open(
+                new HttpStore("https://static.webknossos.org/data/zarr_v3")
+                        .resolve("l4_sample")
+        );
+        Group color = (Group) hierarchy.get("color");
+        Array array = (Array) color.get("1");
+        ucar.ma2.Array outArray = array.read(
+                new long[]{0, 3073, 3073, 513}, // offset
+                new int[]{1, 64, 64, 64} // shape
+        );
+    }
+
+    @Test
+    public void testReadme2() throws IOException, ZarrException {
+        Array array = Array.create(
+                new FilesystemStore(TESTOUTPUT).resolve("testoutput", "color", "1"),
+                Array.metadataBuilder()
+                        .withShape(1, 4096, 4096, 1536)
+                        .withDataType(DataType.UINT32)
+                        .withChunkShape(1, 1024, 1024, 1024)
+                        .withFillValue(0)
+                        .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc()))
+                        .build()
+        );
+        ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1, 2, 2}, new int[]{1, 2, 3, 4});
+        array.write(
+                new long[]{0, 0, 0, 0}, // offset
+                data
+        );
+        ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2});
+        assert MultiArrayUtils.allValuesEqual(data, output);
+
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"1", "2-2-1", "4-4-1", "16-16-4"})
+    public void testReadL4Sample(String mag) throws IOException, ZarrException {
+        StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/").resolve("l4_sample", "color", mag);
+        StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", mag);
+
+        Array httpArray = Array.open(httpStoreHandle);
+        Array localArray = Array.open(localStoreHandle);
+        System.out.println(httpArray);
+        System.out.println(localArray);
+
+        ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64});
+        ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64});
+
+        assert MultiArrayUtils.allValuesEqual(httpData1, localData1);
+
+        //offset to where l4_sample contains non-zero values
+        long[] offset = new long[4];
+        long[] originalOffset = new long[]{0, 3073, 3073, 513};
+        long[] originalShape = new long[]{1, 4096, 4096, 2048};
+        long[] arrayShape = httpArray.metadata.shape;
+        for (int i = 0; i < 4; i++) {
+            offset[i] = originalOffset[i] / (originalShape[i] / arrayShape[i]);
+        }
+
+        ucar.ma2.Array httpData2 = httpArray.read(offset, new int[]{1, 64, 64, 64});
+        ucar.ma2.Array localData2 = localArray.read(offset, new int[]{1, 64, 64, 64});
+
+        assert MultiArrayUtils.allValuesEqual(httpData2, localData2);
+    }
 }