From 67260d5fbf438e3ed17fa1e0c4141945dcc9cca8 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 6 Nov 2025 12:39:15 +0100 Subject: [PATCH 1/7] Improve javadoc --- .../janelia/saalfeldlab/n5/shard/Nesting.java | 120 ++++++++++++++---- 1 file changed, 97 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java index 892900ae4..208cf507a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java @@ -144,7 +144,15 @@ public String toString() { } /** - * A nested grid of blocks used to coordinate the relationships of shards and the blocks / chunks they contain. + * A nested grid of blocks used to coordinate the relationships of shards + * and the blocks / chunks they contain. + *

+ * The nesting depth ({@link #numLevels()}) of the {@code NestedGrid} is 1 + * for non-sharded datasets, 2 for simple sharded datasets (where shards + * contain datablocks), and ≥3 for nested sharded datasets. + *

+ * Positions with {@code level=0} refer to the DataBlock grid, positions + * with {@code level=1} refer to first-level Shard grid, and so on. */ public static class NestedGrid { @@ -152,12 +160,19 @@ public static class NestedGrid { private final int numDimensions; - // relativeToBase[i][d] is block size at level i relative to level 0 + /** + * relativeToBase[i][d] is block size (in dimension d) at level i relative to level 0 + */ private final int[][] relativeToBase; - // relativeToAdjacent[i][d] is block size at level i relative to level i-1 + /** + * relativeToAdjacent[i][d] is block size (in dimension d) at level i relative to level i-1 + */ private final int[][] relativeToAdjacent; + /** + * blockSizes[l][d] is the block size in pixels at level l in dimension d + */ private final int[][] blockSizes; /** @@ -227,10 +242,31 @@ public int numDimensions() { return numDimensions; } - public int[] getBlockSize(int level) { + /** + * Get the block size in pixels at the given {@code level}. + */ + public int[] getBlockSize(final int level) { return blockSizes[level]; } + /** + * Computes the absolute {@code targetPos} grid position at {@code + * targetLevel} for the given {@code sourcePos} grid position at {@code + * sourceLevel}. + *

+ * For example, this can be used to compute the coordinates on the shard + * grid ({@code targetLevel==1}) of the shard containing a given + * datablock ({@code sourcePos} at {@code sourceLevel==0}). + * + * @param sourcePos + * a grid position at {@code sourceLevel} + * @param sourceLevel + * nesting level of {@code sourcePos} + * @param targetPos + * the grid position at {@code targetLevel} will be stored here + * @param targetLevel + * nesting level of {@code targetPos} + */ public void absolutePosition( final long[] sourcePos, final int sourceLevel, @@ -243,23 +279,14 @@ public void absolutePosition( } } - public void relativePosition( - final long[] sourcePos, - final int sourceLevel, - final long[] targetPos, - final int targetLevel) { - absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel); - if (targetLevel < numLevels - 1) { - final int[] rj = relativeToAdjacent[targetLevel + 1]; - for (int d = 0; d < numDimensions; ++d) { - targetPos[d] %= rj[d]; - } - } - } - /** - * The absolute position of * source position for the given source level at the target level. - * + * Get the absolute grid position at {@code targetLevel} for the given + * {@code sourcePos} grid position at {@code sourceLevel}. + *

+ * For example, this can be used to compute the coordinates on the shard + * grid ({@code targetLevel==1}) of the shard containing a given + * datablock ({@code sourcePos} at {@code sourceLevel==0}). + * * @param sourcePos the source position j * @param sourceLevel the source level * @param targetLevel the target level @@ -275,9 +302,13 @@ public long[] absolutePosition( } /** - * The absolute position of the level 0 source position at - * the target level. - * + * Get the absolute grid position at {@code targetLevel} for the given + * {@code sourcePos} block grid position (level 0). + *

+ * For example, this can be used to compute the coordinates on the shard + * grid ({@code targetLevel==1}) of the shard containing a given + * datablock ({@code sourcePos}. + * * @param sourcePos the source position j * @param targetLevel the target level * @return absolute position at the target level @@ -288,6 +319,42 @@ public long[] absolutePosition( return absolutePosition(sourcePos, 0, targetLevel); } + /** + * Computes the {@code targetPos} grid position at {@code targetLevel} + * for the given {@code sourcePos} grid position at {@code sourceLevel}, + * relative to the containing element at {@code targetLevel+1}. + * (The containing element is a shard for {@code targetLevel+1 < + * numLevels} or the dataset for {@code targetLevel+1 == numLevels}.) + *

+ * For example, this can be used to compute the grid coordinates {@code + * targetLevel==0} of a given datablock ({@code sourcePos} at {@code + * sourceLevel==0}) withing a shard (containing element at level {@code + * targetLevel+1==1}). + *

+ * + * @param sourcePos + * a grid position at {@code sourceLevel} + * @param sourceLevel + * nesting level of {@code sourcePos} + * @param targetPos + * the grid position at {@code targetLevel} will be stored here + * @param targetLevel + * nesting level of {@code targetPos} + */ + public void relativePosition( + final long[] sourcePos, + final int sourceLevel, + final long[] targetPos, + final int targetLevel) { + absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel); + if (targetLevel < numLevels - 1) { + final int[] rj = relativeToAdjacent[targetLevel + 1]; + for (int d = 0; d < numDimensions; ++d) { + targetPos[d] %= rj[d]; + } + } + } + public long[] relativePosition( final long[] sourcePos, final int sourceLevel, @@ -303,6 +370,13 @@ public long[] relativePosition( return relativePosition(sourcePos, 0, targetLevel); } + /** + * Get size of a block at the given {@code level} relative to {@code + * level-1} (that is, in units of {@code level-1} blocks). + *

+ * For example {@code relativeBlockSize(1)} returns the number of + * datablocks in a (non-nested) shard. + */ public int[] relativeBlockSize(final int level) { return relativeToAdjacent[level]; } From edc6f6e171fff320718fb45c33e713548ec530f6 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 7 Nov 2025 19:19:13 +0100 Subject: [PATCH 2/7] Declare N5IOException in readBlocks / writeBlocks methods --- .../org/janelia/saalfeldlab/n5/shard/DatasetAccess.java | 4 ++-- .../janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index a7a2f6d99..6e7f19670 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -26,9 +26,9 @@ public interface DatasetAccess { boolean deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; - List> readBlocks(PositionValueAccess kva, List positions); + List> readBlocks(PositionValueAccess kva, List positions) throws N5IOException; - void writeBlocks(PositionValueAccess kva, List> blocks); + void writeBlocks(PositionValueAccess kva, List> blocks) throws N5IOException; NestedGrid getGrid(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 43029582d..53a07bbc5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -56,7 +56,7 @@ private DataBlock readBlockRecursive( } @Override - public List> readBlocks(PositionValueAccess pva, List positions) { + public List> readBlocks(PositionValueAccess pva, List positions) throws N5IOException { if (grid.numLevels() == 1) { return positions.stream().map(it -> readBlock(pva, it)).collect(Collectors.toList()); @@ -175,7 +175,8 @@ private ReadData writeBlockRecursive( } } - @Override public void writeBlocks(PositionValueAccess pva, List> dataBlocks) { + @Override + public void writeBlocks(final PositionValueAccess pva, final List> dataBlocks) throws N5IOException { if (grid.numLevels() == 1) { dataBlocks.forEach(it -> writeBlock(pva, it)); @@ -317,7 +318,7 @@ private ReadData deleteBlockRecursive( private static ReadData getExistingReadData(final PositionValueAccess pva, final long[] key) { // need to read the shard anyway, and currently (Sept 24 2025) - // have no way to tell if they key exist from what is in this method except to attempt + // have no way to tell if the key exists from what is in this method except to attempt // to materialize and catch the N5NoSuchKeyException try { ReadData existingData = pva.get(key); From 18a1fed6221ba9570e5a1d3e8ea91b250b7fe23f Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 10 Nov 2025 20:24:48 +0100 Subject: [PATCH 3/7] Revise Nesting NestedPosition is always constructed via NestedGrid factory methods. Added methods to determine the pixel coordinates corresponding to a NestedPosition. Cleaned up unused code and comments. --- .../saalfeldlab/n5/shard/DatasetAccess.java | 2 +- .../n5/shard/DefaultDatasetAccess.java | 8 +- .../janelia/saalfeldlab/n5/shard/Nesting.java | 177 ++++++++++++------ 3 files changed, 126 insertions(+), 61 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index 6e7f19670..b031f3bb8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -56,7 +56,7 @@ static Collection> groupInnerPositions(final if (nestedPosition.level() == outerLevel) outerNestedPosition = nestedPosition; else - outerNestedPosition = new NestedPosition(grid, nestedPosition.absolute(0), outerLevel); + outerNestedPosition = grid.nestedPosition(nestedPosition.absolute(outerLevel), outerLevel); final List blocks = blocksPerShard.computeIfAbsent(outerNestedPosition, it -> new ArrayList<>()); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 53a07bbc5..aae7746e4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -33,7 +33,7 @@ public NestedGrid getGrid() { @Override public DataBlock readBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { - final NestedPosition position = new NestedPosition(grid, gridPosition); + final NestedPosition position = grid.nestedPosition(gridPosition); return readBlockRecursive(kva.get(position.key()), position, grid.numLevels() - 1); } @@ -62,7 +62,7 @@ public List> readBlocks(PositionValueAccess pva, List posit return positions.stream().map(it -> readBlock(pva, it)).collect(Collectors.toList()); } - final List blockPositions = positions.stream().map(it -> new NestedPosition(grid, it)).collect(Collectors.toList()); + final List blockPositions = positions.stream().map(grid::nestedPosition).collect(Collectors.toList()); final int outermostLevel = grid.numLevels() - 1; final Collection> blocksPerOutermostShard = groupInnerPositions(grid, blockPositions, outermostLevel); @@ -141,7 +141,7 @@ private List> readShardRecursive( @Override public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException { - final NestedPosition position = new NestedPosition(grid, dataBlock.getGridPosition()); + final NestedPosition position = grid.nestedPosition(dataBlock.getGridPosition()); final long[] key = position.key(); final ReadData existingData = getExistingReadData(pva, key); @@ -255,7 +255,7 @@ private ReadData writeShardRecursive( @Override public boolean deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { - final NestedPosition position = new NestedPosition(grid, gridPosition); + final NestedPosition position = grid.nestedPosition(gridPosition); final long[] key = position.key(); if (grid.numLevels() == 1) { // for non-sharded dataset, don't bother getting the value, just remove the key. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java index 208cf507a..f5cfc3b81 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java @@ -3,29 +3,6 @@ import java.util.ArrayList; -// TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -// -// [ ] NestedGrid -// [ ] validation in constructor -// [ ] test for that validation -// [ ] javadoc -// -// [ ] NestedPosition interface -// [+] LazyNestedPosition class -// [+] fields: NestedGrid, long[] position, int level -// [+] construct with source level 0 -// [+] minimal abs/rel access methods -// [+] toString() -// [-] extract NestedPosition interface -// ==> postpone until necessary -// [ ] equals / hashcode -// [ ] should we have prefix()? suffix()? head()? tail()? -// [+] Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...). -// For nested = {X,Y,Z} compare by Z, then Y, then X. -// For X = {x,y,z} compare by z, then y, then x. (flattening order) -// -// TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - import java.util.Arrays; import java.util.List; @@ -33,30 +10,18 @@ public class Nesting { - public static void main(String[] args) { - final int[][] blockSizes = {{1,1,1}, {3,2,2}, {6,4,4}, {24,24,24}}; - final NestedGrid grid = new NestedGrid(blockSizes); - final NestedPosition pos = new NestedPosition(grid, new long[] {38, 7, 129}); - System.out.println("pos = " + pos); - System.out.println("key = " + Arrays.toString(pos.key())); - } - public static class NestedPosition implements Comparable { private final NestedGrid grid; private final long[] position; private final int level; - public NestedPosition(final NestedGrid grid, final long[] position, final int level) { + protected NestedPosition(final NestedGrid grid, final long[] position, final int level) { this.grid = grid; this.position = position; this.level = level; } - public NestedPosition(final NestedGrid grid, final long[] position) { - this(grid, position, 0); - } - /** * Get the nesting level of this position. *

@@ -84,7 +49,18 @@ public int numDimensions() { * @return relative grid position */ public long[] relative(final int level) { - return grid.relativePosition(position, level); + return grid.relativePosition(position, this.level, level); + } + + /** + * Get the relative grid position at this positions {@link #level()}, + * that is, relative offset within containing the {@code (level+1)} + * element. + * + * @return relative grid position + */ + public long[] relative() { + return relative(level()); } /** @@ -96,13 +72,17 @@ public long[] relative(final int level) { * @return absolute grid position */ public long[] absolute(final int level) { - return grid.absolutePosition(position, level); + return grid.absolutePosition(position, this.level, level); } public long[] key() { return relative(grid.numLevels() - 1); } + public long[] pixelPosition() { + return grid.pixelPosition(position, level); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -117,7 +97,8 @@ public String toString() { return sb.toString(); } - @Override public int compareTo(NestedPosition o) { + @Override + public int compareTo(NestedPosition o) { final int dimensionInequality = Integer.compare(numDimensions(), o.numDimensions()); if (dimensionInequality != 0) @@ -127,13 +108,10 @@ public String toString() { if (levelInequality != 0) return levelInequality; - final long[] otherAbsPos = o.absolute(level); - final long[] absPos = absolute(level); - - for (int i = absPos.length - 1; i >= 0; --i) { - final long diff = absPos[i] - otherAbsPos[i]; + for (int i = position.length - 1; i >= 0; --i) { + final long diff = position[i] - o.position[i]; if (diff != 0) - return (int)diff; + return (int) diff; } return 0; @@ -156,6 +134,52 @@ public String toString() { */ public static class NestedGrid { + /** + * Create a {@code NestedPosition} at the specified nesting {@code + * level} grid {@code position}. + *

+ * Note that {@code position} is in units of grid elements at {@code + * level}. Positions with {@code level=0} refer to the DataBlock grid, + * positions with {@code level=1} refer to first-level Shard grid, and + * so on. + *

+ * The returned {@code NestedPosition} will have + * {@link NestedPosition#level() level()==level}. + * + * @param position + * position at {@code level} + * @param level + * nesting level of {@code position} + * + * @return a NestedPosition representation of the specified grid position and nesting level + */ + public NestedPosition nestedPosition(final long[] position, final int level) { + return new NestedPosition(this, position, level); + } + + /** + * Create a {@code NestedPosition} at the specified block grid {@code + * position} (that is, at nesting level 0). + *

+ * Note that {@code position} is in units of DataBlocks. + *

+ * The returned {@code NestedPosition} will have + * {@link NestedPosition#level() level()==0}. + * + * @param position + * position at level 0 (block grid) + * + * @return a NestedPosition representation of the specified block grid position + */ + public NestedPosition nestedPosition(final long[] position) { + return nestedPosition(position, 0); + } + + + + + + private final int numLevels; private final int numDimensions; @@ -249,6 +273,46 @@ public int[] getBlockSize(final int level) { return blockSizes[level]; } + /** + * Computes the pixel position for the given {@code sourcePos} grid + * position at {@code sourceLevel}. + * + * @param sourcePos + * a grid position at {@code sourceLevel} + * @param sourceLevel + * nesting level of {@code sourcePos} + * @param targetPos + * the pixel position will be stored here + */ + public void pixelPosition( + final long[] sourcePos, + final int sourceLevel, + final long[] targetPos) { + final int[] s = blockSizes[sourceLevel]; + for (int d = 0; d < numDimensions; ++d) { + targetPos[d] = sourcePos[d] * s[d]; + } + } + + /** + * Get the pixel position for the given {@code sourcePos} grid position + * at {@code sourceLevel}. + * + * @param sourcePos + * a grid position at {@code sourceLevel} + * @param sourceLevel + * nesting level of {@code sourcePos} + * + * @return the pixel position + */ + public long[] pixelPosition( + final long[] sourcePos, + final int sourceLevel) { + final long[] targetPos = new long[numDimensions]; + pixelPosition(sourcePos, sourceLevel, targetPos); + return targetPos; + } + /** * Computes the absolute {@code targetPos} grid position at {@code * targetLevel} for the given {@code sourcePos} grid position at {@code @@ -296,6 +360,9 @@ public long[] absolutePosition( final long[] sourcePos, final int sourceLevel, final int targetLevel) { + if (sourceLevel == targetLevel) { + return sourcePos; + } final long[] targetPos = new long[numDimensions]; absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel); return targetPos; @@ -381,10 +448,6 @@ public int[] relativeBlockSize(final int level) { return relativeToAdjacent[level]; } - public int[] absoluteBlockSize(final int level) { - return relativeToBase[level]; - } - /** * Given a block position at a particular level, returns a list of * positions of all sub-blocks at a particular subLevel. @@ -400,22 +463,24 @@ public int[] absoluteBlockSize(final int level) { * the nesting sub-level of positions to return * @return the sub-block positions */ + // TODO: rename to positionsInSubGrid public List positionInSubGrid(long[] position, int level, int subLevel) { - final long[] subPosition = new long[numDimensions()]; - absolutePosition(position, level, subPosition, subLevel); + // find the starting (subLevel grid) coordinates corresponding to the + // first subLevel block in the element at the given level and position + final long[] subPosition = absolutePosition(position, level, subLevel); - final int[] numElementsInSubGrid = absoluteBlockSize(numLevels() - 1); - final GridIterator git = new GridIterator(GridIterator.int2long(numElementsInSubGrid), subPosition); + // find the dimensions (number of subLevel blocks in one level block, along each dimension) + final long[] one = new long[numDimensions]; + Arrays.fill(one, 1); + final long[] dimensions = absolutePosition(one, level, subLevel); - // TODO return NestedPositions instead? + final GridIterator git = new GridIterator(dimensions, subPosition); final ArrayList positions = new ArrayList<>(); while (git.hasNext()) positions.add(git.next().clone()); return positions; } - } - } From e6e390de85c87e2ad85964024add4f3022b0d5e3 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 10 Nov 2025 20:38:51 +0100 Subject: [PATCH 4/7] Add N5Writer.writeRegion methods writeRegion() takes a DataBlockSupplier and a region in pixel coordinates that should be written. It decised which shards/blocks need to be touched and asks the DataBlockSupplier for the respective blocks. Currently only minimal parallelization (over shards) is implemented. This could be improved in the future, if called with a ForkJoinPool executor. Then we could further parallelize the production and encoding of DataBlocks. --- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 38 +++- .../org/janelia/saalfeldlab/n5/N5Writer.java | 55 +++++- .../saalfeldlab/n5/shard/DatasetAccess.java | 69 ++++++- .../n5/shard/DefaultDatasetAccess.java | 95 +++++++++- .../janelia/saalfeldlab/n5/shard/Region.java | 174 ++++++++++++++++++ .../saalfeldlab/n5/shard/WriteRegionTest.java | 164 +++++++++++++++++ 6 files changed, 585 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index d0a9c0e66..960a5970b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -35,6 +35,8 @@ import java.util.Map; import com.google.gson.JsonSyntaxException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; @@ -210,6 +212,41 @@ default boolean removeAttributes(final String pathName, final List attri return removed; } + @Override + default void writeRegion( + final String datasetPath, + final DatasetAttributes datasetAttributes, + final long[] min, + final long[] size, + final DataBlockSupplier dataBlocks, + final boolean writeFully) throws N5Exception { + try { + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), datasetAttributes); + datasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, datasetAttributes.getDimensions(), writeFully); + } catch (final UncheckedIOException e) { + throw new N5IOException( + "Failed to write blocks into dataset " + datasetPath, e); + } + } + + @Override + default void writeRegion( + final String datasetPath, + final DatasetAttributes datasetAttributes, + final long[] min, + final long[] size, + final DataBlockSupplier dataBlocks, + final boolean writeFully, + final ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException { + try { + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), datasetAttributes); + datasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, datasetAttributes.getDimensions(), writeFully, exec); + } catch (final UncheckedIOException e) { + throw new N5IOException( + "Failed to write blocks into dataset " + datasetPath, e); + } + } + @Override default void writeBlocks( final String datasetPath, @@ -239,7 +276,6 @@ default void writeBlock( "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, e); } - } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 6c3112441..222092de8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -28,9 +28,6 @@ */ package org.janelia.saalfeldlab.n5; -import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; @@ -39,6 +36,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; /** * A simple structured container API for hierarchies of chunked @@ -265,6 +264,56 @@ default void writeBlocks( writeBlock(datasetPath, datasetAttributes, block); } + @FunctionalInterface + interface DataBlockSupplier { + + /** + * + * @param gridPos + * @param existingDataBlock + * existing data to be merged into the new data block (maybe {@code null}) + * + * @return data block at the given gridPos + */ + DataBlock get(long[] gridPos, final DataBlock existingDataBlock); + } + + /** + * @param datasetPath the dataset path + * @param datasetAttributes the dataset attributes + * @param min min pixel coordinate of region to write + * @param size size in pixels of region to write + * @param dataBlocks is asked to create blocks within the given region + * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * @throws N5Exception the exception + */ + void writeRegion( + String datasetPath, + DatasetAttributes datasetAttributes, + long[] min, + long[] size, + DataBlockSupplier dataBlocks, + boolean writeFully) throws N5Exception; + + /** + * @param datasetPath the dataset path + * @param datasetAttributes the dataset attributes + * @param min min pixel coordinate of region to write + * @param size size in pixels of region to write + * @param dataBlocks is asked to create blocks within the given region + * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * @param exec used to parallelize over blocks and shards + * @throws N5Exception the exception + */ + void writeRegion( + String datasetPath, + DatasetAttributes datasetAttributes, + long[] min, + long[] size, + DataBlockSupplier dataBlocks, + boolean writeFully, + ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException; + /** * Deletes the block at {@code gridPosition}. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index b031f3bb8..2b68b24d9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -1,15 +1,18 @@ package org.janelia.saalfeldlab.n5.shard; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; -import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.TreeMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.N5Writer.DataBlockSupplier; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; /** * Wrap an instantiated DataBlock/shard codec hierarchy to implement (single and @@ -26,10 +29,66 @@ public interface DatasetAccess { boolean deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; + List> readBlocks(PositionValueAccess kva, List positions) throws N5IOException; void writeBlocks(PositionValueAccess kva, List> blocks) throws N5IOException; + + /** + * + * @param pva + * @param min + * min pixel coordinate of region to write + * @param size + * size in pixels of region to write + * @param blocks + * is asked to create blocks within the given region + * @param datasetDimensions + * @param writeFully + * if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * + * @throws N5IOException + */ + void writeRegion( + PositionValueAccess pva, + long[] min, + long[] size, + DataBlockSupplier blocks, + long[] datasetDimensions, + boolean writeFully + ) throws N5IOException; + + /** + * + * @param pva + * @param min + * min pixel coordinate of region to write + * @param size + * size in pixels of region to write + * @param blocks + * is asked to create blocks within the given region. must be thread-safe. + * @param datasetDimensions + * @param writeFully + * if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * @param exec + * used to parallelize over blocks and shards + * + * @throws N5Exception + * @throws InterruptedException + * @throws ExecutionException + */ + void writeRegion( + PositionValueAccess pva, + long[] min, + long[] size, + DataBlockSupplier blocks, + long[] datasetDimensions, + boolean writeFully, + ExecutorService exec + ) throws N5Exception, InterruptedException, ExecutionException; + + NestedGrid getGrid(); /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index aae7746e4..2a975d694 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -5,17 +5,20 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; +import org.janelia.saalfeldlab.n5.N5Writer.DataBlockSupplier; import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; -import static org.janelia.saalfeldlab.n5.shard.DatasetAccess.*; +import static org.janelia.saalfeldlab.n5.shard.DatasetAccess.groupInnerPositions; public class DefaultDatasetAccess implements DatasetAccess { @@ -253,6 +256,96 @@ private ReadData writeShardRecursive( return codec.encode(new RawShardDataBlock(shardPosition, shard)); } + public void writeRegion( + final PositionValueAccess pva, + final long[] min, + final long[] size, + final DataBlockSupplier blocks, + final long[] datasetDimensions, + final boolean writeFully + ) throws N5IOException { + + final Region region = new Region(min, size, grid, datasetDimensions); + + for (long[] key : Region.gridPositions(region.minPos().key(), region.maxPos().key())) { + final NestedPosition pos = grid.nestedPosition(key, grid.numLevels() - 1); + final boolean nestedWriteFully = writeFully || region.fullyContains(pos); + final ReadData existingData = nestedWriteFully ? null : getExistingReadData(pva, key); + final ReadData modifiedData = writeRegionRecursive(existingData, region, blocks, pos); + pva.put(key, modifiedData); + } + } + + @Override + public void writeRegion( + final PositionValueAccess pva, + final long[] min, + final long[] size, + final DataBlockSupplier blocks, + final long[] datasetDimensions, + final boolean writeFully, + final ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException { + + final Region region = new Region(min, size, grid, datasetDimensions); + + for (long[] key : Region.gridPositions(region.minPos().key(), region.maxPos().key())) { + exec.submit(() -> { + final NestedPosition pos = grid.nestedPosition(key, grid.numLevels() - 1); + final boolean nestedWriteFully = writeFully || region.fullyContains(pos); + final ReadData existingData = nestedWriteFully ? null : getExistingReadData(pva, key); + final ReadData modifiedData = writeRegionRecursive(existingData, region, blocks, pos); + pva.put(key, modifiedData); + }); + } + } + + private ReadData writeRegionRecursive( + final ReadData existingReadData, // may be null + final Region region, + final DataBlockSupplier blocks, + final NestedPosition position + ) { + final boolean writeFully = existingReadData == null; + final int level = position.level(); + String indent = ""; + for (int i = 0; i < 2-level; ++i) + indent += " "; + System.out.println(indent + "position = " + position + (writeFully ? " (writeFully)" : "")); + if ( level == 0 ) { + + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[0]; + final long[] gridPosition = position.absolute(0); + + // If the DataBlock is not fully contained in the region, we will + // get existingReadData != null. In that case, we decode the + // existing DataBlock and pass it to the BlockSupplier for modification. + final DataBlock existingDataBlock = (existingReadData == null) + ? null + : codec.decode(existingReadData, gridPosition); + final DataBlock dataBlock = blocks.get(gridPosition, existingDataBlock); + + return codec.encode(dataBlock); + } else { + + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[level]; + final long[] gridPos = position.absolute(level); + final RawShard shard = existingReadData == null ? + new RawShard(grid.relativeBlockSize(level)) : + codec.decode(existingReadData, gridPos).getData(); + for (NestedPosition pos : region.containedNestedPositions(position)) { + final boolean nestedWriteFully = writeFully || region.fullyContains(pos); + final long[] elementPos = pos.relative(); + final ReadData existingElementData = nestedWriteFully ? null : shard.getElementData(elementPos); + final ReadData modifiedElementData = writeRegionRecursive(existingElementData, region, blocks, pos); + shard.setElementData(modifiedElementData, elementPos); + } + + return codec.encode(new RawShardDataBlock(gridPos, shard)); + } + } + @Override public boolean deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { final NestedPosition position = grid.nestedPosition(gridPosition); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java new file mode 100644 index 000000000..cadb58347 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java @@ -0,0 +1,174 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Bounds, in pixel coordinates, of a region in a dataset. + *

+ * Provides methods to find which blocks and shards are contained in the + * region, iterate sub-NestedPositions, etc. + */ +class Region { + + /** + * The dimensions of the full dataset. + * This is used to decide whether DataBlocks are on the border (and therefore possibly truncated). + *

+ * TODO: + * Currently, the constructor expands this to {@code min + size} (if that is larger than the provided datasetDimensions). + * Do we want this, or should we rather throw an {@code }IllegalArgumentException} in that case? + */ + private final long[] datasetDimensions; + + /** + * The nested grid of the dataset + */ + private final Nesting.NestedGrid grid; + + /** + * min pixel position in the region + */ + private final long[] min; + + /** + * size of the region in pixels + */ + private final long[] size; + + /** + * {@code NestedPosition} of the block containing the min pixel position. + */ + private final Nesting.NestedPosition minPos; + + /** + * {@code NestedPosition} of the block containing the max pixel position. + */ + private final Nesting.NestedPosition maxPos; + + Region(final long[] min, final long[] size, final Nesting.NestedGrid grid, final long[] datasetDimensions) { + this.min = min; + this.size = size; + this.grid = grid; + + final int n = min.length; + this.datasetDimensions = new long[n]; + Arrays.setAll(this.datasetDimensions, d -> Math.max(min[d] + size[d], datasetDimensions[d])); + + final int[] blockSize = grid.getBlockSize(0); + + final long[] minBlock = new long[n]; + Arrays.setAll(minBlock, d -> min[d] / blockSize[d]); + minPos = grid.nestedPosition(minBlock); + + final long[] maxBlock = new long[n]; + Arrays.setAll(maxBlock, d -> (min[d] + size[d] - 1) / blockSize[d]); + maxPos = grid.nestedPosition(maxBlock); + } + + /** + * Get the {@code NestedPosition} of the minimum DataBlock touched by the region. + */ + Nesting.NestedPosition minPos() { + return minPos; + } + + /** + * Get the {@code NestedPosition} of the maximum DataBlock touched by the region. + */ + Nesting.NestedPosition maxPos() { + return maxPos; + } + + /** + * Check whether the Shard or DataBlock corresponding to the given + * position is fully contained inside the region. + * (The {@link Nesting.NestedPosition#level() level} of {@code position} is used + * to determine whether it refers to a DataBlock or a (potentially + * nested) Shard. + * + * @param position + * the NestedPosition to check + * + * @return true, if the given position is fully contained in this region + */ + boolean fullyContains(final Nesting.NestedPosition position) { + + final long[] pmin = position.pixelPosition(); + for (int d = 0; d < pmin.length; d++) { + if (pmin[d] < min[d]) { + return false; + } + } + + final long[] pmax = maxPixelPos(position); + for (int d = 0; d < pmax.length; d++) { + final long m = Math.min(pmax[d], datasetDimensions[d] - 1); + if (m > min[d] + size[d] - 1) { + return false; + } + } + + return true; + } + + // TODO: Revise. Inline? Should this be method of NestedPosition? + private long[] maxPixelPos(final Nesting.NestedPosition position) { + final long[] pos = position.pixelPosition(); + final int[] elementSize = grid.getBlockSize(position.level()); + Arrays.setAll(pos, d -> pos[d] + elementSize[d] - 1); + return pos; + } + + /** + * Returns {@code NestedPosition}s of all nested elements in the given + * {@code position} that are contained in this {@code Region}. + *

+ * The returned {@code NestedPosition}s will all have level {@code + * position.level()-1}. + */ + // TODO: Revise to accept Consumer for handling each position + List containedNestedPositions(final Nesting.NestedPosition position) { + final int level = position.level() - 1; + final long[] gridMinOfRegion = minPos().absolute(level); + final long[] gridMaxOfRegion = maxPos().absolute(level); + + final long[] gridMinOfPosition = position.absolute(level); + final int[] gridSizeOfPosition = grid.relativeBlockSize(level + 1); + + final int n = grid.numDimensions(); + final long[] gridMin = new long[n]; + Arrays.setAll(gridMin, d -> Math.max(gridMinOfRegion[d], gridMinOfPosition[d])); + final long[] gridMax = new long[n]; + Arrays.setAll(gridMax, d -> Math.min(gridMaxOfRegion[d], gridMinOfPosition[d] + gridSizeOfPosition[d] - 1)); + + final List gridPositions = gridPositions(gridMin, gridMax); + final List nestedPositions = new ArrayList<>(); + gridPositions.forEach(p -> nestedPositions.add(grid.nestedPosition(p, level))); + return nestedPositions; + } + + // TODO: Can this fully replace GridIterator? + // TODO: Revise to accept Consumer for handling each position + static List gridPositions(final long[] min, final long[] max) { + final int n = min.length; + final long[] pos = min.clone(); + int numElements = 1; + for (int d = 0; d < n; ++d) { + numElements *= (int) (max[d] - min[d] + 1); + } + long[][] positions = new long[numElements][n]; + Arrays.setAll(positions[0], j -> pos[j]); + for (int i = 1; i < numElements; i++) { + for (int d = 0; d < n; ++d) { + if (++pos[d] <= max[d]) { + Arrays.setAll(positions[i], j -> pos[j]); + break; + } + pos[d] = min[d]; + } + } + return Arrays.asList(positions); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java new file mode 100644 index 000000000..0fc1c8546 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java @@ -0,0 +1,164 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.Arrays; +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.N5Writer.DataBlockSupplier; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; + +public class WriteRegionTest { + + +// #...............................#...............................#...............................#...............................# +// $.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$ +// |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...| +// 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96 + + public static void main(String[] args) { + +// int[] datablockSize = {3, 3, 3}; +// int[] level1ShardSize = {6, 6, 6}; +// int[] level2ShardSize = {24, 24, 24}; + int[] datablockSize = {3}; + int[] level1ShardSize = {6}; + int[] level2ShardSize = {24}; + + // DataBlocks are 3x3x3 + // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) + // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) + final BlockCodecInfo c0 = new N5BlockCodecInfo(); + final ShardCodecInfo c1 = new DefaultShardCodecInfo( + datablockSize, + c0, + new DataCodecInfo[] {new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[] {new RawCompression()}, + IndexLocation.END + ); + final ShardCodecInfo c2 = new DefaultShardCodecInfo( + level1ShardSize, + c1, + new DataCodecInfo[] {new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[] {new RawCompression()}, + IndexLocation.START + ); + + TestDatasetAttributes attributes = new TestDatasetAttributes( + new long[] {}, + level2ShardSize, + DataType.INT8, + c2, + new RawCompression()); + + final DatasetAccess datasetAccess = attributes.datasetAccess(); + final PositionValueAccess store = new TestPositionValueAccess(); + + // --------------------------------------------------------------- + // Some "tests" + // TODO: Turn into unit tests + // --------------------------------------------------------------- + + // write some blocks, filled with constant values + final int[] dataBlockSize = c1.getInnerBlockSize(); +// final long[] datasetDimensions = {100, 100, 100}; +// final long[] regionMin = {9,9,9}; +// final long[] regionSize = {15,15,15}; + final long[] datasetDimensions = {96}; + + // #...............................#...............................#...............................#...............................# + // $.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$ + // |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...| + // 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96 + + final long[] regionMin = {93}; + final long[] regionSize = {1}; + + DataBlockSupplier blocks = (gridPos, existing) -> { +// System.out.println("BlockSupplier.get(" + Arrays.toString(gridPos) + ", " + existing + ")"); + return createDataBlock(dataBlockSize, gridPos.clone(), (byte) gridPos[0]); + }; + datasetAccess.writeRegion(store, + regionMin, + regionSize, + blocks, + datasetDimensions, + false); + + // verify that the written blocks can be read back with the correct values +// checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), true, 1); +// checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); +// checkBlock(datasetAccess.readBlock(store, new long[] {0, 1, 0}), true, 3); +// checkBlock(datasetAccess.readBlock(store, new long[] {1, 1, 0}), true, 4); +// checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5); +// checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6); + + // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) +// datasetAccess.deleteBlock(store, new long[] {0, 0, 0}); +// checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); +// checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); + + // if a shard becomes empty the corresponding key should be deleted +// if ( store.get(new long[] {1, 0, 0}) == null ) { +// throw new IllegalStateException("expected non-null readData"); +// } +// datasetAccess.deleteBlock(store, new long[] {8, 4, 1}); +// if ( store.get(new long[] {1, 0, 0}) != null ) { +// throw new IllegalStateException("expected null readData"); +// } + + // deleting a non-existent block should not fail +// datasetAccess.deleteBlock(store, new long[] {0, 0, 8}); + + System.out.println("all good"); + } + + private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { + + if (dataBlock == null) { + if (expectedNonNull) { + throw new IllegalStateException("expected non-null dataBlock"); + } + } else { + if (!expectedNonNull) { + throw new IllegalStateException("expected null dataBlock"); + } + final byte[] bytes = dataBlock.getData(); + for (byte b : bytes) { + if (b != (byte) expectedFillValue) { + throw new IllegalStateException("expected all values to be " + expectedFillValue); + } + } + } + } + + private static DataBlock createDataBlock(int[] size, long[] gridPosition, int fillValue) { + final byte[] bytes = new byte[DataBlock.getNumElements(size)]; + Arrays.fill(bytes, (byte) fillValue); + return new ByteArrayDataBlock(size, gridPosition, bytes); + } + + public static class TestDatasetAttributes extends DatasetAttributes { + + public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType dataType, BlockCodecInfo blockCodecInfo, + DataCodecInfo... dataCodecInfos) { + + super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); + } + + public DatasetAccess datasetAccess() { + + // to make this accessible for the test + return createDatasetAccess(); + } + + } + +} From d6f97552c63f6ede207dde052c16a4d02a8a3800 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 10 Nov 2025 20:42:42 +0100 Subject: [PATCH 5/7] Add N5Writer.deleteBlocks method with trivial default implementation This should be handled by a dedicated DatasetAccess method --- .../org/janelia/saalfeldlab/n5/N5Writer.java | 18 ++++++++++++++++++ .../saalfeldlab/n5/shard/DatasetAccess.java | 3 +++ 2 files changed, 21 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 222092de8..3f15be8f9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -327,6 +327,24 @@ boolean deleteBlock( final String datasetPath, final long... gridPosition) throws N5Exception; + /** + * Deletes the blocks at the given {@code gridPositions}. + * + * @param datasetPath dataset path + * @param gridPositions a list of grid positions + * @return {@code true} if any of the specified blocks existed and was deleted + * @throws N5Exception if any of the block exists but could not be deleted + */ + default boolean deleteBlocks( + final String datasetPath, + final List gridPositions) throws N5Exception { + boolean deleted = false; + for (long[] pos : gridPositions) { + deleted |= deleteBlock(datasetPath, pos); + } + return deleted; + } + /** * Save a {@link Serializable} as an N5 {@link DataBlock} at a given offset. * The diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index 2b68b24d9..3b1cedefe 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -34,6 +34,9 @@ public interface DatasetAccess { void writeBlocks(PositionValueAccess kva, List> blocks) throws N5IOException; +// TODO: +// boolean deleteBlocks(PositionValueAccess kva, List positions) throws N5IOException; + /** * From cf8ee4eb092cc2d9202b215905437c4cfe0fe6da Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 10 Nov 2025 20:43:40 +0100 Subject: [PATCH 6/7] Clean up WIP --- .../janelia/saalfeldlab/n5/shard/Nesting.java | 40 ---- .../saalfeldlab/n5/util/FinalPosition.java | 38 ---- .../saalfeldlab/n5/util/GridIterator.java | 178 ------------------ .../janelia/saalfeldlab/n5/util/Position.java | 66 ------- .../saalfeldlab/n5/demo/BlockIterators.java | 93 --------- .../saalfeldlab/n5/shard/RawShardTest.java | 4 +- .../saalfeldlab/n5/shard/ShardIndexTest.java | 105 ----------- 7 files changed, 1 insertion(+), 523 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/util/Position.java delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java index f5cfc3b81..146f650bd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java @@ -1,12 +1,7 @@ package org.janelia.saalfeldlab.n5.shard; -import java.util.ArrayList; - import java.util.Arrays; -import java.util.List; - -import org.janelia.saalfeldlab.n5.util.GridIterator; public class Nesting { @@ -447,40 +442,5 @@ public long[] relativePosition( public int[] relativeBlockSize(final int level) { return relativeToAdjacent[level]; } - - /** - * Given a block position at a particular level, returns a list of - * positions of all sub-blocks at a particular subLevel. - *

- * Can be used to get a list of chunk positions for a shard with a - * particular position. - * - * @param position - * a position - * @param level - * the nesting level of the given position - * @param subLevel - * the nesting sub-level of positions to return - * @return the sub-block positions - */ - // TODO: rename to positionsInSubGrid - public List positionInSubGrid(long[] position, int level, int subLevel) { - - // find the starting (subLevel grid) coordinates corresponding to the - // first subLevel block in the element at the given level and position - final long[] subPosition = absolutePosition(position, level, subLevel); - - // find the dimensions (number of subLevel blocks in one level block, along each dimension) - final long[] one = new long[numDimensions]; - Arrays.fill(one, 1); - final long[] dimensions = absolutePosition(one, level, subLevel); - - final GridIterator git = new GridIterator(dimensions, subPosition); - final ArrayList positions = new ArrayList<>(); - while (git.hasNext()) - positions.add(git.next().clone()); - - return positions; - } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java b/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java deleted file mode 100644 index 1b7076d54..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.janelia.saalfeldlab.n5.util; - -/* - * An immutable {@Position}. - */ -public class FinalPosition implements Position { - - public final long[] position; - - public FinalPosition(long[] position) { - this.position = position; - } - - public FinalPosition(Position p) { - this.position = p.get().clone(); - } - - @Override - public long[] get() { - return position; - } - - @Override - public long get(int i) { - return position[i]; - } - - @Override - public String toString() { - return Position.toString(this); - } - - @Override - public boolean equals(Object obj) { - return Position.equals(this, obj); - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java deleted file mode 100644 index 43258efaf..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java +++ /dev/null @@ -1,178 +0,0 @@ -package org.janelia.saalfeldlab.n5.util; - -import java.util.Iterator; - -/** - * Essentially imglib2's IntervalIterator, but N5 does not depend on imglib2. - */ -public class GridIterator implements Iterator { - - final protected long[] dimensions; - - final protected long[] steps; - - final protected long[] position; - - final protected int[] intPosition; - - final protected long[] min; - - final protected int lastIndex; - - protected int index = -1; - - public GridIterator(final long[] dimensions, final long[] min) { - - final int n = dimensions.length; - this.dimensions = new long[n]; - this.position = new long[n]; - this.intPosition = new int[n]; - this.min = min; - steps = new long[n]; - - final int m = n - 1; - long k = steps[0] = 1; - for (int d = 0; d < m; ) { - final long dimd = dimensions[d]; - this.dimensions[d] = dimd; - k *= dimd; - steps[++d] = k; - } - final long dimm = dimensions[m]; - this.dimensions[m] = dimm; - lastIndex = (int)(k * dimm - 1); - } - - public GridIterator(final long[] dimensions) { - - this(dimensions, new long[dimensions.length]); - } - - public GridIterator(final int[] dimensions) { - - this(int2long(dimensions)); - } - - public void fwd() { - - ++index; - } - - public void reset() { - - index = -1; - } - - @Override - public boolean hasNext() { - - return index < lastIndex; - } - - @Override - public long[] next() { - - fwd(); - indexToPosition(index, dimensions, min, position); - return position; - } - - public int[] nextInt() { - - next(); - long2int(position, intPosition); - return intPosition; - } - - public int getIndex() { - - return index; - } - - public static void indexToPosition(long index, final long[] dimensions, final long[] offset, - final long[] position) { - - for (int dim = 0; dim < dimensions.length; dim++) { - position[dim] = (index % dimensions[dim]) + offset[dim]; - index /= dimensions[dim]; - } - } - - public static void indexToPosition(long index, final int[] dimensions, final long[] offset, - final long[] position) { - - for (int dim = 0; dim < dimensions.length; dim++) { - position[dim] = (index % dimensions[dim]) + offset[dim]; - index /= dimensions[dim]; - } - } - - final static public long positionToIndex(final long[] dimensions, final long[] position) { - long idx = 0; - int cumulativeSize = 1; - for (int i = 0; i < position.length; i++) { - idx += position[i] * cumulativeSize; - cumulativeSize *= dimensions[i]; - } - return idx; - } - - final static public long positionToIndex(final long[] dimensions, final int[] position) { - long idx = 0; - int cumulativeSize = 1; - for (int i = 0; i < position.length; i++) { - idx += position[i] * cumulativeSize; - cumulativeSize *= dimensions[i]; - } - return idx; - } - - final static public long positionToIndex(final int[] dimensions, final long[] position) { - long idx = 0; - int cumulativeSize = 1; - for (int i = 0; i < position.length; i++) { - idx += position[i] * cumulativeSize; - cumulativeSize *= dimensions[i]; - } - return idx; - } - - final static public long positionToIndex(final int[] dimensions, final int[] position) { - long idx = 0; - int cumulativeSize = 1; - for (int i = 0; i < position.length; i++) { - idx += position[i] * cumulativeSize; - cumulativeSize *= dimensions[i]; - } - return idx; - } - - public static int[] long2int(final long[] src, final int[] tgt) { - - for (int d = 0; d < tgt.length; ++d) - tgt[d] = (int)src[d]; - - return tgt; - } - - public static int[] long2int(final long[] a) { - - final int[] i = new int[a.length]; - - for (int d = 0; d < a.length; ++d) - i[d] = (int)a[d]; - - return i; - } - - public static long[] int2long(final int[] i) { - - final long[] l = new long[i.length]; - - for (int d = 0; d < l.length; ++d) - l[d] = i[d]; - - return l; - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java deleted file mode 100644 index 2403835b9..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.janelia.saalfeldlab.n5.util; - -import java.util.Arrays; - -/* - * A wrapper around a primitive long array that is lexicographically {@link Comparable} - * and for which we can test equality. - */ -public interface Position extends Comparable { - - long[] get(); - - long get(int i); - - default int numDimensions() { - return get().length; - } - - @Override - default int compareTo(Position other) { - - // use Arrays.compare when we update to Java 9+ - final int N = numDimensions() > other.numDimensions() ? numDimensions() : other.numDimensions(); - for (int i = 0; i < N; i++) { - final long diff = get(i) - other.get(i); - if (diff != 0) - return (int) diff; - } - return 0; - } - - static boolean equals(final Position a, final Object b) { - - if (a == null && b == null) - return true; - - if (b == null) - return false; - - if (!(b instanceof Position)) - return false; - - final Position other = (Position) b; - if (other.numDimensions() != a.numDimensions()) - return false; - - for (int i = 0; i < a.numDimensions(); i++) - if (other.get(i) != a.get(i)) - return false; - - return true; - } - - static String toString(Position p) { - return "Position: " + Arrays.toString(p.get()); - } - - static Position wrap(final long[] p) { - return new FinalPosition(p); - } - - static Position wrap(final int[] p) { - return new FinalPosition(GridIterator.int2long(p)); - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java deleted file mode 100644 index 20b69082a..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.janelia.saalfeldlab.n5.demo; - -import java.util.Arrays; -import java.util.Iterator; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.RawCompression; -import org.janelia.saalfeldlab.n5.codec.CodecInfo; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo; -import org.janelia.saalfeldlab.n5.util.GridIterator; - -public class BlockIterators { - - public static void main(String[] args) { - -// blockIterator(); -// shardBlockIterator(); - } - - public static void shardBlockIterator() { - -// final DatasetAttributes attrs = new DatasetAttributes( -// new long[] {12, 8}, // image size -// new int[] {6, 4}, // shard size -// new int[] {2, 2}, // block size -// DataType.UINT8, -// new ShardingCodec( -// new int[] {2, 2}, -// new CodecInfo[] { new N5BlockCodecInfo() }, -// new DeterministicSizeCodecInfo[] { new RawBlockCodecInfo() }, -// IndexLocation.END -// )); -// -// shardPositions(attrs) -// .forEach(x -> System.out.println(Arrays.toString(x))); - } - -// public static void blockIterator() { -// -// final DatasetAttributes attrs = new DatasetAttributes( -// new long[] {12, 8}, -// new int[] {2, 2}, -// DataType.UINT8, -// new RawCompression()); -// -// blockPositions(attrs).forEach(x -> System.out.println(Arrays.toString(x))); -// } -// -// public static long[] blockGridSize(final DatasetAttributes attrs ) { -// // this could be a nice method for DatasetAttributes -// -// return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getBlockSize()[i])).toArray(); -// -// } -// -// public static long[] shardGridSize(final DatasetAttributes attrs ) { -// // this could be a nice method for DatasetAttributes -// -// return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getShardSize()[i])).toArray(); -// -// } -// -// public static Stream blockPositions( DatasetAttributes attrs ) { -// return toStream(new GridIterator(blockGridSize(attrs))); -// } -// -// public static Stream shardPositions( DatasetAttributes attrs ) { -// -// final int[] blocksPerShard = attrs.getBlocksPerShard(); -// return toStream( new GridIterator(shardGridSize(attrs))) -// .flatMap( shardPosition -> { -// -// final int nd = attrs.getNumDimensions(); -// final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]); -// return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); -// }); -// } -// -// public static Stream toStream( final Iterator it ) { -// return StreamSupport.stream( Spliterators.spliteratorUnknownSize( -// it, Spliterator.ORDERED), -// false); -// } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java index eaa895420..bf0721499 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java @@ -51,8 +51,6 @@ public static void main(String[] args) { final DatasetAccess datasetAccess = attributes.datasetAccess(); - // TODO: N5Reader/Writer needs to provide a PositionValueAccess implementation on top of its KVA. - // The read/write/deleteBlock methods would getDataAccess() from the DatasetAttributes and call it with that PositionValueAccess. final PositionValueAccess store = new TestPositionValueAccess(); @@ -122,7 +120,7 @@ private static DataBlock createDataBlock(int[] size, long[] gridPosition Arrays.fill(bytes, (byte) fillValue); return new ByteArrayDataBlock(size, gridPosition, bytes); } - + public static class TestDatasetAttributes extends DatasetAttributes { public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType dataType, BlockCodecInfo blockCodecInfo, diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java deleted file mode 100644 index ea1406d83..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ /dev/null @@ -1,105 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; -import org.janelia.saalfeldlab.n5.N5FSTest; -import org.janelia.saalfeldlab.n5.N5KeyValueWriter; -import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter; -import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; -import org.janelia.saalfeldlab.n5.util.GridIterator; -import org.junit.After; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Paths; - -import static org.junit.Assert.assertEquals; - -public class ShardIndexTest { - - private static final N5FSTest tempN5Factory = new N5FSTest(); - - @After - public void removeTempWriters() { - - tempN5Factory.removeTempWriters(); - } - - @Test - @Ignore - public void testOffsetIndex() { - - // TODO -// int[] shardBlockGridSize = new int[]{5, 4, 3}; -// ShardIndex index = new ShardIndex( -// shardBlockGridSize, -// IndexLocation.END, new RawBlockCodecInfo()); -// -// GridIterator it = new GridIterator(shardBlockGridSize); -// int i = 0; -// while (it.hasNext()) { -// int j = index.getOffsetIndex(GridIterator.long2int(it.next())); -// assertEquals(i, j); -// i += 2; -// } -// -// shardBlockGridSize = new int[]{5, 4, 3, 13}; -// index = new ShardIndex( -// shardBlockGridSize, -// IndexLocation.END, new RawBlockCodecInfo()); -// -// it = new GridIterator(shardBlockGridSize); -// i = 0; -// while (it.hasNext()) { -// int j = index.getOffsetIndex(GridIterator.long2int(it.next())); -// assertEquals(i, j); -// i += 2; -// } - - } - - @Test - @Ignore - public void writeReadTest() throws IOException { - - // TODO - -// final N5KeyValueWriter writer = (N5KeyValueWriter)tempN5Factory.createTempN5Writer(); -// final KeyValueAccess kva = writer.getKeyValueAccess(); -// -// final int[] shardBlockGridSize = new int[]{6, 5}; -// final IndexLocation indexLocation = IndexLocation.END; -// final IndexCodecAdapter indexCodecAdapter = new IndexCodecAdapter( -// new RawBlockCodecInfo(), -// new Crc32cChecksumCodec() -// ); -// -// final ShardIndex index = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecAdapter); -// index.set(0, 6, new int[]{0, 0}); -// index.set(19, 32, new int[]{1, 0}); -// index.set(93, 111, new int[]{3, 0}); -// index.set(143, 1, new int[]{1, 2}); -// -// final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "indexTest").toString(); -// try ( -// final LockedChannel channel = kva.lockForWriting(path); -// final OutputStream out = channel.newOutputStream() -// ) { -// -// ShardIndex.write(out, index); -// } -// -// final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecAdapter); -// try ( -// final LockedChannel channel = kva.lockForReading(path); -// final InputStream in = channel.newInputStream() -// ) { -// ShardIndex.read(in, indexRead); -// } -// assertEquals(index, indexRead); - } -} From 93a04c7eb3cfad6bec1642d06c20a182d4263465 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Mon, 10 Nov 2025 20:52:55 +0100 Subject: [PATCH 7/7] Consistent argument naming --- .../saalfeldlab/n5/shard/DatasetAccess.java | 14 +++++++------- .../saalfeldlab/n5/shard/DefaultDatasetAccess.java | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index 3b1cedefe..b9c6cb29b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -23,19 +23,19 @@ */ public interface DatasetAccess { - DataBlock readBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; + DataBlock readBlock(PositionValueAccess pva, long[] gridPosition) throws N5IOException; - void writeBlock(PositionValueAccess kva, DataBlock dataBlock) throws N5IOException; + void writeBlock(PositionValueAccess pva, DataBlock dataBlock) throws N5IOException; - boolean deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; + boolean deleteBlock(PositionValueAccess pva, long[] gridPosition) throws N5IOException; - List> readBlocks(PositionValueAccess kva, List positions) throws N5IOException; + List> readBlocks(PositionValueAccess pva, List positions) throws N5IOException; - void writeBlocks(PositionValueAccess kva, List> blocks) throws N5IOException; + void writeBlocks(PositionValueAccess pva, List> blocks) throws N5IOException; // TODO: -// boolean deleteBlocks(PositionValueAccess kva, List positions) throws N5IOException; +// boolean deleteBlocks(PositionValueAccess pva, List positions) throws N5IOException; /** @@ -91,7 +91,6 @@ void writeRegion( ExecutorService exec ) throws N5Exception, InterruptedException, ExecutionException; - NestedGrid getGrid(); /** @@ -104,6 +103,7 @@ void writeRegion( * @param outerLevel of the outerLevel shard position to group by. must be in range {@code [1, NestedGrid.numLevels() - 1]} * @return map of outerLevel shard positions to inner level block positions */ + // TODO: move to Nesting static Collection> groupInnerPositions(final NestedGrid grid, final List innerPositions, final int outerLevel) { if (outerLevel < 1 || outerLevel >= grid.numLevels()) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 2a975d694..f68362a68 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -35,9 +35,9 @@ public NestedGrid getGrid() { } @Override - public DataBlock readBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { + public DataBlock readBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { final NestedPosition position = grid.nestedPosition(gridPosition); - return readBlockRecursive(kva.get(position.key()), position, grid.numLevels() - 1); + return readBlockRecursive(pva.get(position.key()), position, grid.numLevels() - 1); } private DataBlock readBlockRecursive( @@ -347,23 +347,23 @@ private ReadData writeRegionRecursive( } @Override - public boolean deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { + public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { final NestedPosition position = grid.nestedPosition(gridPosition); final long[] key = position.key(); if (grid.numLevels() == 1) { // for non-sharded dataset, don't bother getting the value, just remove the key. try { - return kva.remove(key); + return pva.remove(key); } catch (final Exception e) { throw new N5Exception("The shard at " + Arrays.toString(key) + " could not be deleted.", e); } } else { - final ReadData existingData = kva.get(key); + final ReadData existingData = pva.get(key); final ReadData modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); if (existingData != null && modifiedData == null) { - return kva.remove(key); + return pva.remove(key); } else if (modifiedData != existingData) { - kva.put(key, modifiedData); + pva.put(key, modifiedData); return true; } else { return false;