diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index d0a9c0e66..960a5970b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -35,6 +35,8 @@ import java.util.Map; import com.google.gson.JsonSyntaxException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; @@ -210,6 +212,41 @@ default boolean removeAttributes(final String pathName, final List attri return removed; } + @Override + default void writeRegion( + final String datasetPath, + final DatasetAttributes datasetAttributes, + final long[] min, + final long[] size, + final DataBlockSupplier dataBlocks, + final boolean writeFully) throws N5Exception { + try { + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), datasetAttributes); + datasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, datasetAttributes.getDimensions(), writeFully); + } catch (final UncheckedIOException e) { + throw new N5IOException( + "Failed to write blocks into dataset " + datasetPath, e); + } + } + + @Override + default void writeRegion( + final String datasetPath, + final DatasetAttributes datasetAttributes, + final long[] min, + final long[] size, + final DataBlockSupplier dataBlocks, + final boolean writeFully, + final ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException { + try { + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), datasetAttributes); + datasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, datasetAttributes.getDimensions(), writeFully, exec); + } catch (final UncheckedIOException e) { + throw new N5IOException( + "Failed to write blocks into dataset " + datasetPath, e); + } + } + @Override default void writeBlocks( final String datasetPath, @@ -239,7 +276,6 @@ default void writeBlock( "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, e); } - } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 6c3112441..3f15be8f9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -28,9 +28,6 @@ */ package org.janelia.saalfeldlab.n5; -import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; @@ -39,6 +36,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; /** * A simple structured container API for hierarchies of chunked @@ -265,6 +264,56 @@ default void writeBlocks( writeBlock(datasetPath, datasetAttributes, block); } + @FunctionalInterface + interface DataBlockSupplier { + + /** + * + * @param gridPos + * @param existingDataBlock + * existing data to be merged into the new data block (maybe {@code null}) + * + * @return data block at the given gridPos + */ + DataBlock get(long[] gridPos, final DataBlock existingDataBlock); + } + + /** + * @param datasetPath the dataset path + * @param datasetAttributes the dataset attributes + * @param min min pixel coordinate of region to write + * @param size size in pixels of region to write + * @param dataBlocks is asked to create blocks within the given region + * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * @throws N5Exception the exception + */ + void writeRegion( + String datasetPath, + DatasetAttributes datasetAttributes, + long[] min, + long[] size, + DataBlockSupplier dataBlocks, + boolean writeFully) throws N5Exception; + + /** + * @param datasetPath the dataset path + * @param datasetAttributes the dataset attributes + * @param min min pixel coordinate of region to write + * @param size size in pixels of region to write + * @param dataBlocks is asked to create blocks within the given region + * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * @param exec used to parallelize over blocks and shards + * @throws N5Exception the exception + */ + void writeRegion( + String datasetPath, + DatasetAttributes datasetAttributes, + long[] min, + long[] size, + DataBlockSupplier dataBlocks, + boolean writeFully, + ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException; + /** * Deletes the block at {@code gridPosition}. * @@ -278,6 +327,24 @@ boolean deleteBlock( final String datasetPath, final long... gridPosition) throws N5Exception; + /** + * Deletes the blocks at the given {@code gridPositions}. + * + * @param datasetPath dataset path + * @param gridPositions a list of grid positions + * @return {@code true} if any of the specified blocks existed and was deleted + * @throws N5Exception if any of the block exists but could not be deleted + */ + default boolean deleteBlocks( + final String datasetPath, + final List gridPositions) throws N5Exception { + boolean deleted = false; + for (long[] pos : gridPositions) { + deleted |= deleteBlock(datasetPath, pos); + } + return deleted; + } + /** * Save a {@link Serializable} as an N5 {@link DataBlock} at a given offset. * The diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index a7a2f6d99..b9c6cb29b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -1,15 +1,18 @@ package org.janelia.saalfeldlab.n5.shard; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; -import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.TreeMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.N5Writer.DataBlockSupplier; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; /** * Wrap an instantiated DataBlock/shard codec hierarchy to implement (single and @@ -20,15 +23,73 @@ */ public interface DatasetAccess { - DataBlock readBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; + DataBlock readBlock(PositionValueAccess pva, long[] gridPosition) throws N5IOException; + + void writeBlock(PositionValueAccess pva, DataBlock dataBlock) throws N5IOException; + + boolean deleteBlock(PositionValueAccess pva, long[] gridPosition) throws N5IOException; - void writeBlock(PositionValueAccess kva, DataBlock dataBlock) throws N5IOException; - boolean deleteBlock(PositionValueAccess kva, long[] gridPosition) throws N5IOException; + List> readBlocks(PositionValueAccess pva, List positions) throws N5IOException; - List> readBlocks(PositionValueAccess kva, List positions); + void writeBlocks(PositionValueAccess pva, List> blocks) throws N5IOException; - void writeBlocks(PositionValueAccess kva, List> blocks); +// TODO: +// boolean deleteBlocks(PositionValueAccess pva, List positions) throws N5IOException; + + + /** + * + * @param pva + * @param min + * min pixel coordinate of region to write + * @param size + * size in pixels of region to write + * @param blocks + * is asked to create blocks within the given region + * @param datasetDimensions + * @param writeFully + * if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * + * @throws N5IOException + */ + void writeRegion( + PositionValueAccess pva, + long[] min, + long[] size, + DataBlockSupplier blocks, + long[] datasetDimensions, + boolean writeFully + ) throws N5IOException; + + /** + * + * @param pva + * @param min + * min pixel coordinate of region to write + * @param size + * size in pixels of region to write + * @param blocks + * is asked to create blocks within the given region. must be thread-safe. + * @param datasetDimensions + * @param writeFully + * if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * @param exec + * used to parallelize over blocks and shards + * + * @throws N5Exception + * @throws InterruptedException + * @throws ExecutionException + */ + void writeRegion( + PositionValueAccess pva, + long[] min, + long[] size, + DataBlockSupplier blocks, + long[] datasetDimensions, + boolean writeFully, + ExecutorService exec + ) throws N5Exception, InterruptedException, ExecutionException; NestedGrid getGrid(); @@ -42,6 +103,7 @@ public interface DatasetAccess { * @param outerLevel of the outerLevel shard position to group by. must be in range {@code [1, NestedGrid.numLevels() - 1]} * @return map of outerLevel shard positions to inner level block positions */ + // TODO: move to Nesting static Collection> groupInnerPositions(final NestedGrid grid, final List innerPositions, final int outerLevel) { if (outerLevel < 1 || outerLevel >= grid.numLevels()) @@ -56,7 +118,7 @@ static Collection> groupInnerPositions(final if (nestedPosition.level() == outerLevel) outerNestedPosition = nestedPosition; else - outerNestedPosition = new NestedPosition(grid, nestedPosition.absolute(0), outerLevel); + outerNestedPosition = grid.nestedPosition(nestedPosition.absolute(outerLevel), outerLevel); final List blocks = blocksPerShard.computeIfAbsent(outerNestedPosition, it -> new ArrayList<>()); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 43029582d..f68362a68 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -5,17 +5,20 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; +import org.janelia.saalfeldlab.n5.N5Writer.DataBlockSupplier; import org.janelia.saalfeldlab.n5.codec.BlockCodec; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; import org.janelia.saalfeldlab.n5.shard.Nesting.NestedPosition; -import static org.janelia.saalfeldlab.n5.shard.DatasetAccess.*; +import static org.janelia.saalfeldlab.n5.shard.DatasetAccess.groupInnerPositions; public class DefaultDatasetAccess implements DatasetAccess { @@ -32,9 +35,9 @@ public NestedGrid getGrid() { } @Override - public DataBlock readBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { - final NestedPosition position = new NestedPosition(grid, gridPosition); - return readBlockRecursive(kva.get(position.key()), position, grid.numLevels() - 1); + public DataBlock readBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { + final NestedPosition position = grid.nestedPosition(gridPosition); + return readBlockRecursive(pva.get(position.key()), position, grid.numLevels() - 1); } private DataBlock readBlockRecursive( @@ -56,13 +59,13 @@ private DataBlock readBlockRecursive( } @Override - public List> readBlocks(PositionValueAccess pva, List positions) { + public List> readBlocks(PositionValueAccess pva, List positions) throws N5IOException { if (grid.numLevels() == 1) { return positions.stream().map(it -> readBlock(pva, it)).collect(Collectors.toList()); } - final List blockPositions = positions.stream().map(it -> new NestedPosition(grid, it)).collect(Collectors.toList()); + final List blockPositions = positions.stream().map(grid::nestedPosition).collect(Collectors.toList()); final int outermostLevel = grid.numLevels() - 1; final Collection> blocksPerOutermostShard = groupInnerPositions(grid, blockPositions, outermostLevel); @@ -141,7 +144,7 @@ private List> readShardRecursive( @Override public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException { - final NestedPosition position = new NestedPosition(grid, dataBlock.getGridPosition()); + final NestedPosition position = grid.nestedPosition(dataBlock.getGridPosition()); final long[] key = position.key(); final ReadData existingData = getExistingReadData(pva, key); @@ -175,7 +178,8 @@ private ReadData writeBlockRecursive( } } - @Override public void writeBlocks(PositionValueAccess pva, List> dataBlocks) { + @Override + public void writeBlocks(final PositionValueAccess pva, final List> dataBlocks) throws N5IOException { if (grid.numLevels() == 1) { dataBlocks.forEach(it -> writeBlock(pva, it)); @@ -252,24 +256,114 @@ private ReadData writeShardRecursive( return codec.encode(new RawShardDataBlock(shardPosition, shard)); } + public void writeRegion( + final PositionValueAccess pva, + final long[] min, + final long[] size, + final DataBlockSupplier blocks, + final long[] datasetDimensions, + final boolean writeFully + ) throws N5IOException { + + final Region region = new Region(min, size, grid, datasetDimensions); + + for (long[] key : Region.gridPositions(region.minPos().key(), region.maxPos().key())) { + final NestedPosition pos = grid.nestedPosition(key, grid.numLevels() - 1); + final boolean nestedWriteFully = writeFully || region.fullyContains(pos); + final ReadData existingData = nestedWriteFully ? null : getExistingReadData(pva, key); + final ReadData modifiedData = writeRegionRecursive(existingData, region, blocks, pos); + pva.put(key, modifiedData); + } + } + + @Override + public void writeRegion( + final PositionValueAccess pva, + final long[] min, + final long[] size, + final DataBlockSupplier blocks, + final long[] datasetDimensions, + final boolean writeFully, + final ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException { + + final Region region = new Region(min, size, grid, datasetDimensions); + + for (long[] key : Region.gridPositions(region.minPos().key(), region.maxPos().key())) { + exec.submit(() -> { + final NestedPosition pos = grid.nestedPosition(key, grid.numLevels() - 1); + final boolean nestedWriteFully = writeFully || region.fullyContains(pos); + final ReadData existingData = nestedWriteFully ? null : getExistingReadData(pva, key); + final ReadData modifiedData = writeRegionRecursive(existingData, region, blocks, pos); + pva.put(key, modifiedData); + }); + } + } + + private ReadData writeRegionRecursive( + final ReadData existingReadData, // may be null + final Region region, + final DataBlockSupplier blocks, + final NestedPosition position + ) { + final boolean writeFully = existingReadData == null; + final int level = position.level(); + String indent = ""; + for (int i = 0; i < 2-level; ++i) + indent += " "; + System.out.println(indent + "position = " + position + (writeFully ? " (writeFully)" : "")); + if ( level == 0 ) { + + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[0]; + final long[] gridPosition = position.absolute(0); + + // If the DataBlock is not fully contained in the region, we will + // get existingReadData != null. In that case, we decode the + // existing DataBlock and pass it to the BlockSupplier for modification. + final DataBlock existingDataBlock = (existingReadData == null) + ? null + : codec.decode(existingReadData, gridPosition); + final DataBlock dataBlock = blocks.get(gridPosition, existingDataBlock); + + return codec.encode(dataBlock); + } else { + + @SuppressWarnings("unchecked") + final BlockCodec codec = (BlockCodec) codecs[level]; + final long[] gridPos = position.absolute(level); + final RawShard shard = existingReadData == null ? + new RawShard(grid.relativeBlockSize(level)) : + codec.decode(existingReadData, gridPos).getData(); + for (NestedPosition pos : region.containedNestedPositions(position)) { + final boolean nestedWriteFully = writeFully || region.fullyContains(pos); + final long[] elementPos = pos.relative(); + final ReadData existingElementData = nestedWriteFully ? null : shard.getElementData(elementPos); + final ReadData modifiedElementData = writeRegionRecursive(existingElementData, region, blocks, pos); + shard.setElementData(modifiedElementData, elementPos); + } + + return codec.encode(new RawShardDataBlock(gridPos, shard)); + } + } + @Override - public boolean deleteBlock(final PositionValueAccess kva, final long[] gridPosition) throws N5IOException { - final NestedPosition position = new NestedPosition(grid, gridPosition); + public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { + final NestedPosition position = grid.nestedPosition(gridPosition); final long[] key = position.key(); if (grid.numLevels() == 1) { // for non-sharded dataset, don't bother getting the value, just remove the key. try { - return kva.remove(key); + return pva.remove(key); } catch (final Exception e) { throw new N5Exception("The shard at " + Arrays.toString(key) + " could not be deleted.", e); } } else { - final ReadData existingData = kva.get(key); + final ReadData existingData = pva.get(key); final ReadData modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); if (existingData != null && modifiedData == null) { - return kva.remove(key); + return pva.remove(key); } else if (modifiedData != existingData) { - kva.put(key, modifiedData); + pva.put(key, modifiedData); return true; } else { return false; @@ -317,7 +411,7 @@ private ReadData deleteBlockRecursive( private static ReadData getExistingReadData(final PositionValueAccess pva, final long[] key) { // need to read the shard anyway, and currently (Sept 24 2025) - // have no way to tell if they key exist from what is in this method except to attempt + // have no way to tell if the key exists from what is in this method except to attempt // to materialize and catch the N5NoSuchKeyException try { ReadData existingData = pva.get(key); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java index 892900ae4..146f650bd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java @@ -1,62 +1,22 @@ package org.janelia.saalfeldlab.n5.shard; -import java.util.ArrayList; - -// TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -// -// [ ] NestedGrid -// [ ] validation in constructor -// [ ] test for that validation -// [ ] javadoc -// -// [ ] NestedPosition interface -// [+] LazyNestedPosition class -// [+] fields: NestedGrid, long[] position, int level -// [+] construct with source level 0 -// [+] minimal abs/rel access methods -// [+] toString() -// [-] extract NestedPosition interface -// ==> postpone until necessary -// [ ] equals / hashcode -// [ ] should we have prefix()? suffix()? head()? tail()? -// [+] Implement Comparable so that we can sort and aggregate for N5Reader.readBlocks(...). -// For nested = {X,Y,Z} compare by Z, then Y, then X. -// For X = {x,y,z} compare by z, then y, then x. (flattening order) -// -// TODO ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - import java.util.Arrays; -import java.util.List; - -import org.janelia.saalfeldlab.n5.util.GridIterator; public class Nesting { - public static void main(String[] args) { - final int[][] blockSizes = {{1,1,1}, {3,2,2}, {6,4,4}, {24,24,24}}; - final NestedGrid grid = new NestedGrid(blockSizes); - final NestedPosition pos = new NestedPosition(grid, new long[] {38, 7, 129}); - System.out.println("pos = " + pos); - System.out.println("key = " + Arrays.toString(pos.key())); - } - public static class NestedPosition implements Comparable { private final NestedGrid grid; private final long[] position; private final int level; - public NestedPosition(final NestedGrid grid, final long[] position, final int level) { + protected NestedPosition(final NestedGrid grid, final long[] position, final int level) { this.grid = grid; this.position = position; this.level = level; } - public NestedPosition(final NestedGrid grid, final long[] position) { - this(grid, position, 0); - } - /** * Get the nesting level of this position. *

@@ -84,7 +44,18 @@ public int numDimensions() { * @return relative grid position */ public long[] relative(final int level) { - return grid.relativePosition(position, level); + return grid.relativePosition(position, this.level, level); + } + + /** + * Get the relative grid position at this positions {@link #level()}, + * that is, relative offset within containing the {@code (level+1)} + * element. + * + * @return relative grid position + */ + public long[] relative() { + return relative(level()); } /** @@ -96,13 +67,17 @@ public long[] relative(final int level) { * @return absolute grid position */ public long[] absolute(final int level) { - return grid.absolutePosition(position, level); + return grid.absolutePosition(position, this.level, level); } public long[] key() { return relative(grid.numLevels() - 1); } + public long[] pixelPosition() { + return grid.pixelPosition(position, level); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -117,7 +92,8 @@ public String toString() { return sb.toString(); } - @Override public int compareTo(NestedPosition o) { + @Override + public int compareTo(NestedPosition o) { final int dimensionInequality = Integer.compare(numDimensions(), o.numDimensions()); if (dimensionInequality != 0) @@ -127,13 +103,10 @@ public String toString() { if (levelInequality != 0) return levelInequality; - final long[] otherAbsPos = o.absolute(level); - final long[] absPos = absolute(level); - - for (int i = absPos.length - 1; i >= 0; --i) { - final long diff = absPos[i] - otherAbsPos[i]; + for (int i = position.length - 1; i >= 0; --i) { + final long diff = position[i] - o.position[i]; if (diff != 0) - return (int)diff; + return (int) diff; } return 0; @@ -144,20 +117,81 @@ public String toString() { } /** - * A nested grid of blocks used to coordinate the relationships of shards and the blocks / chunks they contain. + * A nested grid of blocks used to coordinate the relationships of shards + * and the blocks / chunks they contain. + *

+ * The nesting depth ({@link #numLevels()}) of the {@code NestedGrid} is 1 + * for non-sharded datasets, 2 for simple sharded datasets (where shards + * contain datablocks), and ≥3 for nested sharded datasets. + *

+ * Positions with {@code level=0} refer to the DataBlock grid, positions + * with {@code level=1} refer to first-level Shard grid, and so on. */ public static class NestedGrid { + /** + * Create a {@code NestedPosition} at the specified nesting {@code + * level} grid {@code position}. + *

+ * Note that {@code position} is in units of grid elements at {@code + * level}. Positions with {@code level=0} refer to the DataBlock grid, + * positions with {@code level=1} refer to first-level Shard grid, and + * so on. + *

+ * The returned {@code NestedPosition} will have + * {@link NestedPosition#level() level()==level}. + * + * @param position + * position at {@code level} + * @param level + * nesting level of {@code position} + * + * @return a NestedPosition representation of the specified grid position and nesting level + */ + public NestedPosition nestedPosition(final long[] position, final int level) { + return new NestedPosition(this, position, level); + } + + /** + * Create a {@code NestedPosition} at the specified block grid {@code + * position} (that is, at nesting level 0). + *

+ * Note that {@code position} is in units of DataBlocks. + *

+ * The returned {@code NestedPosition} will have + * {@link NestedPosition#level() level()==0}. + * + * @param position + * position at level 0 (block grid) + * + * @return a NestedPosition representation of the specified block grid position + */ + public NestedPosition nestedPosition(final long[] position) { + return nestedPosition(position, 0); + } + + + + + + private final int numLevels; private final int numDimensions; - // relativeToBase[i][d] is block size at level i relative to level 0 + /** + * relativeToBase[i][d] is block size (in dimension d) at level i relative to level 0 + */ private final int[][] relativeToBase; - // relativeToAdjacent[i][d] is block size at level i relative to level i-1 + /** + * relativeToAdjacent[i][d] is block size (in dimension d) at level i relative to level i-1 + */ private final int[][] relativeToAdjacent; + /** + * blockSizes[l][d] is the block size in pixels at level l in dimension d + */ private final int[][] blockSizes; /** @@ -227,39 +261,91 @@ public int numDimensions() { return numDimensions; } - public int[] getBlockSize(int level) { + /** + * Get the block size in pixels at the given {@code level}. + */ + public int[] getBlockSize(final int level) { return blockSizes[level]; } - public void absolutePosition( + /** + * Computes the pixel position for the given {@code sourcePos} grid + * position at {@code sourceLevel}. + * + * @param sourcePos + * a grid position at {@code sourceLevel} + * @param sourceLevel + * nesting level of {@code sourcePos} + * @param targetPos + * the pixel position will be stored here + */ + public void pixelPosition( final long[] sourcePos, final int sourceLevel, - final long[] targetPos, - final int targetLevel) { - final int[] sk = relativeToBase[sourceLevel]; - final int[] si = relativeToBase[targetLevel]; + final long[] targetPos) { + final int[] s = blockSizes[sourceLevel]; for (int d = 0; d < numDimensions; ++d) { - targetPos[d] = sourcePos[d] * sk[d] / si[d]; + targetPos[d] = sourcePos[d] * s[d]; } } - public void relativePosition( + /** + * Get the pixel position for the given {@code sourcePos} grid position + * at {@code sourceLevel}. + * + * @param sourcePos + * a grid position at {@code sourceLevel} + * @param sourceLevel + * nesting level of {@code sourcePos} + * + * @return the pixel position + */ + public long[] pixelPosition( + final long[] sourcePos, + final int sourceLevel) { + final long[] targetPos = new long[numDimensions]; + pixelPosition(sourcePos, sourceLevel, targetPos); + return targetPos; + } + + /** + * Computes the absolute {@code targetPos} grid position at {@code + * targetLevel} for the given {@code sourcePos} grid position at {@code + * sourceLevel}. + *

+ * For example, this can be used to compute the coordinates on the shard + * grid ({@code targetLevel==1}) of the shard containing a given + * datablock ({@code sourcePos} at {@code sourceLevel==0}). + * + * @param sourcePos + * a grid position at {@code sourceLevel} + * @param sourceLevel + * nesting level of {@code sourcePos} + * @param targetPos + * the grid position at {@code targetLevel} will be stored here + * @param targetLevel + * nesting level of {@code targetPos} + */ + public void absolutePosition( final long[] sourcePos, final int sourceLevel, final long[] targetPos, final int targetLevel) { - absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel); - if (targetLevel < numLevels - 1) { - final int[] rj = relativeToAdjacent[targetLevel + 1]; - for (int d = 0; d < numDimensions; ++d) { - targetPos[d] %= rj[d]; - } + final int[] sk = relativeToBase[sourceLevel]; + final int[] si = relativeToBase[targetLevel]; + for (int d = 0; d < numDimensions; ++d) { + targetPos[d] = sourcePos[d] * sk[d] / si[d]; } } /** - * The absolute position of * source position for the given source level at the target level. - * + * Get the absolute grid position at {@code targetLevel} for the given + * {@code sourcePos} grid position at {@code sourceLevel}. + *

+ * For example, this can be used to compute the coordinates on the shard + * grid ({@code targetLevel==1}) of the shard containing a given + * datablock ({@code sourcePos} at {@code sourceLevel==0}). + * * @param sourcePos the source position j * @param sourceLevel the source level * @param targetLevel the target level @@ -269,15 +355,22 @@ public long[] absolutePosition( final long[] sourcePos, final int sourceLevel, final int targetLevel) { + if (sourceLevel == targetLevel) { + return sourcePos; + } final long[] targetPos = new long[numDimensions]; absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel); return targetPos; } /** - * The absolute position of the level 0 source position at - * the target level. - * + * Get the absolute grid position at {@code targetLevel} for the given + * {@code sourcePos} block grid position (level 0). + *

+ * For example, this can be used to compute the coordinates on the shard + * grid ({@code targetLevel==1}) of the shard containing a given + * datablock ({@code sourcePos}. + * * @param sourcePos the source position j * @param targetLevel the target level * @return absolute position at the target level @@ -288,6 +381,42 @@ public long[] absolutePosition( return absolutePosition(sourcePos, 0, targetLevel); } + /** + * Computes the {@code targetPos} grid position at {@code targetLevel} + * for the given {@code sourcePos} grid position at {@code sourceLevel}, + * relative to the containing element at {@code targetLevel+1}. + * (The containing element is a shard for {@code targetLevel+1 < + * numLevels} or the dataset for {@code targetLevel+1 == numLevels}.) + *

+ * For example, this can be used to compute the grid coordinates {@code + * targetLevel==0} of a given datablock ({@code sourcePos} at {@code + * sourceLevel==0}) withing a shard (containing element at level {@code + * targetLevel+1==1}). + *

+ * + * @param sourcePos + * a grid position at {@code sourceLevel} + * @param sourceLevel + * nesting level of {@code sourcePos} + * @param targetPos + * the grid position at {@code targetLevel} will be stored here + * @param targetLevel + * nesting level of {@code targetPos} + */ + public void relativePosition( + final long[] sourcePos, + final int sourceLevel, + final long[] targetPos, + final int targetLevel) { + absolutePosition(sourcePos, sourceLevel, targetPos, targetLevel); + if (targetLevel < numLevels - 1) { + final int[] rj = relativeToAdjacent[targetLevel + 1]; + for (int d = 0; d < numDimensions; ++d) { + targetPos[d] %= rj[d]; + } + } + } + public long[] relativePosition( final long[] sourcePos, final int sourceLevel, @@ -303,45 +432,15 @@ public long[] relativePosition( return relativePosition(sourcePos, 0, targetLevel); } - public int[] relativeBlockSize(final int level) { - return relativeToAdjacent[level]; - } - - public int[] absoluteBlockSize(final int level) { - return relativeToBase[level]; - } - /** - * Given a block position at a particular level, returns a list of - * positions of all sub-blocks at a particular subLevel. + * Get size of a block at the given {@code level} relative to {@code + * level-1} (that is, in units of {@code level-1} blocks). *

- * Can be used to get a list of chunk positions for a shard with a - * particular position. - * - * @param position - * a position - * @param level - * the nesting level of the given position - * @param subLevel - * the nesting sub-level of positions to return - * @return the sub-block positions + * For example {@code relativeBlockSize(1)} returns the number of + * datablocks in a (non-nested) shard. */ - public List positionInSubGrid(long[] position, int level, int subLevel) { - - final long[] subPosition = new long[numDimensions()]; - absolutePosition(position, level, subPosition, subLevel); - - final int[] numElementsInSubGrid = absoluteBlockSize(numLevels() - 1); - final GridIterator git = new GridIterator(GridIterator.int2long(numElementsInSubGrid), subPosition); - - // TODO return NestedPositions instead? - final ArrayList positions = new ArrayList<>(); - while (git.hasNext()) - positions.add(git.next().clone()); - - return positions; + public int[] relativeBlockSize(final int level) { + return relativeToAdjacent[level]; } - } - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java new file mode 100644 index 000000000..cadb58347 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java @@ -0,0 +1,174 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Bounds, in pixel coordinates, of a region in a dataset. + *

+ * Provides methods to find which blocks and shards are contained in the + * region, iterate sub-NestedPositions, etc. + */ +class Region { + + /** + * The dimensions of the full dataset. + * This is used to decide whether DataBlocks are on the border (and therefore possibly truncated). + *

+ * TODO: + * Currently, the constructor expands this to {@code min + size} (if that is larger than the provided datasetDimensions). + * Do we want this, or should we rather throw an {@code }IllegalArgumentException} in that case? + */ + private final long[] datasetDimensions; + + /** + * The nested grid of the dataset + */ + private final Nesting.NestedGrid grid; + + /** + * min pixel position in the region + */ + private final long[] min; + + /** + * size of the region in pixels + */ + private final long[] size; + + /** + * {@code NestedPosition} of the block containing the min pixel position. + */ + private final Nesting.NestedPosition minPos; + + /** + * {@code NestedPosition} of the block containing the max pixel position. + */ + private final Nesting.NestedPosition maxPos; + + Region(final long[] min, final long[] size, final Nesting.NestedGrid grid, final long[] datasetDimensions) { + this.min = min; + this.size = size; + this.grid = grid; + + final int n = min.length; + this.datasetDimensions = new long[n]; + Arrays.setAll(this.datasetDimensions, d -> Math.max(min[d] + size[d], datasetDimensions[d])); + + final int[] blockSize = grid.getBlockSize(0); + + final long[] minBlock = new long[n]; + Arrays.setAll(minBlock, d -> min[d] / blockSize[d]); + minPos = grid.nestedPosition(minBlock); + + final long[] maxBlock = new long[n]; + Arrays.setAll(maxBlock, d -> (min[d] + size[d] - 1) / blockSize[d]); + maxPos = grid.nestedPosition(maxBlock); + } + + /** + * Get the {@code NestedPosition} of the minimum DataBlock touched by the region. + */ + Nesting.NestedPosition minPos() { + return minPos; + } + + /** + * Get the {@code NestedPosition} of the maximum DataBlock touched by the region. + */ + Nesting.NestedPosition maxPos() { + return maxPos; + } + + /** + * Check whether the Shard or DataBlock corresponding to the given + * position is fully contained inside the region. + * (The {@link Nesting.NestedPosition#level() level} of {@code position} is used + * to determine whether it refers to a DataBlock or a (potentially + * nested) Shard. + * + * @param position + * the NestedPosition to check + * + * @return true, if the given position is fully contained in this region + */ + boolean fullyContains(final Nesting.NestedPosition position) { + + final long[] pmin = position.pixelPosition(); + for (int d = 0; d < pmin.length; d++) { + if (pmin[d] < min[d]) { + return false; + } + } + + final long[] pmax = maxPixelPos(position); + for (int d = 0; d < pmax.length; d++) { + final long m = Math.min(pmax[d], datasetDimensions[d] - 1); + if (m > min[d] + size[d] - 1) { + return false; + } + } + + return true; + } + + // TODO: Revise. Inline? Should this be method of NestedPosition? + private long[] maxPixelPos(final Nesting.NestedPosition position) { + final long[] pos = position.pixelPosition(); + final int[] elementSize = grid.getBlockSize(position.level()); + Arrays.setAll(pos, d -> pos[d] + elementSize[d] - 1); + return pos; + } + + /** + * Returns {@code NestedPosition}s of all nested elements in the given + * {@code position} that are contained in this {@code Region}. + *

+ * The returned {@code NestedPosition}s will all have level {@code + * position.level()-1}. + */ + // TODO: Revise to accept Consumer for handling each position + List containedNestedPositions(final Nesting.NestedPosition position) { + final int level = position.level() - 1; + final long[] gridMinOfRegion = minPos().absolute(level); + final long[] gridMaxOfRegion = maxPos().absolute(level); + + final long[] gridMinOfPosition = position.absolute(level); + final int[] gridSizeOfPosition = grid.relativeBlockSize(level + 1); + + final int n = grid.numDimensions(); + final long[] gridMin = new long[n]; + Arrays.setAll(gridMin, d -> Math.max(gridMinOfRegion[d], gridMinOfPosition[d])); + final long[] gridMax = new long[n]; + Arrays.setAll(gridMax, d -> Math.min(gridMaxOfRegion[d], gridMinOfPosition[d] + gridSizeOfPosition[d] - 1)); + + final List gridPositions = gridPositions(gridMin, gridMax); + final List nestedPositions = new ArrayList<>(); + gridPositions.forEach(p -> nestedPositions.add(grid.nestedPosition(p, level))); + return nestedPositions; + } + + // TODO: Can this fully replace GridIterator? + // TODO: Revise to accept Consumer for handling each position + static List gridPositions(final long[] min, final long[] max) { + final int n = min.length; + final long[] pos = min.clone(); + int numElements = 1; + for (int d = 0; d < n; ++d) { + numElements *= (int) (max[d] - min[d] + 1); + } + long[][] positions = new long[numElements][n]; + Arrays.setAll(positions[0], j -> pos[j]); + for (int i = 1; i < numElements; i++) { + for (int d = 0; d < n; ++d) { + if (++pos[d] <= max[d]) { + Arrays.setAll(positions[i], j -> pos[j]); + break; + } + pos[d] = min[d]; + } + } + return Arrays.asList(positions); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java b/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java deleted file mode 100644 index 1b7076d54..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/FinalPosition.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.janelia.saalfeldlab.n5.util; - -/* - * An immutable {@Position}. - */ -public class FinalPosition implements Position { - - public final long[] position; - - public FinalPosition(long[] position) { - this.position = position; - } - - public FinalPosition(Position p) { - this.position = p.get().clone(); - } - - @Override - public long[] get() { - return position; - } - - @Override - public long get(int i) { - return position[i]; - } - - @Override - public String toString() { - return Position.toString(this); - } - - @Override - public boolean equals(Object obj) { - return Position.equals(this, obj); - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java b/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java deleted file mode 100644 index 43258efaf..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/GridIterator.java +++ /dev/null @@ -1,178 +0,0 @@ -package org.janelia.saalfeldlab.n5.util; - -import java.util.Iterator; - -/** - * Essentially imglib2's IntervalIterator, but N5 does not depend on imglib2. - */ -public class GridIterator implements Iterator { - - final protected long[] dimensions; - - final protected long[] steps; - - final protected long[] position; - - final protected int[] intPosition; - - final protected long[] min; - - final protected int lastIndex; - - protected int index = -1; - - public GridIterator(final long[] dimensions, final long[] min) { - - final int n = dimensions.length; - this.dimensions = new long[n]; - this.position = new long[n]; - this.intPosition = new int[n]; - this.min = min; - steps = new long[n]; - - final int m = n - 1; - long k = steps[0] = 1; - for (int d = 0; d < m; ) { - final long dimd = dimensions[d]; - this.dimensions[d] = dimd; - k *= dimd; - steps[++d] = k; - } - final long dimm = dimensions[m]; - this.dimensions[m] = dimm; - lastIndex = (int)(k * dimm - 1); - } - - public GridIterator(final long[] dimensions) { - - this(dimensions, new long[dimensions.length]); - } - - public GridIterator(final int[] dimensions) { - - this(int2long(dimensions)); - } - - public void fwd() { - - ++index; - } - - public void reset() { - - index = -1; - } - - @Override - public boolean hasNext() { - - return index < lastIndex; - } - - @Override - public long[] next() { - - fwd(); - indexToPosition(index, dimensions, min, position); - return position; - } - - public int[] nextInt() { - - next(); - long2int(position, intPosition); - return intPosition; - } - - public int getIndex() { - - return index; - } - - public static void indexToPosition(long index, final long[] dimensions, final long[] offset, - final long[] position) { - - for (int dim = 0; dim < dimensions.length; dim++) { - position[dim] = (index % dimensions[dim]) + offset[dim]; - index /= dimensions[dim]; - } - } - - public static void indexToPosition(long index, final int[] dimensions, final long[] offset, - final long[] position) { - - for (int dim = 0; dim < dimensions.length; dim++) { - position[dim] = (index % dimensions[dim]) + offset[dim]; - index /= dimensions[dim]; - } - } - - final static public long positionToIndex(final long[] dimensions, final long[] position) { - long idx = 0; - int cumulativeSize = 1; - for (int i = 0; i < position.length; i++) { - idx += position[i] * cumulativeSize; - cumulativeSize *= dimensions[i]; - } - return idx; - } - - final static public long positionToIndex(final long[] dimensions, final int[] position) { - long idx = 0; - int cumulativeSize = 1; - for (int i = 0; i < position.length; i++) { - idx += position[i] * cumulativeSize; - cumulativeSize *= dimensions[i]; - } - return idx; - } - - final static public long positionToIndex(final int[] dimensions, final long[] position) { - long idx = 0; - int cumulativeSize = 1; - for (int i = 0; i < position.length; i++) { - idx += position[i] * cumulativeSize; - cumulativeSize *= dimensions[i]; - } - return idx; - } - - final static public long positionToIndex(final int[] dimensions, final int[] position) { - long idx = 0; - int cumulativeSize = 1; - for (int i = 0; i < position.length; i++) { - idx += position[i] * cumulativeSize; - cumulativeSize *= dimensions[i]; - } - return idx; - } - - public static int[] long2int(final long[] src, final int[] tgt) { - - for (int d = 0; d < tgt.length; ++d) - tgt[d] = (int)src[d]; - - return tgt; - } - - public static int[] long2int(final long[] a) { - - final int[] i = new int[a.length]; - - for (int d = 0; d < a.length; ++d) - i[d] = (int)a[d]; - - return i; - } - - public static long[] int2long(final int[] i) { - - final long[] l = new long[i.length]; - - for (int d = 0; d < l.length; ++d) - l[d] = i[d]; - - return l; - } - -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java b/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java deleted file mode 100644 index 2403835b9..000000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/util/Position.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.janelia.saalfeldlab.n5.util; - -import java.util.Arrays; - -/* - * A wrapper around a primitive long array that is lexicographically {@link Comparable} - * and for which we can test equality. - */ -public interface Position extends Comparable { - - long[] get(); - - long get(int i); - - default int numDimensions() { - return get().length; - } - - @Override - default int compareTo(Position other) { - - // use Arrays.compare when we update to Java 9+ - final int N = numDimensions() > other.numDimensions() ? numDimensions() : other.numDimensions(); - for (int i = 0; i < N; i++) { - final long diff = get(i) - other.get(i); - if (diff != 0) - return (int) diff; - } - return 0; - } - - static boolean equals(final Position a, final Object b) { - - if (a == null && b == null) - return true; - - if (b == null) - return false; - - if (!(b instanceof Position)) - return false; - - final Position other = (Position) b; - if (other.numDimensions() != a.numDimensions()) - return false; - - for (int i = 0; i < a.numDimensions(); i++) - if (other.get(i) != a.get(i)) - return false; - - return true; - } - - static String toString(Position p) { - return "Position: " + Arrays.toString(p.get()); - } - - static Position wrap(final long[] p) { - return new FinalPosition(p); - } - - static Position wrap(final int[] p) { - return new FinalPosition(GridIterator.int2long(p)); - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java deleted file mode 100644 index 20b69082a..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/demo/BlockIterators.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.janelia.saalfeldlab.n5.demo; - -import java.util.Arrays; -import java.util.Iterator; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -import org.janelia.saalfeldlab.n5.DataType; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.RawCompression; -import org.janelia.saalfeldlab.n5.codec.CodecInfo; -import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.DeterministicSizeCodecInfo; -import org.janelia.saalfeldlab.n5.util.GridIterator; - -public class BlockIterators { - - public static void main(String[] args) { - -// blockIterator(); -// shardBlockIterator(); - } - - public static void shardBlockIterator() { - -// final DatasetAttributes attrs = new DatasetAttributes( -// new long[] {12, 8}, // image size -// new int[] {6, 4}, // shard size -// new int[] {2, 2}, // block size -// DataType.UINT8, -// new ShardingCodec( -// new int[] {2, 2}, -// new CodecInfo[] { new N5BlockCodecInfo() }, -// new DeterministicSizeCodecInfo[] { new RawBlockCodecInfo() }, -// IndexLocation.END -// )); -// -// shardPositions(attrs) -// .forEach(x -> System.out.println(Arrays.toString(x))); - } - -// public static void blockIterator() { -// -// final DatasetAttributes attrs = new DatasetAttributes( -// new long[] {12, 8}, -// new int[] {2, 2}, -// DataType.UINT8, -// new RawCompression()); -// -// blockPositions(attrs).forEach(x -> System.out.println(Arrays.toString(x))); -// } -// -// public static long[] blockGridSize(final DatasetAttributes attrs ) { -// // this could be a nice method for DatasetAttributes -// -// return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getBlockSize()[i])).toArray(); -// -// } -// -// public static long[] shardGridSize(final DatasetAttributes attrs ) { -// // this could be a nice method for DatasetAttributes -// -// return IntStream.range(0, attrs.getNumDimensions()).mapToLong(i -> (long)Math.ceil((double)attrs.getDimensions()[i] / attrs.getShardSize()[i])).toArray(); -// -// } -// -// public static Stream blockPositions( DatasetAttributes attrs ) { -// return toStream(new GridIterator(blockGridSize(attrs))); -// } -// -// public static Stream shardPositions( DatasetAttributes attrs ) { -// -// final int[] blocksPerShard = attrs.getBlocksPerShard(); -// return toStream( new GridIterator(shardGridSize(attrs))) -// .flatMap( shardPosition -> { -// -// final int nd = attrs.getNumDimensions(); -// final long[] min = attrs.getBlockPositionFromShardPosition(shardPosition, new int[nd]); -// return toStream(new GridIterator(GridIterator.int2long(blocksPerShard), min)); -// }); -// } -// -// public static Stream toStream( final Iterator it ) { -// return StreamSupport.stream( Spliterators.spliteratorUnknownSize( -// it, Spliterator.ORDERED), -// false); -// } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java index eaa895420..bf0721499 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/RawShardTest.java @@ -51,8 +51,6 @@ public static void main(String[] args) { final DatasetAccess datasetAccess = attributes.datasetAccess(); - // TODO: N5Reader/Writer needs to provide a PositionValueAccess implementation on top of its KVA. - // The read/write/deleteBlock methods would getDataAccess() from the DatasetAttributes and call it with that PositionValueAccess. final PositionValueAccess store = new TestPositionValueAccess(); @@ -122,7 +120,7 @@ private static DataBlock createDataBlock(int[] size, long[] gridPosition Arrays.fill(bytes, (byte) fillValue); return new ByteArrayDataBlock(size, gridPosition, bytes); } - + public static class TestDatasetAttributes extends DatasetAttributes { public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType dataType, BlockCodecInfo blockCodecInfo, diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java deleted file mode 100644 index ea1406d83..000000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardIndexTest.java +++ /dev/null @@ -1,105 +0,0 @@ -package org.janelia.saalfeldlab.n5.shard; - -import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; -import org.janelia.saalfeldlab.n5.N5FSTest; -import org.janelia.saalfeldlab.n5.N5KeyValueWriter; -import org.janelia.saalfeldlab.n5.codec.IndexCodecAdapter; -import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; -import org.janelia.saalfeldlab.n5.codec.checksum.Crc32cChecksumCodec; -import org.janelia.saalfeldlab.n5.util.GridIterator; -import org.junit.After; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Paths; - -import static org.junit.Assert.assertEquals; - -public class ShardIndexTest { - - private static final N5FSTest tempN5Factory = new N5FSTest(); - - @After - public void removeTempWriters() { - - tempN5Factory.removeTempWriters(); - } - - @Test - @Ignore - public void testOffsetIndex() { - - // TODO -// int[] shardBlockGridSize = new int[]{5, 4, 3}; -// ShardIndex index = new ShardIndex( -// shardBlockGridSize, -// IndexLocation.END, new RawBlockCodecInfo()); -// -// GridIterator it = new GridIterator(shardBlockGridSize); -// int i = 0; -// while (it.hasNext()) { -// int j = index.getOffsetIndex(GridIterator.long2int(it.next())); -// assertEquals(i, j); -// i += 2; -// } -// -// shardBlockGridSize = new int[]{5, 4, 3, 13}; -// index = new ShardIndex( -// shardBlockGridSize, -// IndexLocation.END, new RawBlockCodecInfo()); -// -// it = new GridIterator(shardBlockGridSize); -// i = 0; -// while (it.hasNext()) { -// int j = index.getOffsetIndex(GridIterator.long2int(it.next())); -// assertEquals(i, j); -// i += 2; -// } - - } - - @Test - @Ignore - public void writeReadTest() throws IOException { - - // TODO - -// final N5KeyValueWriter writer = (N5KeyValueWriter)tempN5Factory.createTempN5Writer(); -// final KeyValueAccess kva = writer.getKeyValueAccess(); -// -// final int[] shardBlockGridSize = new int[]{6, 5}; -// final IndexLocation indexLocation = IndexLocation.END; -// final IndexCodecAdapter indexCodecAdapter = new IndexCodecAdapter( -// new RawBlockCodecInfo(), -// new Crc32cChecksumCodec() -// ); -// -// final ShardIndex index = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecAdapter); -// index.set(0, 6, new int[]{0, 0}); -// index.set(19, 32, new int[]{1, 0}); -// index.set(93, 111, new int[]{3, 0}); -// index.set(143, 1, new int[]{1, 2}); -// -// final String path = Paths.get(Paths.get(writer.getURI()).toAbsolutePath().toString(), "indexTest").toString(); -// try ( -// final LockedChannel channel = kva.lockForWriting(path); -// final OutputStream out = channel.newOutputStream() -// ) { -// -// ShardIndex.write(out, index); -// } -// -// final ShardIndex indexRead = new ShardIndex(shardBlockGridSize, indexLocation, indexCodecAdapter); -// try ( -// final LockedChannel channel = kva.lockForReading(path); -// final InputStream in = channel.newInputStream() -// ) { -// ShardIndex.read(in, indexRead); -// } -// assertEquals(index, indexRead); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java new file mode 100644 index 000000000..0fc1c8546 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java @@ -0,0 +1,164 @@ +package org.janelia.saalfeldlab.n5.shard; + +import java.util.Arrays; +import org.janelia.saalfeldlab.n5.ByteArrayDataBlock; +import org.janelia.saalfeldlab.n5.DataBlock; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.DatasetAttributes; +import org.janelia.saalfeldlab.n5.RawCompression; +import org.janelia.saalfeldlab.n5.codec.BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; +import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; +import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.N5Writer.DataBlockSupplier; +import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; + +public class WriteRegionTest { + + +// #...............................#...............................#...............................#...............................# +// $.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$ +// |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...| +// 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96 + + public static void main(String[] args) { + +// int[] datablockSize = {3, 3, 3}; +// int[] level1ShardSize = {6, 6, 6}; +// int[] level2ShardSize = {24, 24, 24}; + int[] datablockSize = {3}; + int[] level1ShardSize = {6}; + int[] level2ShardSize = {24}; + + // DataBlocks are 3x3x3 + // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) + // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) + final BlockCodecInfo c0 = new N5BlockCodecInfo(); + final ShardCodecInfo c1 = new DefaultShardCodecInfo( + datablockSize, + c0, + new DataCodecInfo[] {new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[] {new RawCompression()}, + IndexLocation.END + ); + final ShardCodecInfo c2 = new DefaultShardCodecInfo( + level1ShardSize, + c1, + new DataCodecInfo[] {new RawCompression()}, + new RawBlockCodecInfo(), + new DataCodecInfo[] {new RawCompression()}, + IndexLocation.START + ); + + TestDatasetAttributes attributes = new TestDatasetAttributes( + new long[] {}, + level2ShardSize, + DataType.INT8, + c2, + new RawCompression()); + + final DatasetAccess datasetAccess = attributes.datasetAccess(); + final PositionValueAccess store = new TestPositionValueAccess(); + + // --------------------------------------------------------------- + // Some "tests" + // TODO: Turn into unit tests + // --------------------------------------------------------------- + + // write some blocks, filled with constant values + final int[] dataBlockSize = c1.getInnerBlockSize(); +// final long[] datasetDimensions = {100, 100, 100}; +// final long[] regionMin = {9,9,9}; +// final long[] regionSize = {15,15,15}; + final long[] datasetDimensions = {96}; + + // #...............................#...............................#...............................#...............................# + // $.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$ + // |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...| + // 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96 + + final long[] regionMin = {93}; + final long[] regionSize = {1}; + + DataBlockSupplier blocks = (gridPos, existing) -> { +// System.out.println("BlockSupplier.get(" + Arrays.toString(gridPos) + ", " + existing + ")"); + return createDataBlock(dataBlockSize, gridPos.clone(), (byte) gridPos[0]); + }; + datasetAccess.writeRegion(store, + regionMin, + regionSize, + blocks, + datasetDimensions, + false); + + // verify that the written blocks can be read back with the correct values +// checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), true, 1); +// checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); +// checkBlock(datasetAccess.readBlock(store, new long[] {0, 1, 0}), true, 3); +// checkBlock(datasetAccess.readBlock(store, new long[] {1, 1, 0}), true, 4); +// checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5); +// checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6); + + // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) +// datasetAccess.deleteBlock(store, new long[] {0, 0, 0}); +// checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); +// checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); + + // if a shard becomes empty the corresponding key should be deleted +// if ( store.get(new long[] {1, 0, 0}) == null ) { +// throw new IllegalStateException("expected non-null readData"); +// } +// datasetAccess.deleteBlock(store, new long[] {8, 4, 1}); +// if ( store.get(new long[] {1, 0, 0}) != null ) { +// throw new IllegalStateException("expected null readData"); +// } + + // deleting a non-existent block should not fail +// datasetAccess.deleteBlock(store, new long[] {0, 0, 8}); + + System.out.println("all good"); + } + + private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { + + if (dataBlock == null) { + if (expectedNonNull) { + throw new IllegalStateException("expected non-null dataBlock"); + } + } else { + if (!expectedNonNull) { + throw new IllegalStateException("expected null dataBlock"); + } + final byte[] bytes = dataBlock.getData(); + for (byte b : bytes) { + if (b != (byte) expectedFillValue) { + throw new IllegalStateException("expected all values to be " + expectedFillValue); + } + } + } + } + + private static DataBlock createDataBlock(int[] size, long[] gridPosition, int fillValue) { + final byte[] bytes = new byte[DataBlock.getNumElements(size)]; + Arrays.fill(bytes, (byte) fillValue); + return new ByteArrayDataBlock(size, gridPosition, bytes); + } + + public static class TestDatasetAttributes extends DatasetAttributes { + + public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType dataType, BlockCodecInfo blockCodecInfo, + DataCodecInfo... dataCodecInfos) { + + super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); + } + + public DatasetAccess datasetAccess() { + + // to make this accessible for the test + return createDatasetAccess(); + } + + } + +}