diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java index 70ed4ba60..59b7a6534 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java @@ -1,6 +1,8 @@ package org.janelia.saalfeldlab.n5; +import org.apache.commons.io.input.ProxyInputStream; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import java.io.*; @@ -23,7 +25,15 @@ public Reader newReader() throws N5Exception.N5IOException { @Override public InputStream newInputStream() throws N5Exception.N5IOException { - return kva.createReadData(key).inputStream(); + + VolatileReadData volatileReadData = kva.createReadData(key); + return new ProxyInputStream(volatileReadData.inputStream()) { + @Override + public void close() throws IOException { + super.close(); + volatileReadData.close(); + } + }; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java b/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java index 520a2b367..643d43bd0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ChannelLock.java @@ -2,12 +2,11 @@ import java.io.Closeable; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; /** * Holds a channel and system-level file lock (shared for writing, non-shared @@ -17,11 +16,14 @@ class ChannelLock implements Closeable { private final FileChannel channel; - private final FileLock lock; + + //Used to ensure a hard reference exists until we close + @SuppressWarnings({"unused", "FieldCanBeLocal"}) + private final FileLock unusedHardRef; private ChannelLock(final FileChannel channel, final FileLock lock) { this.channel = channel; - this.lock = lock; + this.unusedHardRef = lock; } public void close() throws IOException { @@ -55,7 +57,7 @@ FileChannel getChannel() { */ static ChannelLock lock(final Path path, final boolean forWriting) throws IOException { - final FileChannel channel = openFileChannel(path, forWriting); + final FileChannel channel = FsIoPolicy.openFileChannel(path, forWriting); try { while (true) { try { @@ -93,7 +95,7 @@ static ChannelLock tryLock(final Path path, final boolean forWriting) throws IOE FileChannel channel = null; try { - channel = openFileChannel(path, forWriting); + channel = FsIoPolicy.openFileChannel(path, forWriting); final FileLock lock = channel.tryLock(0, Long.MAX_VALUE, !forWriting); return lock == null ? null : new ChannelLock(channel, lock); } catch (Exception e) { @@ -102,32 +104,12 @@ static ChannelLock tryLock(final Path path, final boolean forWriting) throws IOE } } - /** - * Opens a file channel. If the channel is opened {@code forWriting}, - * then this may create the file and the parent directories as needed. - * - * @throws IOException - * if the channel cannot be opened - */ - private static FileChannel openFileChannel(final Path path, final boolean forWriting) throws IOException { - - if (forWriting) { - final Path parent = path.getParent(); - if (parent != null) { - Files.createDirectories(parent); - } - return FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE); - } else { - return FileChannel.open(path, StandardOpenOption.READ); - } - } - private static void closeQuietly(final FileChannel fileChannel) { if (fileChannel != null) { try { fileChannel.close(); - } catch (final IOException ignored) { + } catch (final IOException | UncheckedIOException ignored) { } - } + } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 08bebf1e6..4e0ff8d9a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -37,6 +37,7 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; + /** * Enumerates available data types. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index af25c729d..cbc602763 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -89,13 +89,13 @@ public class DatasetAttributes implements Serializable { private final long[] dimensions; + // number of samples per chunk per dimension + private final int[] chunkSize; + // number of samples per block per dimension + // identical to chunkSize for non-sharded datasets private final int[] blockSize; - // TODO add a getter? - // the shard size - private final int[] outerBlockSize; - private final DataType dataType; private final JsonElement defaultValue; @@ -108,7 +108,7 @@ public class DatasetAttributes implements Serializable { public DatasetAttributes( final long[] dimensions, - final int[] outerBlockSize, + final int[] blockSize, final DataType dataType, final JsonElement defaultValue, final BlockCodecInfo blockCodecInfo, @@ -117,7 +117,7 @@ public DatasetAttributes( this.dimensions = dimensions; this.dataType = dataType; - this.outerBlockSize = outerBlockSize; + this.blockSize = blockSize; this.defaultValue = defaultValue == null ? JsonNull.INSTANCE : defaultValue; this.blockCodecInfo = blockCodecInfo == null ? defaultBlockCodecInfo() : blockCodecInfo; @@ -131,7 +131,7 @@ public DatasetAttributes( .toArray(DataCodecInfo[]::new); access = createDatasetAccess(); - blockSize = access.getGrid().getBlockSize(0); + chunkSize = access.getGrid().getBlockSize(0); } public DatasetAttributes( @@ -189,7 +189,7 @@ public DatasetAttributes( this(dimensions, blockSize, dataType, new DataCodecInfo[0]); } - protected DatasetAccess createDatasetAccess() { + private DatasetAccess createDatasetAccess() { final int m = nestingDepth(blockCodecInfo); @@ -200,7 +200,7 @@ protected DatasetAccess createDatasetAccess() { // NestedGrid validates block sizes, so instantiate it before creating the blockCodecs // blockCodecInfo.create below could fail unexpecedly with invalid // blockSizes so validate first - blockSizes[m - 1] = outerBlockSize; + blockSizes[m - 1] = blockSize; BlockCodecInfo tmpInfo = blockCodecInfo; for (int l = m - 1; l > 0; --l) { final ShardCodecInfo info = (ShardCodecInfo)tmpInfo; @@ -210,7 +210,7 @@ protected DatasetAccess createDatasetAccess() { BlockCodecInfo currentBlockCodecInfo = blockCodecInfo; DataCodecInfo[] currentDataCodecInfos = dataCodecInfos; - + DatasetCodecInfo[] datasetCodecInfos = this.datasetCodecInfos; final NestedGrid grid = new NestedGrid(blockSizes, dimensions); @@ -274,6 +274,11 @@ public int getNumDimensions() { return dimensions.length; } + public int[] getChunkSize() { + + return chunkSize; + } + public int[] getBlockSize() { return blockSize; @@ -318,9 +323,9 @@ public DataType getDataType() { * * @return the {@code DatasetAccess} for this dataset */ - DatasetAccess getDatasetAccess() { + protected DatasetAccess getDatasetAccess() { - return (DatasetAccess)access; + return (DatasetAccess) access; } /** @@ -334,7 +339,6 @@ public NestedGrid getNestedBlockGrid() { return getDatasetAccess().getGrid(); } - public BlockCodecInfo getBlockCodecInfo() { return blockCodecInfo; @@ -359,7 +363,7 @@ public HashMap asMap() { final HashMap map = new HashMap<>(); map.put(DIMENSIONS_KEY, dimensions); - map.put(BLOCK_SIZE_KEY, blockSize); + map.put(BLOCK_SIZE_KEY, chunkSize); map.put(DATA_TYPE_KEY, dataType); map.put(COMPRESSION_KEY, getCompression()); return map; @@ -425,7 +429,7 @@ public static class DatasetAttributesAdapter implements JsonSerializer= 0 && offset + length > channelSize) return true; } + /** + * Opens a file channel. If the channel is opened {@code forWriting}, + * then this may create the file and the parent directories as needed. + * + * @throws IOException + * if the channel cannot be opened + */ + static FileChannel openFileChannel(final Path path, final boolean forWriting) throws IOException { + + if (forWriting) { + final Path parent = path.getParent(); + /* if not null and not directory, it will call `createDirectories` but we expect it to throw an IOException */ + if (parent != null && !parent.toFile().isDirectory()) { + Files.createDirectories(parent); + } + return FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + } else { + return FileChannel.open(path, StandardOpenOption.READ); + } + } + + + /** + * This method is necessary to handle the situtation where writing is successful, but `close` fails on the file channel. + * This has been observed to happen fairly consistently on MacOS when writing to a file mounted over SMB. + * + * @param readData to write to the {@code Path} + * @param path to write to + * @throws IOException if writing failed. + */ + private static void writeToPathIgnoreCloseException(ReadData readData, Path path) throws IOException { + + FileChannel channel = openFileChannel(path, true); + OutputStream os = Channels.newOutputStream(channel); + + try { + readData.writeTo(os); + os.flush(); + channel.force(true); + } catch (Throwable e) { + os.close(); + channel.close(); + throw e; + } + + /* if we get here, the write succeeded, and the os/channel may not be closed yet */ + try { + os.close(); + channel.close(); + } catch (IOException | UncheckedIOException ignore) { + /* Ignore; we know the data was written already. */ + } + } + public static class Unsafe implements IoPolicy { @Override public void write(String key, ReadData readData) throws IOException { final Path path = Paths.get(key); - Files.copy(readData.inputStream(), path, StandardCopyOption.REPLACE_EXISTING); + writeToPathIgnoreCloseException(readData, path); } @Override @@ -69,6 +125,8 @@ public VolatileReadData read(String key) throws IOException { @Override public void delete(final String key) throws IOException { final Path path = Paths.get(key); + if (!Files.isRegularFile(path)) + Files.delete(path); try (LockedFileChannel ignore = FILE_LOCK_MANAGER.lockForWriting(path)) { Files.delete(path); } @@ -110,6 +168,7 @@ public ReadData materialize(final long offset, final long length) { throw new N5Exception.N5IOException("FileLazyRead is already closed."); } + ReadData readData = null; try (final FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) { channel.position(offset); @@ -127,13 +186,19 @@ public ReadData materialize(final long offset, final long length) { final byte[] data = new byte[(int) size]; final ByteBuffer buf = ByteBuffer.wrap(data); channel.read(buf); - return ReadData.from(data); + readData = ReadData.from(data); } catch (final NoSuchFileException e) { throw new N5Exception.N5NoSuchKeyException("No such file", e); } catch (IOException | UncheckedIOException e) { - throw new N5Exception.N5IOException(e); + /* Occasionally (frequently for some source remote mounted file systems) this can throw exceptions during + * `channel.close()` which is called automatically in the try-with-resources block. In this case, we have + * successfully read the data, and we can return it, and ignore the exception. + * */ + if (readData == null) + throw new N5Exception.N5IOException(e); } + return readData; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 0e40a1c88..25eb01994 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -94,7 +94,7 @@ default JsonElement getAttributes(final String pathName) throws N5Exception { } @Override - default DataBlock readBlock( + default DataBlock readChunk( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { @@ -103,7 +103,7 @@ default DataBlock readBlock( try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), convertedDatasetAttributes); - return convertedDatasetAttributes. getDatasetAccess().readBlock(posKva, gridPosition); + return convertedDatasetAttributes. getDatasetAccess().readChunk(posKva, gridPosition); } catch (N5Exception.N5NoSuchKeyException e) { return null; @@ -111,18 +111,18 @@ default DataBlock readBlock( } @Override - default List> readBlocks( + default List> readChunks( final String pathName, final DatasetAttributes datasetAttributes, final List blockPositions) throws N5Exception { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), convertedDatasetAttributes); - return convertedDatasetAttributes. getDatasetAccess().readBlocks(posKva, blockPositions); + return convertedDatasetAttributes. getDatasetAccess().readChunks(posKva, blockPositions); } @Override - default DataBlock readShard( + default DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { @@ -132,7 +132,7 @@ default DataBlock readShard( try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(pathName), convertedDatasetAttributes); - return convertedDatasetAttributes. getDatasetAccess().readShard(posKva, gridPosition, shardLevel); + return convertedDatasetAttributes. getDatasetAccess().readBlock(posKva, gridPosition, shardLevel); } catch (N5Exception.N5NoSuchKeyException e) { return null; @@ -172,7 +172,7 @@ default String absoluteAttributesPath(final String normalPath) { } @Override - default boolean shardExists( + default boolean blockExists( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index f5bacc94f..96e164326 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -84,7 +84,7 @@ default void createGroup(final String path) throws N5Exception { /** * Helper method that writes an attributes tree into the store - * + *

* TODO This method is not part of the public API and should be protected * in Java versions greater than 8 * @@ -225,12 +225,12 @@ default void writeRegion( final DatasetAttributes datasetAttributes, final long[] min, final long[] size, - final DataBlockSupplier dataBlocks, + final DataBlockSupplier chunkSupplier, final boolean writeFully) throws N5Exception { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes); - convertedDatasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, writeFully); + convertedDatasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, chunkSupplier, writeFully); } catch (final UncheckedIOException e) { throw new N5IOException( "Failed to write blocks into dataset " + datasetPath, e); @@ -243,13 +243,13 @@ default void writeRegion( final DatasetAttributes datasetAttributes, final long[] min, final long[] size, - final DataBlockSupplier dataBlocks, + final DataBlockSupplier chunkSupplier, final boolean writeFully, final ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes); - convertedDatasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, dataBlocks, writeFully, exec); + convertedDatasetAttributes.getDatasetAccess().writeRegion(posKva, min, size, chunkSupplier, writeFully, exec); } catch (final UncheckedIOException e) { throw new N5IOException( "Failed to write blocks into dataset " + datasetPath, e); @@ -257,52 +257,52 @@ default void writeRegion( } @Override - default void writeBlocks( + default void writeChunks( final String datasetPath, final DatasetAttributes datasetAttributes, - final DataBlock... dataBlocks) throws N5Exception { + final DataBlock... chunks) throws N5Exception { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(datasetPath), convertedDatasetAttributes); - convertedDatasetAttributes.getDatasetAccess().writeBlocks(posKva, Arrays.asList(dataBlocks)); + convertedDatasetAttributes.getDatasetAccess().writeChunks(posKva, Arrays.asList(chunks)); } catch (final UncheckedIOException e) { throw new N5IOException( - "Failed to write blocks into dataset " + datasetPath, e); + "Failed to write chunks into dataset " + datasetPath, e); } } @Override - default void writeBlock( + default void writeChunk( final String path, final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws N5Exception { + final DataBlock chunk) throws N5Exception { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes); - convertedDatasetAttributes. getDatasetAccess().writeBlock(posKva, dataBlock); + convertedDatasetAttributes. getDatasetAccess().writeChunk(posKva, chunk); } catch (final UncheckedIOException e) { throw new N5IOException( - "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, + "Failed to write chunk " + Arrays.toString(chunk.getGridPosition()) + " into dataset " + path, e); } } @Override - default void writeShard( + default void writeBlock( final String path, final DatasetAttributes datasetAttributes, - final DataBlock shard) throws N5Exception { + final DataBlock dataBlock) throws N5Exception { final DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); final int shardLevel = convertedDatasetAttributes.getNestedBlockGrid().numLevels() - 1; try { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), convertedDatasetAttributes); - convertedDatasetAttributes. getDatasetAccess().writeShard(posKva, shard, shardLevel); + convertedDatasetAttributes. getDatasetAccess().writeBlock(posKva, dataBlock, shardLevel); } catch (final UncheckedIOException e) { throw new N5IOException( - "Failed to write block " + Arrays.toString(shard.getGridPosition()) + " into dataset " + path, + "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, e); } } @@ -326,16 +326,26 @@ default boolean deleteBlock( final long... gridPosition) throws N5Exception { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); - return datasetAttributes.getDatasetAccess().deleteBlock(posKva, gridPosition); + return posKva.remove(gridPosition); + } + + @Override + default boolean deleteChunk( + final String path, + final DatasetAttributes datasetAttributes, + final long... gridPosition) throws N5Exception { + + final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); + return datasetAttributes.getDatasetAccess().deleteChunk(posKva, gridPosition); } @Override - default boolean deleteBlocks( + default boolean deleteChunks( final String path, final DatasetAttributes datasetAttributes, final List gridPositions) throws N5Exception { final PositionValueAccess posKva = PositionValueAccess.fromKva(getKeyValueAccess(), getURI(), N5URI.normalizeGroupPath(path), datasetAttributes); - return datasetAttributes.getDatasetAccess().deleteBlocks(posKva, gridPositions); + return datasetAttributes.getDatasetAccess().deleteChunks(posKva, gridPositions); } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 3d3db1aeb..e7155bbda 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -94,7 +94,7 @@ static T readAttribute( * @param gson * used to deserialize {@code attribute} * @param type - * to desrialize {@code attribute} as + * to deserialize {@code attribute} as * @param * return type represented by {@link Type type} * @return the deserialized attribute object, or {@code null} if @@ -548,7 +548,7 @@ static void writeAttribute( } /** - * Writes the attributes JsonElemnt to a given {@link Writer}. + * Writes the attributes JsonElement to a given {@link Writer}. * This will overwrite any existing attributes. * * @param writer diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java index b678dfe41..0e84071ab 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LockedFileChannel.java @@ -12,8 +12,7 @@ /** * LockedFileChannel implementation for both read and write operations. */ -// TODO: This only has to be public because of a test in another package. Fix that -public class LockedFileChannel implements LockedChannel { +class LockedFileChannel implements LockedChannel { private final FileChannel channel; private ReleaseLock releaseLock; @@ -45,32 +44,15 @@ public InputStream newInputStream() throws N5Exception.N5IOException { @Override public Writer newWriter() throws N5Exception.N5IOException { - truncateChannel(); return Channels.newWriter(channel, StandardCharsets.UTF_8.name()); } @Override public OutputStream newOutputStream() throws N5Exception.N5IOException { - truncateChannel(); return Channels.newOutputStream(channel); } - // TODO: This only has to be public because of a test in another package. Fix that - public FileChannel getFileChannel() { - - return channel; - } - - private void truncateChannel() throws N5Exception.N5IOException { - - try { - channel.truncate(0); - } catch (final IOException e) { - throw new N5Exception.N5IOException("Failed to truncate channel", e); - } - } - @Override public void close() throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java index a97eeab51..5881e33a7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java @@ -28,9 +28,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.io.UncheckedIOException; - public class N5Exception extends RuntimeException { public N5Exception() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index aebbfa67d..acd8a1de7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -45,6 +45,7 @@ import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; /** * A simple structured container for hierarchies of chunked @@ -55,7 +56,7 @@ */ public interface N5Reader extends AutoCloseable { - public static class Version { + class Version { private final int major; private final int minor; @@ -192,17 +193,17 @@ public boolean isCompatible(final Version version) { /** * SemVer version of this N5 spec. */ - public static final Version NO_VERSION = new Version(0, 0, 0); + Version NO_VERSION = new Version(0, 0, 0); /** * SemVer version of this N5 spec. */ - public static final Version VERSION = new Version(4, 0, 0); + Version VERSION = new Version(4, 0, 0); /** * Version attribute key. */ - public static final String VERSION_KEY = "n5"; + String VERSION_KEY = "n5"; /** * Get the SemVer version of this container as specified in the 'version' @@ -245,9 +246,9 @@ default Version getVersion() throws N5Exception { * the exception */ T getAttribute( - final String pathName, - final String key, - final Class clazz) throws N5Exception; + String pathName, + String key, + Class clazz) throws N5Exception; /** * Reads an attribute. @@ -265,9 +266,9 @@ T getAttribute( * the exception */ T getAttribute( - final String pathName, - final String key, - final Type type) throws N5Exception; + String pathName, + String key, + Type type) throws N5Exception; /** * Get mandatory dataset attributes. @@ -279,7 +280,7 @@ T getAttribute( * @throws N5Exception * the exception */ - DatasetAttributes getDatasetAttributes(final String pathName) throws N5Exception; + DatasetAttributes getDatasetAttributes(String pathName) throws N5Exception; /** * Some implementations may need to convert arbitrary DatasetAttributes to their specific equivalent variant. @@ -298,7 +299,7 @@ default DatasetAttributes getConvertedDatasetAttributes(final DatasetAttributes /** - * Reads a {@link DataBlock}. + * Reads a chunk as a {@link DataBlock}. * * @param * the DataBlock data type @@ -312,13 +313,13 @@ default DatasetAttributes getConvertedDatasetAttributes(final DatasetAttributes * @throws N5Exception * the exception */ - DataBlock readBlock( - final String pathName, - final DatasetAttributes datasetAttributes, - final long... gridPosition) throws N5Exception; + DataBlock readChunk( + String pathName, + DatasetAttributes datasetAttributes, + long... gridPosition) throws N5Exception; /** - * Reads multiple {@link DataBlock}s. + * Reads multiple chunks as {@link DataBlock}s. *

* Implementations may optimize / batch read operations when possible, e.g. * in the case that the datasets are sharded. @@ -335,7 +336,7 @@ DataBlock readBlock( * @throws N5Exception * the exception */ - default List> readBlocks( + default List> readChunks( final String pathName, final DatasetAttributes datasetAttributes, final List gridPositions) throws N5Exception { @@ -343,16 +344,16 @@ default List> readBlocks( DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); final ArrayList> blocks = new ArrayList<>(); for( final long[] p : gridPositions ) - blocks.add(readBlock(pathName, convertedDatasetAttributes, p)); + blocks.add(readChunk(pathName, convertedDatasetAttributes, p)); return blocks; } /** - * Reads a shard as a {@link DataBlock}. + * Reads a block, returning a {@link DataBlock}. Will be a chunk or shard (if the dataset is sharded). *

- * A "shard" is the largest level of the datasets {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid} - * This method's behavior is identical to readBlock for un-sharded datasets. + * A block is the highest (coarsest) level of the dataset's {@link NestedGrid} + * This method's behavior is identical to {@link #readChunk} for un-sharded datasets. * * @param * the DataBlock data type @@ -361,39 +362,44 @@ default List> readBlocks( * @param datasetAttributes * the dataset attributes * @param gridPosition - * the position in the shard grid + * the position in the block grid * @return the data block * @throws N5Exception * the exception * * @see DatasetAttributes#getNestedBlockGrid() */ - DataBlock readShard( - final String pathName, - final DatasetAttributes datasetAttributes, - final long... gridPosition) throws N5Exception; + DataBlock readBlock( + String pathName, + DatasetAttributes datasetAttributes, + long... gridPosition) throws N5Exception; /** - * Checks if a shard (or block for non-sharded datasets) exists at the - * given grid position without reading the data. + * Checks if a block exists at the given grid position without reading the data. + *

+ * A block is the highest (coarsest) level of the dataset's {@link + * NestedGrid}, that is, a shard if the dataset is sharded, or a chunk + * otherwise. *

* This method only checks for the presence of the key value for the gridPosition, it does not - * read or validate the contents. + * read or validate the contents. As a result, this method refers to chunks in un-sharded datasets + * or shards in sharded datasets. * * @param pathName * dataset path * @param datasetAttributes * the dataset attributes * @param gridPosition - * the shard grid position (or block position for non-sharded datasets) - * @return true if the shard (or block) file exists + * the block grid position + * @return true if the block file exists * @throws N5Exception * the exception */ - boolean shardExists( - final String pathName, - final DatasetAttributes datasetAttributes, - final long... gridPosition) throws N5Exception; + boolean blockExists( + String pathName, + DatasetAttributes datasetAttributes, + long... gridPosition) throws N5Exception; + /** * Load a {@link DataBlock} as a {@link Serializable}. The offset is given * in @@ -419,7 +425,7 @@ default T readSerializedBlock( final long... gridPosition) throws N5Exception, ClassNotFoundException { DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(attributes); - final DataBlock block = readBlock(dataset, convertedDatasetAttributes, gridPosition); + final DataBlock block = readChunk(dataset, convertedDatasetAttributes, gridPosition); if (block == null) return null; @@ -438,7 +444,7 @@ default T readSerializedBlock( * group path * @return true if the path exists */ - boolean exists(final String pathName); + boolean exists(String pathName); /** * Test whether a dataset exists at a given path. @@ -463,7 +469,7 @@ default boolean datasetExists(final String pathName) throws N5Exception { * @throws N5Exception * an exception is thrown if pathName is not a valid group */ - String[] list(final String pathName) throws N5Exception; + String[] list(String pathName) throws N5Exception; /** * Recursively list all groups and datasets in the given path. @@ -600,7 +606,7 @@ default String[] deepListDatasets(final String pathName) throws N5Exception { * Helper method to recursively list all groups and datasets. This method is not part of the * public API and is accessible only because Java 8 does not support private * interface methods yet. - * + *

* TODO make private when committing to Java versions newer than 8 * * @param n5 the n5 reader @@ -672,7 +678,7 @@ default String[] deepList( results.add(result.substring(normalPathName.length() + groupSeparator.length())); } - return results.stream().toArray(String[]::new); + return results.toArray(new String[0]); } /** @@ -759,7 +765,7 @@ default String[] deepListDatasets( results.add(result.substring(normalPathName.length() + groupSeparator.length())); } - return results.stream().toArray(String[]::new); + return results.toArray(new String[0]); } /** @@ -867,7 +873,7 @@ static void deepListHelper( * @return a map of attribute keys to their inferred class * @throws N5Exception if an error occurred during listing */ - Map> listAttributes(final String pathName) throws N5Exception; + Map> listAttributes(String pathName) throws N5Exception; /** * Returns the symbol that is used to separate nodes in a group path. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java index c086acc05..9bd9fa326 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java @@ -36,6 +36,7 @@ import java.nio.charset.CharsetDecoder; import java.nio.charset.CoderResult; import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -53,7 +54,7 @@ */ public class N5URI { - private static final Charset UTF8 = Charset.forName("UTF-8"); + private static final Charset UTF8 = StandardCharsets.UTF_8; public static final Pattern ARRAY_INDEX = Pattern.compile("\\[([0-9]+)]"); final URI uri; private final String scheme; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 22ab44667..d0faf18f8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -6,13 +6,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -39,6 +39,7 @@ import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; /** * A simple structured container API for hierarchies of chunked @@ -55,7 +56,7 @@ public interface N5Writer extends N5Reader { * @param groupPath group path * @param attributePath the key * @param attribute the attribute - * @param the attribute type type + * @param the attribute type * @throws N5Exception the exception */ default void setAttribute( @@ -77,8 +78,8 @@ default void setAttribute( * @throws N5Exception the exception */ void setAttributes( - final String groupPath, - final Map attributes) throws N5Exception; + String groupPath, + Map attributes) throws N5Exception; /** * Remove the attribute from group {@code pathName} with key {@code key}. @@ -141,7 +142,7 @@ default void setDatasetAttributes( final String datasetPath, final DatasetAttributes datasetAttributes) throws N5Exception { - setAttribute(datasetPath, "/", getConvertedDatasetAttributes(datasetAttributes)); + setAttributes(datasetPath, getConvertedDatasetAttributes(datasetAttributes).asMap()); } /** @@ -164,7 +165,7 @@ default void setVersion() throws N5Exception { * @param groupPath the path * @throws N5Exception the exception */ - void createGroup(final String groupPath) throws N5Exception; + void createGroup(String groupPath) throws N5Exception; /** * Removes a group or dataset (directory and all contained files). @@ -182,7 +183,7 @@ default void setVersion() throws N5Exception { * @return true if removal was successful, false otherwise * @throws N5Exception the exception */ - boolean remove(final String groupPath) throws N5Exception; + boolean remove(String groupPath) throws N5Exception; /** * Removes the N5 container. @@ -212,7 +213,7 @@ default DatasetAttributes createDataset( final String normalPath = N5URI.normalizeGroupPath(datasetPath); createGroup(normalPath); - DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); + final DatasetAttributes convertedDatasetAttributes = getConvertedDatasetAttributes(datasetAttributes); setDatasetAttributes(normalPath, convertedDatasetAttributes); return convertedDatasetAttributes; } @@ -241,45 +242,47 @@ default DatasetAttributes createDataset( } /** - * Writes a {@link DataBlock}. + * Writes a chunk represented by a {@link DataBlock}. * * @param datasetPath dataset path * @param datasetAttributes the dataset attributes - * @param dataBlock the data block + * @param chunk the chunk as a DataBlock * @param the data block data type * @throws N5Exception the exception */ - void writeBlock( - final String datasetPath, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws N5Exception; + void writeChunk( + String datasetPath, + DatasetAttributes datasetAttributes, + DataBlock chunk) throws N5Exception; /** - * Write multiple data blocks, useful for request aggregation. + * Write multiple chunks represented by {@link DataBlock}s, useful for aggregation. * * @param datasetPath dataset path * @param datasetAttributes the dataset attributes - * @param dataBlocks the data block + * @param chunks the chunks * @param the data block data type * @throws N5Exception the exception */ - default void writeBlocks( + default void writeChunks( final String datasetPath, final DatasetAttributes datasetAttributes, - final DataBlock... dataBlocks) throws N5Exception { + final DataBlock... chunks) throws N5Exception { // default method is naive DatasetAttributes convertedAttributes = getConvertedDatasetAttributes(datasetAttributes); - for (DataBlock block : dataBlocks) { - writeBlock(datasetPath, convertedAttributes, block); + for (DataBlock block : chunks) { + writeChunk(datasetPath, convertedAttributes, block); } } /** - * Writes a shard stored as a {@link DataBlock}. + * Writes a block stored as a {@link DataBlock}. + *

+ * A block is the highest (coarsest) level of the dataset's {@link NestedGrid}, + * that is, a shard if the dataset is sharded, or a chunk otherwise. *

- * A "shard" is the largest level of the datasets {@link org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid}. - * This method's behavior is identical to writeBlock for un-sharded datasets. + * This method's behavior is identical to {@link #writeChunk} for un-sharded datasets. * * @param pathName dataset path * @param datasetAttributes the dataset attributes @@ -289,10 +292,10 @@ default void writeBlocks( * * @see DatasetAttributes#getNestedBlockGrid() */ - void writeShard( - final String pathName, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws N5Exception; + void writeBlock( + String pathName, + DatasetAttributes datasetAttributes, + DataBlock dataBlock) throws N5Exception; @FunctionalInterface interface DataBlockSupplier { @@ -301,11 +304,11 @@ interface DataBlockSupplier { * * @param gridPos * @param existingDataBlock - * existing data to be merged into the new data block (maybe {@code null}) + * existing data to be merged into the new data block (may be {@code null}) * * @return data block at the given gridPos */ - DataBlock get(long[] gridPos, final DataBlock existingDataBlock); + DataBlock get(long[] gridPos, DataBlock existingDataBlock); } /** @@ -313,26 +316,26 @@ interface DataBlockSupplier { * @param datasetAttributes the dataset attributes * @param min min pixel coordinate of region to write * @param size size in pixels of region to write - * @param dataBlocks is asked to create blocks within the given region + * @param chunkSupplier is asked to create chunks within the given region * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. * @throws N5Exception the exception */ void writeRegion( - final String datasetPath, - final DatasetAttributes datasetAttributes, - final long[] min, - final long[] size, - final DataBlockSupplier dataBlocks, - final boolean writeFully) throws N5Exception; + String datasetPath, + DatasetAttributes datasetAttributes, + long[] min, + long[] size, + DataBlockSupplier chunkSupplier, + boolean writeFully) throws N5Exception; /** * @param datasetPath the dataset path * @param datasetAttributes the dataset attributes * @param min min pixel coordinate of region to write * @param size size in pixels of region to write - * @param dataBlocks is asked to create blocks within the given region - * @param writeFully if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. - * @param exec used to parallelize over blocks and shards + * @param chunkSupplier is asked to create chunks within the given region + * @param writeFully if false, merge existing data in shards/chunks that overlap the region boundary. if true, override everything. + * @param exec used to parallelize over blocks (chunks and shards) * @throws N5Exception the exception */ void writeRegion( @@ -340,12 +343,19 @@ void writeRegion( DatasetAttributes datasetAttributes, long[] min, long[] size, - DataBlockSupplier dataBlocks, + DataBlockSupplier chunkSupplier, boolean writeFully, ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException; /** * Deletes the block at {@code gridPosition}. + *

+ * A block is the highest (coarsest) level of the dataset's {@link NestedGrid}, + * that is, a shard if the dataset is sharded, or a chunk otherwise. + * Note that {@code gridPosition} is in units of blocks at the highest + * (coarsest) level, too. + *

+ * This method's behavior is identical to {@link #deleteChunk} for un-sharded datasets. * * @param datasetPath dataset path * @param gridPosition position of block to be deleted @@ -362,6 +372,13 @@ default boolean deleteBlock( /** * Deletes the block at {@code gridPosition}. + *

+ * A block is the highest (coarsest) level of the dataset's {@link NestedGrid}, + * that is, a shard if the dataset is sharded, or a chunk otherwise. + * Note that {@code gridPosition} is in units of blocks at the highest + * (coarsest) level, too. + *

+ * This method's behavior is identical to {@link #deleteChunk} for un-sharded datasets. * * @param datasetPath the dataset path * @param datasetAttributes the dataset attributes @@ -376,28 +393,64 @@ boolean deleteBlock( long... gridPosition) throws N5Exception; /** - * Deletes the blocks at the given {@code gridPositions}. + * Deletes the chunk at {@code gridPosition}. + *

+ * Note that {@code gridPosition} is in units of chunks. * * @param datasetPath dataset path - * @param gridPositions a list of grid positions - * @return {@code true} if any of the specified blocks existed and was deleted - * @throws N5Exception if any of the block exists but could not be deleted + * @param gridPosition position of chunk to be deleted + * @throws N5Exception if the chunk exists but could not be deleted + * + * @return {@code true} if the chunk at {@code gridPosition} existed and was deleted. + */ + default boolean deleteChunk( + final String datasetPath, + final long... gridPosition) throws N5Exception { + final DatasetAttributes datasetAttributes = getDatasetAttributes(datasetPath); + return deleteChunk(datasetPath, datasetAttributes, gridPosition); + } + + /** + * Deletes the chunk at {@code gridPosition}. + *

+ * Note that {@code gridPosition} is in units of chunks. + * + * @param datasetPath the dataset path + * @param datasetAttributes the dataset attributes + * @param gridPosition position of chunk to be deleted + * @throws N5Exception if the chunk exists but could not be deleted + * + * @return {@code true} if the chunk at {@code gridPosition} existed and was deleted. */ - default boolean deleteBlocks( + boolean deleteChunk( String datasetPath, DatasetAttributes datasetAttributes, - List gridPositions) throws N5Exception { + long... gridPosition) throws N5Exception; + + /** + * Deletes the chunks at the given {@code gridPositions}. + *

+ * Note that {@code gridPositions} are in units of chunks. + * + * @param datasetPath dataset path + * @param gridPositions a list of grid positions + * @return {@code true} if any of the specified chunks existed and was deleted + * @throws N5Exception if any of the chunks did exist but could not be deleted + */ + default boolean deleteChunks( + final String datasetPath, + final DatasetAttributes datasetAttributes, + final List gridPositions) throws N5Exception { boolean deleted = false; for (long[] pos : gridPositions) { - deleted |= deleteBlock(datasetPath, datasetAttributes, pos); + deleted |= deleteChunk(datasetPath, datasetAttributes, pos); } return deleted; } /** * Save a {@link Serializable} as an N5 {@link DataBlock} at a given offset. - * The - * offset is given in {@link DataBlock} grid coordinates. + * The offset is given in {@link DataBlock} grid coordinates. * * @param object the object to serialize * @param datasetPath the dataset path @@ -419,6 +472,6 @@ default void writeSerializedBlock( } final byte[] bytes = byteOutputStream.toByteArray(); final DataBlock dataBlock = new ByteArrayDataBlock(null, gridPosition, bytes); - writeBlock(datasetPath, datasetAttributes, dataBlock); + writeChunk(datasetPath, datasetAttributes, dataBlock); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java index a863683cf..6a37247f8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DataCodecInfo.java @@ -32,7 +32,7 @@ import org.janelia.saalfeldlab.n5.serialization.NameConfig; /** - * Used to Create {@code DataCodec}s, which transform one {@link ReadData} into another, + * Used to create {@code DataCodec}s, which transform one {@link ReadData} into another, * for example, applying compression. */ @NameConfig.Prefix("data-codec") diff --git a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java index 06618ef6f..60f92762b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/codec/DeterministicSizeCodecInfo.java @@ -29,13 +29,14 @@ package org.janelia.saalfeldlab.n5.codec; /** - * A {@link CodecInfo} that can deterministically determine the size of encoded data from the size of the raw data and vice versa from the data length alone (i.e. encoding is data - * independent). + * A {@link CodecInfo} that can deterministically determine the size of encoded + * data from the size of the raw data and vice versa from the data length alone + * (i.e. encoding is data independent). */ public interface DeterministicSizeCodecInfo extends CodecInfo { - public abstract long encodedSize(long size); + long encodedSize(long size); - public abstract long decodedSize(long size); + long decodedSize(long size); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java index eff501ba2..06e753366 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java @@ -28,6 +28,8 @@ */ package org.janelia.saalfeldlab.n5.readdata; +import java.util.ArrayList; +import java.util.Collection; import java.util.Comparator; /** @@ -61,7 +63,7 @@ default long end() { } static boolean equals(final Range r0, final Range r1) { - if (r0 == null && r1==null) { + if (r0 == null && r1 == null) { return true; } else if (r0 == null || r1 == null) { return false; @@ -119,4 +121,45 @@ public String toString() { return new DefaultRange(offset, length); } + + /** + * Returns a potentially new collection of {@link Range}s such that + * adjacent or overlapping Ranges are combined. + *

+ * If the input ranges are non-adjacent the input instance is returned, but if aggregation + * occurs, the result will be a new, sorted List. + * + * @param ranges + * a collection of Ranges + * + * @return collection with adjacent or overlapping ranges merged + */ + static Collection aggregate(final Collection ranges) { + + if (ranges.size() == 0) + return ranges; + + final ArrayList sortedRanges = new ArrayList<>(ranges); + sortedRanges.sort(Range.COMPARATOR); + + final ArrayList result = new ArrayList<>(); + Range lo = null; + for (Range hi : sortedRanges) { + + if (lo == null) + lo = hi; + else if (lo.end() >= hi.offset()) { + // merge + lo = Range.at(lo.offset(), Math.max(lo.end(), hi.end()) - lo.offset()); + } else { + result.add(lo); + lo = hi; + } + } + result.add(lo); + + final boolean wereMerges = ranges.size() != result.size(); + return wereMerges ? result : ranges; + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java index 122a399ef..25da8b0b8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java @@ -1,7 +1,7 @@ package org.janelia.saalfeldlab.n5.readdata; -import java.io.InputStream; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.prefetch.AggregatingPrefetchLazyRead; /** * During its life-time, the content of a {@code VolatileReadData} should not be @@ -29,7 +29,8 @@ public interface VolatileReadData extends ReadData, AutoCloseable { * @return a new VolatileReadData */ static VolatileReadData from(final LazyRead lazyRead) { - return new LazyReadData(lazyRead); + final LazyRead aggregatingLazyRead = new AggregatingPrefetchLazyRead(lazyRead); + return new LazyReadData(aggregatingLazyRead); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java new file mode 100644 index 000000000..dcc0097a1 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java @@ -0,0 +1,44 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.util.ArrayList; +import java.util.Collection; + +import java.util.List; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.Range; + +/** + * A {@link SliceTrackingLazyRead} that implements {@link #prefetch} to + * aggregate overlapping / adjacent ranges and then materialize each aggregated + * range. + */ +public class AggregatingPrefetchLazyRead extends SliceTrackingLazyRead { + + public AggregatingPrefetchLazyRead(final LazyRead delegate) { + super(delegate); + } + + /** + * Indicates that the given slices will be subsequently read. + *

+ * This implementation groups overlapping / adjacent {@link Range}s into single read requests. + * + * @param ranges + * slice ranges to prefetch + * + * @throws N5IOException + * if any I/O error occurs + */ + @Override + public void prefetch(final Collection ranges) throws N5IOException { + + final List filteredRanges = new ArrayList<>(ranges); + filteredRanges.removeIf(this::isCovered); + final Collection aggregatedRanges = Range.aggregate(filteredRanges); + for (final Range slice : aggregatedRanges) { + materialize(slice.offset(), slice.length()); + } + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/EnclosingPrefetchLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/EnclosingPrefetchLazyRead.java new file mode 100644 index 000000000..b0d6fdc7a --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/EnclosingPrefetchLazyRead.java @@ -0,0 +1,49 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.util.Collection; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.Range; + +/** + * A {@link SliceTrackingLazyRead} that implements {@link #prefetch} to + * materialize the bounding range of all requested ranges. + */ +public class EnclosingPrefetchLazyRead extends SliceTrackingLazyRead { + + public EnclosingPrefetchLazyRead(final LazyRead delegate) { + super(delegate); + } + + /** + * Indicates that the given slices will be subsequently read. + * {@code LazyRead} implementations (optionally) may take steps to prepare + * for these subsequent slices. + *

+ * Minimal implementation: Find offset and length covering all ranges that + * are not yet fully covered by existing slices. Then materialize the slice + * covering that range. + * + * @param ranges + * slice ranges to prefetch + * + * @throws N5IOException + * if any I/O error occurs + */ + @Override + public void prefetch(final Collection ranges) throws N5IOException { + + long fromIndex = Long.MAX_VALUE; + long toIndex = Long.MIN_VALUE; + for (final Range slice : ranges) { + if (!isCovered(slice)) { + fromIndex = Math.min(fromIndex, slice.offset()); + toIndex = Math.max(toIndex, slice.end()); + } + } + + if (fromIndex < toIndex) { + materialize(fromIndex, toIndex - fromIndex); + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java new file mode 100644 index 000000000..5835c473e --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java @@ -0,0 +1,90 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.Range; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +/** + * A {@link LazyRead} that wraps a delegate {@code LazyRead} and keeps track of + * all slices that have been {@link #materialize materialized}. + *

+ * When materializing a new slice, we first check whether it is completely + * covered by a materialized slice that we already track. If so, then we just + * return a slice on the existing materialized slice. If not, we materialize the + * slice from the delegate track it. + */ +public class SliceTrackingLazyRead implements LazyRead { + + protected static class Slice implements Range { + + // Offset and length in the delegate + private final long offset; + private final long length; + + // Data of this slice + private final ReadData data; + + Slice(final long offset, final long length, final ReadData data) { + this.offset = offset; + this.length = length; + this.data = data; + } + + @Override + public long offset() { + return offset; + } + + @Override + public long length() { + return length; + } + + @Override + public String toString() { + return "{" + offset + ", " + length + '}'; + } + } + + protected final List slices = new ArrayList<>(); + + /** + * The {@code LazyRead} providing our data. + */ + private final LazyRead delegate; + + public SliceTrackingLazyRead(final LazyRead delegate) { + this.delegate = delegate; + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + @Override + public ReadData materialize(final long offset, final long length) throws N5IOException { + final Slice containing = Slices.findContainingSlice(slices, offset, length); + if (containing != null) { + return containing.data.slice(offset - containing.offset, length); + } else { + final ReadData data = delegate.materialize(offset, length); + Slices.addSlice(slices, new Slice(offset, length, data)); + return data; + } + } + + @Override + public long size() throws N5IOException { + return delegate.size(); + } + + protected boolean isCovered(final Range slice) { + + return Slices.findContainingSlice(slices, slice) != null; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java new file mode 100644 index 000000000..d3d0dfb11 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java @@ -0,0 +1,116 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import org.janelia.saalfeldlab.n5.readdata.Range; + +class Slices { + + private Slices() { + // utility class. should not be instantiated. + } + + /** + * In an ordered list of {@code slices}, find a slice that completely contains the given range. + *

+ * Pre-conditions: + *

    + *
  1. Slices are ordered by offset.
  2. + *
  3. If two slices overlap, no slice is fully contained within the other. + * (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
  4. + *
+ * + * @param slices + * ordered list of slices + * @param offset + * start of the range to cover + * @param length + * length of the range to cover + * + * @return a slice that completely contains the requested range, or {@code null} if no such slice exists + */ + static T findContainingSlice(final List slices, final long offset, final long length) { + + // Find the slice with the largest slice.offset <= offset. + final int i = Collections.binarySearch(slices, Range.at(offset, 0), Comparator.comparingLong(Range::offset)); + + // Largest index of a slice with slice.offset <= offset. + final int index = i < 0 ? -i - 2 : i; + if (index < 0) { + // We find no overlapping slice, because + // slices[0].offset is already too large. + return null; + } + + final T slice = slices.get(index); + if (slice.end() < offset + length) { + return null; + } + + return slice; + } + + /** + * In an ordered list of {@code slices}, find a slice that completely contains the given range. + *

+ * Pre-conditions: + *

    + *
  1. Slices are ordered by offset.
  2. + *
  3. If two slices overlap, no slice is fully contained within the other. + * (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
  4. + *
+ * + * @param slices + * ordered list of slices + * @param range + * range to cover + * + * @return a slice that completely contains the requested range, or {@code null} if no such slice exists + */ + static T findContainingSlice(final List slices, final Range range) { + return findContainingSlice(slices, range.offset(), range.length()); + } + + /** + * Add a new {@code slice} to the {@code slice} list. + *

+ * Note, that the new {@code slice} is expected to not be fully contained in + * an existing slice! + *

+ * Pre/post-conditions: + *

    + *
  1. Slices are ordered by offset.
  2. + *
  3. If two slices overlap, no slice is fully contained within the other. + * (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
  4. + *
+ *

+ * The new {@code slice} will be inserted into the list at the correct position + * (such that {@code slices} remains ordered by slice offset), and all existing + * slices that are fully contained in the new {@code slice} will be removed. + * + * @param slices + * ordered list of slices + * @param slice + * slice to be inserted + */ + static void addSlice(final List slices, final T slice) { + + final int i = Collections.binarySearch(slices, slice, Comparator.comparingLong(Range::offset)); + final int from = i < 0 ? -i - 1 : i; + + int to = from; + while (to < slices.size() && slices.get(to).end() <= slice.end()) { + ++to; + } + + if (from == to) { + // empty range: just insert + slices.add(from, slice); + } else { + // overwrite the first element in range, remove the rest + slices.set(from, slice); + slices.subList(from + 1, to).clear(); + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java index b09df6782..30441c3a9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DatasetAccess.java @@ -38,7 +38,7 @@ import org.janelia.saalfeldlab.n5.shard.Nesting.NestedGrid; /** - * Wrap an instantiated DataBlock/shard codec hierarchy to implement (single and + * Wrap an instantiated DataBlock/block codec hierarchy to implement (single and * batch) DataBlock read/write methods. * * @param @@ -47,32 +47,32 @@ public interface DatasetAccess { /** - * Read the {@code DataBlock} at the given {@code gridPosition}. + * Read the chunk at the given {@code gridPosition}. *

- * If the requested block doesn't exist, then this method will return {@code + * If the requested chunk doesn't exist, then this method will return {@code * null}. {@code N5IOException} will be thrown if something goes wrong * reading or decoding data for an existing key. * * @param pva * dataset storage * @param gridPosition - * grid position of the DataBlock to read + * grid position of the chunks to read * * @return the DataBlock or {@code null} * * @throws N5IOException * if any error occurs while reading or decoding the block */ - DataBlock readBlock(PositionValueAccess pva, long[] gridPosition) throws N5IOException; + DataBlock readChunk(PositionValueAccess pva, long[] gridPosition) throws N5IOException; /** - * Read the {@code DataBlock}s at the given {@code gridPositions}. + * Read the chunks at the given {@code gridPositions}. *

* The returned {@code List>} is in the same order as the - * requested {@code gridPositions}. That is, the {@code DataBlock} at indwex + * requested {@code gridPositions}. That is, the {@code DataBlock} at index * {@code i} has grid coordinates {@code gridPositions.get(i)}. *

- * If a requested block doesn't exist, then the corresponding element in the + * If a requested chunk doesn't exist, then the corresponding element in the * result list will be {@code null}. ({@code N5IOException} will only be * thrown if something goes wrong reading or decoding data for an existing * key.) @@ -80,33 +80,50 @@ public interface DatasetAccess { * @param pva * dataset storage * @param gridPositions - * list of grid position of the DataBlocks to read + * list of grid positions of the chunks to read * @return list of DataBlocks * * @throws N5IOException * if any error occurs while reading or decoding blocks */ - List> readBlocks(PositionValueAccess pva, List gridPositions) throws N5IOException; + List> readChunks(PositionValueAccess pva, List gridPositions) throws N5IOException; - void writeBlock(PositionValueAccess pva, DataBlock dataBlock) throws N5IOException; - - void writeBlocks(PositionValueAccess pva, List> blocks) throws N5IOException; + /** + * Writes a chunk to the {@link DataBlock#getGridPosition() grid position} + * specified by {@code chunk}. + */ + void writeChunk(PositionValueAccess pva, DataBlock chunk) throws N5IOException; - boolean deleteBlock(PositionValueAccess pva, long[] gridPosition) throws N5IOException; + /** + * Writes multiple chunks to the {@link DataBlock#getGridPosition() grid + * positions} specified by the respective {@code chunks}. + */ + void writeChunks(PositionValueAccess pva, List> chunks) throws N5IOException; - boolean deleteBlocks(PositionValueAccess pva, List positions) throws N5IOException; + /** + * Deletes the chunk at {@code gridPosition}. + * + * @return true if it existed and was deleted. + */ + boolean deleteChunk(PositionValueAccess pva, long[] gridPosition) throws N5IOException; /** + * Deletes the chunks at the given {@code positions}. * + * @return true if any existed and were deleted. + */ + boolean deleteChunks(PositionValueAccess pva, List gridPositions) throws N5IOException; + + /** * @param pva * @param min * min pixel coordinate of region to write * @param size * size in pixels of region to write - * @param blocks - * is asked to create blocks within the given region + * @param chunkSupplier + * is asked to create chunks within the given region * @param writeFully - * if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * if false, merge existing data in blocks/chunks that overlap the region boundary. if true, override everything. * * @throws N5IOException */ @@ -114,7 +131,7 @@ void writeRegion( PositionValueAccess pva, long[] min, long[] size, - DataBlockSupplier blocks, + DataBlockSupplier chunkSupplier, boolean writeFully ) throws N5IOException; @@ -125,12 +142,12 @@ void writeRegion( * min pixel coordinate of region to write * @param size * size in pixels of region to write - * @param blocks - * is asked to create blocks within the given region. must be thread-safe. + * @param chunkSupplier + * is asked to create chunks within the given region. must be thread-safe. * @param writeFully - * if false, merge existing data in shards/blocks that overlap the region boundary. if true, override everything. + * if false, merge existing data in blocks/chunks that overlap the region boundary. if true, override everything. * @param exec - * used to parallelize over blocks and shards + * used to parallelize over chunks and blocks * * @throws N5Exception * @throws InterruptedException @@ -140,15 +157,15 @@ void writeRegion( PositionValueAccess pva, long[] min, long[] size, - DataBlockSupplier blocks, + DataBlockSupplier chunkSupplier, boolean writeFully, ExecutorService exec ) throws N5Exception, InterruptedException, ExecutionException; /** - * Read a full shard at {@code shardGridPosition} at the given nesting - * {@code level}. The shard data is rearranged and assembled into a (large) - * {@code DataBlock}. + * Read a block at {@code shardGridPosition} at the given nesting {@code + * level}. The data is read as chunks and then rearranged and assembled into + * a (large) {@code DataBlock}. *

* The {@code getGridPosition()} of the returned {@code DataBlock} is the * grid position with respect to {@code level}. For example, if {@code @@ -163,27 +180,27 @@ void writeRegion( * @return * @throws N5IOException */ - DataBlock readShard(PositionValueAccess pva, long[] shardGridPosition, int level) throws N5IOException; + DataBlock readBlock(PositionValueAccess pva, long[] shardGridPosition, int level) throws N5IOException; /** - * Write a full shard the given nesting {@code level}. The shard data is + * Write a full block at the given nesting {@code level}. The block data is * given as a (large) {@code DataBlock} that will be sliced, rearranged, and - * written as (level-0) DataBlocks. + * written as chunks (level-0 DataBlocks). *

* {@code dataBlock.getGridPosition()} is the grid position with respect to * {@code level}. For example, if {@code level==1}, then this refers to the - * position on the shard grid. + * position on the level-1 shard grid. * * @param pva * dataset storage * @param dataBlock - * shard/block to write + * block to write * @param level - * grid level of the shard/block to write. + * grid level of the block to write. * * @throws N5IOException */ - void writeShard(PositionValueAccess pva, DataBlock dataBlock, int level) throws N5IOException; + void writeBlock(PositionValueAccess pva, DataBlock dataBlock, int level) throws N5IOException; NestedGrid getGrid(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index fc1db3609..66e0dd548 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -71,20 +71,17 @@ public NestedGrid getGrid() { return grid; } - // - // -- readBlock ----------------------------------------------------------- - @Override - public DataBlock readBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { + public DataBlock readChunk(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { final NestedPosition position = grid.nestedPosition(gridPosition); try (final VolatileReadData readData = pva.get(position.key())) { - return readBlockRecursive(readData, position, grid.numLevels() - 1); + return readChunkRecursive(readData, position, grid.numLevels() - 1); } catch (N5NoSuchKeyException ignored) { return null; } } - private DataBlock readBlockRecursive( + private DataBlock readChunkRecursive( final ReadData readData, final NestedPosition position, final int level) { @@ -98,31 +95,28 @@ private DataBlock readBlockRecursive( @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[level]; final RawShard shard = codec.decode(readData, position.absolute(level)).getData(); - return readBlockRecursive(shard.getElementData(position.relative(level - 1)), position, level - 1); + return readChunkRecursive(shard.getElementData(position.relative(level - 1)), position, level - 1); } } - // - // -- readBlocks ---------------------------------------------------------- - @Override - public List> readBlocks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { + public List> readChunks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { - // for non-sharded datasets, just read the blocks individually + // for non-sharded datasets, just read the chunks individually if (grid.numLevels() == 1) { - return gridPositions.stream().map(pos -> readBlock(pva, pos)).collect(Collectors.toList()); + return gridPositions.stream().map(pos -> readChunk(pva, pos)).collect(Collectors.toList()); } - // Create a list of DataBlockRequests and sort it such that requests + // Create a list of ChunkRequests and sort it such that requests // from the same (nested) shard are grouped contiguously. - final DataBlockRequests requests = createReadRequests(gridPositions); - final List> duplicates = requests.removeDuplicates(); + final ChunkRequests requests = createReadRequests(gridPositions); + final List> duplicates = requests.removeDuplicates(); - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { + final List> split = requests.split(); + for (final ChunkRequests subRequests : split) { final long[] key = subRequests.relativeGridPosition(); try (final VolatileReadData readData = pva.get(key)) { - readBlocksRecursive(readData, subRequests); + readChunksRecursive(readData, subRequests); } catch (N5NoSuchKeyException ignored) { // the key didn't exist (as we found out when lazy-reading the index). // we don't have to do anything: all subRequest blocks remain null. @@ -130,18 +124,18 @@ public List> readBlocks(final PositionValueAccess pva, final List requests + final ChunkRequests requests ) { assert !requests.requests.isEmpty(); assert requests.level > 0; @@ -156,42 +150,48 @@ private void readBlocksRecursive( final RawShard shard = codec.decode(readData, requests.gridPosition()).getData(); if (level == 1 ) { - //Base case; read the blocks + //Base case; read the chunks // TODO: collect all the elementPos that we will need and prefetch // Probably best to add a prefetch method to RawShard? - for (final DataBlockRequest request : requests) { + // Here's an attempt at that. + // Don't love that we have to build a list of positions + // Consider making DataBlockRequest package private and passing them directly + final ArrayList positions = new ArrayList<>(); + for (final ChunkRequest request : requests) { + positions.add(request.position.relative(0)); + } + shard.prefetch(positions); + + for (final ChunkRequest request : requests) { final long[] elementPos = request.position.relative(0); final ReadData elementData = shard.getElementData(elementPos); - request.block = readBlockRecursive(elementData, request.position, 0); + request.chunk = readChunkRecursive(elementData, request.position, 0); } } else { // level > 1 - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { + final List> split = requests.split(); + for (final ChunkRequests subRequests : split) { final long[] subShardPosition = subRequests.relativeGridPosition(); final ReadData elementData = shard.getElementData(subShardPosition); - readBlocksRecursive(elementData, subRequests); + readChunksRecursive(elementData, subRequests); } } } - // - // -- writeBlock ---------------------------------------------------------- - @Override - public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlock) throws N5IOException { + public void writeChunk(final PositionValueAccess pva, final DataBlock chunk) throws N5IOException { if (grid.numLevels() == 1) { @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[0]; - pva.set(dataBlock.getGridPosition(), codec.encode(dataBlock)); + pva.set(chunk.getGridPosition(), codec.encode(chunk)); } else { - final NestedPosition position = grid.nestedPosition(dataBlock.getGridPosition()); + final NestedPosition position = grid.nestedPosition(chunk.getGridPosition()); final long[] key = position.key(); final ReadData modifiedData; try (final VolatileReadData existingData = pva.get(key)) { - modifiedData = writeBlockRecursive(existingData, dataBlock, position, grid.numLevels() - 1); + modifiedData = writeChunkRecursive(existingData, chunk, position, grid.numLevels() - 1); // Here, we are about to write the shard data, but with the new block modified. // Need to make sure that the read operations happen now before pva.set acquires a write lock modifiedData.materialize(); @@ -200,15 +200,15 @@ public void writeBlock(final PositionValueAccess pva, final DataBlock dataBlo } } - private ReadData writeBlockRecursive( + private ReadData writeChunkRecursive( final ReadData existingReadData, - final DataBlock dataBlock, + final DataBlock chunk, final NestedPosition position, final int level) { if (level == 0) { @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[0]; - return codec.encode(dataBlock); + return codec.encode(chunk); } else { @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[level]; @@ -218,34 +218,31 @@ private ReadData writeBlockRecursive( final ReadData existingElementData = (level == 1) ? null // if level == 1, we don't need to extract the nested (DataBlock) ReadData because it will be overridden anyway : shard.getElementData(elementPos); - final ReadData modifiedElementData = writeBlockRecursive(existingElementData, dataBlock, position, level - 1); + final ReadData modifiedElementData = writeChunkRecursive(existingElementData, chunk, position, level - 1); shard.setElementData(modifiedElementData, elementPos); return codec.encode(new RawShardDataBlock(gridPos, shard)); } } - // - // -- writeBlocks --------------------------------------------------------- - @Override - public void writeBlocks(final PositionValueAccess pva, final List> dataBlocks) throws N5IOException { + public void writeChunks(final PositionValueAccess pva, final List> chunks) throws N5IOException { if (grid.numLevels() == 1) { @SuppressWarnings("unchecked") final BlockCodec codec = (BlockCodec) codecs[0]; - dataBlocks.forEach(dataBlock -> pva.set(dataBlock.getGridPosition(), codec.encode(dataBlock))); + chunks.forEach(chunk -> pva.set(chunk.getGridPosition(), codec.encode(chunk))); } else { - // Create a list of DataBlockRequests, sorted such that requests from + // Create a list of ChunkRequests, sorted such that requests from // the same (nested) shard are grouped contiguously. - final DataBlockRequests requests = createWriteRequests(dataBlocks); + final ChunkRequests requests = createWriteRequests(chunks); requests.removeDuplicates(); - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { + final List> split = requests.split(); + for (final ChunkRequests subRequests : split) { final boolean writeFully = subRequests.coversShard(); final long[] shardKey = subRequests.relativeGridPosition(); final ReadData modifiedData; try (final VolatileReadData existingData = writeFully ? null : pva.get(shardKey)) { - modifiedData = writeBlocksRecursive(existingData, subRequests); + modifiedData = writeChunksRecursive(existingData, subRequests); // Here, we are about to write the shard data, but with the new blocks modified. // Need to make sure that the read operations happen now before pva.set acquires a write lock modifiedData.materialize(); @@ -259,11 +256,11 @@ public void writeBlocks(final PositionValueAccess pva, final List> * Bulk Write operation on a shard. * * @param existingReadData encoded existing shard data (to decode and partially override) - * @param requests for blocks within the shard to be written + * @param requests for chunks within the shard to be written */ - private ReadData writeBlocksRecursive( + private ReadData writeChunksRecursive( final ReadData existingReadData, // may be null - final DataBlockRequests requests + final ChunkRequests requests ) { assert !requests.requests.isEmpty(); assert requests.level > 0; @@ -278,18 +275,18 @@ private ReadData writeBlocksRecursive( if ( level == 1 ) { // Base case, write the blocks - for (final DataBlockRequest request : requests) { - final ReadData elementData = writeBlockRecursive(null, request.block, request.position, 0); + for (final ChunkRequest request : requests) { + final ReadData elementData = writeChunkRecursive(null, request.chunk, request.position, 0); final long[] elementPos = request.position.relative(0); shard.setElementData(elementData, elementPos); } } else { // level > 1 - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { + final List> split = requests.split(); + for (final ChunkRequests subRequests : split) { final boolean nestedWriteFully = writeFully || subRequests.coversShard(); final long[] elementPos = subRequests.relativeGridPosition(); final ReadData existingElementData = nestedWriteFully ? null : shard.getElementData(elementPos); - final ReadData modifiedElementData = writeBlocksRecursive(existingElementData, subRequests); + final ReadData modifiedElementData = writeChunksRecursive(existingElementData, subRequests); shard.setElementData(modifiedElementData, elementPos); } } @@ -297,15 +294,12 @@ private ReadData writeBlocksRecursive( return codec.encode(new RawShardDataBlock(gridPos, shard)); } - // - // -- writeRegion --------------------------------------------------------- - @Override public void writeRegion( final PositionValueAccess pva, final long[] min, final long[] size, - final DataBlockSupplier blocks, + final DataBlockSupplier chunkSupplier, final boolean writeFully ) throws N5IOException { @@ -316,8 +310,8 @@ public void writeRegion( final boolean nestedWriteFully = writeFully || region.fullyContains(pos); final ReadData modifiedData; try (final VolatileReadData existingData = nestedWriteFully ? null : pva.get(key)) { - modifiedData = writeRegionRecursive(existingData, region, blocks, pos); - // Here, we are about to write the shard data, but with the new block modified. + modifiedData = writeRegionRecursive(existingData, region, chunkSupplier, pos); + // Here, we are about to write the shard data, but with the new shard modified. // Need to make sure that the read operations happen now before pva.set acquires a write lock if (existingData != null && modifiedData != null) { modifiedData.materialize(); @@ -332,7 +326,7 @@ public void writeRegion( final PositionValueAccess pva, final long[] min, final long[] size, - final DataBlockSupplier blocks, + final DataBlockSupplier chunkSupplier, final boolean writeFully, final ExecutorService exec) throws N5Exception, InterruptedException, ExecutionException { @@ -344,7 +338,7 @@ public void writeRegion( final boolean nestedWriteFully = writeFully || region.fullyContains(pos); final ReadData modifiedData; try (final VolatileReadData existingData = nestedWriteFully ? null : pva.get(key)) { - modifiedData = writeRegionRecursive(existingData, region, blocks, pos); + modifiedData = writeRegionRecursive(existingData, region, chunkSupplier, pos); // Here, we are about to write the shard data, but with the new block modified. // Need to make sure that the read operations happen now before pva.set acquires a write lock if (existingData != null && modifiedData != null) { @@ -359,7 +353,7 @@ public void writeRegion( private ReadData writeRegionRecursive( final ReadData existingReadData, // may be null final Region region, - final DataBlockSupplier blocks, + final DataBlockSupplier chunkSupplier, final NestedPosition position ) { final boolean writeFully = existingReadData == null; @@ -375,21 +369,21 @@ private ReadData writeRegionRecursive( // existing DataBlock and pass it to the BlockSupplier for modification. // (This might fail with N5NoSuchKeyException if existingReadData // lazily points to non-existent data.) - DataBlock existingDataBlock = null; + DataBlock existingChunk = null; if (existingReadData != null) { try { - existingDataBlock = codec.decode(existingReadData, gridPosition); + existingChunk = codec.decode(existingReadData, gridPosition); } catch (N5NoSuchKeyException ignored) { } } - final DataBlock dataBlock = blocks.get(gridPosition, existingDataBlock); + final DataBlock chunk = chunkSupplier.get(gridPosition, existingChunk); - // null blocks may be provided when they contain only the fill value - // and only non-empty blocks should be written, for example - if (dataBlock == null) + // null chunks may be provided when they contain only the fill value + // and only non-empty chunks should be written, for example + if (chunk == null) return null; - return codec.encode(dataBlock); + return codec.encode(chunk); } else { @SuppressWarnings("unchecked") @@ -400,7 +394,7 @@ private ReadData writeRegionRecursive( final boolean nestedWriteFully = writeFully || region.fullyContains(pos); final long[] elementPos = pos.relative(); final ReadData existingElementData = nestedWriteFully ? null : shard.getElementData(elementPos); - final ReadData modifiedElementData = writeRegionRecursive(existingElementData, region, blocks, pos); + final ReadData modifiedElementData = writeRegionRecursive(existingElementData, region, chunkSupplier, pos); shard.setElementData(modifiedElementData, elementPos); } @@ -413,10 +407,10 @@ private ReadData writeRegionRecursive( } // - // -- deleteBlock --------------------------------------------------------- + // -- deleteChunk --------------------------------------------------------- @Override - public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { + public boolean deleteChunk(final PositionValueAccess pva, final long[] gridPosition) throws N5IOException { if (grid.numLevels() == 1) { // for non-sharded dataset, don't bother getting the value, just remove the key. return pva.remove(gridPosition); @@ -425,18 +419,18 @@ public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosit final long[] key = position.key(); final ReadData modifiedData; try (final VolatileReadData existingData = pva.get(key)) { - modifiedData = deleteBlockRecursive(existingData, position, grid.numLevels() - 1); + modifiedData = deleteChunkRecursive(existingData, position, grid.numLevels() - 1); if (modifiedData == existingData) { // nothing changed, the blocks we wanted to delete didn't exist anyway return false; } else if (modifiedData != null) { - // Here, we are about to write the shard data, but without the block to be deleted. + // Here, we are about to write the shard data, but without the chunk to be deleted. // Need to make sure that the read operations happen now before pva.set acquires a write lock modifiedData.materialize(); } } catch (final N5NoSuchKeyException e) { // the key didn't exist (as we found out when lazy-reading the index) - // so nothing changed, the blocks we wanted to delete didn't exist anyway + // so nothing changed, the chunks we wanted to delete didn't exist anyway return false; } if (modifiedData == null) { @@ -448,7 +442,7 @@ public boolean deleteBlock(final PositionValueAccess pva, final long[] gridPosit } } - private ReadData deleteBlockRecursive( + private ReadData deleteChunkRecursive( final ReadData existingReadData, final NestedPosition position, final int level) throws N5NoSuchKeyException { @@ -462,11 +456,11 @@ private ReadData deleteBlockRecursive( final long[] elementPos = position.relative(level - 1); final ReadData existingElementData = shard.getElementData(elementPos); if (existingElementData == null) { - // The DataBlock (or the whole nested shard containing it) does not exist. + // The chunk (or the whole nested shard containing it) does not exist. // This shard remains unchanged. return existingReadData; } else { - final ReadData modifiedElementData = deleteBlockRecursive(existingElementData, position, level - 1); + final ReadData modifiedElementData = deleteChunkRecursive(existingElementData, position, level - 1); if (modifiedElementData == existingElementData) { // The nested shard was not modified. // This shard remains unchanged. @@ -474,7 +468,7 @@ private ReadData deleteBlockRecursive( } shard.setElementData(modifiedElementData, elementPos); if (modifiedElementData == null) { - // The DataBlock or nested shard was removed. + // The chunk or nested shard was removed. // Check whether this shard becomes empty. if (shard.isEmpty()) { // This shard is empty and should be removed. @@ -487,12 +481,12 @@ private ReadData deleteBlockRecursive( } // - // -- deleteBlocks -------------------------------------------------------- + // -- deleteChunks -------------------------------------------------------- @Override - public boolean deleteBlocks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { + public boolean deleteChunks(final PositionValueAccess pva, final List gridPositions) throws N5IOException { - // for non-sharded datasets, just delete the blocks individually + // for non-sharded datasets, just delete the chunks individually if (grid.numLevels() == 1) { boolean deleted = false; for (long[] pos : gridPositions) { @@ -501,31 +495,31 @@ public boolean deleteBlocks(final PositionValueAccess pva, final List gr return deleted; } else { - // Create a list of DataBlockRequests and sort it such that requests + // Create a list of ChunkRequests and sort it such that requests // from the same (nested) shard are grouped contiguously. // Despite the name, createReadRequests() works for delete requests as well ... - final DataBlockRequests requests = createReadRequests(gridPositions); + final ChunkRequests requests = createReadRequests(gridPositions); requests.removeDuplicates(); boolean deleted = false; - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { + final List> split = requests.split(); + for (final ChunkRequests subRequests : split) { final boolean writeFully = subRequests.coversShard(); final long[] key = subRequests.relativeGridPosition(); final ReadData modifiedData; try (final VolatileReadData existingData = writeFully ? null : pva.get(key)) { - modifiedData = deleteBlocksRecursive(existingData, subRequests);; + modifiedData = deleteChunksRecursive(existingData, subRequests);; if (modifiedData == existingData) { - // nothing changed, the blocks we wanted to delete didn't exist anyway + // nothing changed, the chunks we wanted to delete didn't exist anyway continue; } else if (existingData != null && modifiedData != null) { - // Here, we are about to write the shard data, but without the block to be deleted. + // Here, we are about to write the shard data, but without the chunk to be deleted. // Need to make sure that the read operations happen now before pva.set acquires a write lock modifiedData.materialize(); } } catch (final N5NoSuchKeyException e) { // the key didn't exist (as we found out when lazy-reading the index) - // so nothing changed, the blocks we wanted to delete didn't exist anyway + // so nothing changed, the chunks we wanted to delete didn't exist anyway continue; } if (modifiedData == null) { @@ -543,11 +537,11 @@ public boolean deleteBlocks(final PositionValueAccess pva, final List gr * Bulk Delete operation on a shard. * * @param existingReadData encoded existing shard data (to decode and partially override) - * @param requests for blocks within the shard to be deleted + * @param requests for chunks within the shard to be deleted */ - private ReadData deleteBlocksRecursive( + private ReadData deleteChunksRecursive( final ReadData existingReadData, // may be null - final DataBlockRequests requests + final ChunkRequests requests ) { assert !requests.requests.isEmpty(); assert requests.level > 0; @@ -565,8 +559,8 @@ private ReadData deleteBlocksRecursive( boolean modified = false; boolean shardElementSetToNull = false; if ( level == 1 ) { - // Base case, delete the blocks - for (final DataBlockRequest request : requests) { + // Base case, delete the chunks + for (final ChunkRequest request : requests) { final long[] elementPos = request.position.relative(0); if (shard.getElementData(elementPos) != null) { shard.setElementData(null, elementPos); @@ -575,12 +569,12 @@ private ReadData deleteBlocksRecursive( } } } else { // level > 1 - final List> split = requests.split(); - for (final DataBlockRequests subRequests : split) { + final List> split = requests.split(); + for (final ChunkRequests subRequests : split) { final boolean writeFully = subRequests.coversShard(); final long[] elementPos = subRequests.relativeGridPosition(); final ReadData existingElementData = writeFully ? null : shard.getElementData(elementPos); - final ReadData modifiedElementData = deleteBlocksRecursive(existingElementData, subRequests); + final ReadData modifiedElementData = deleteChunksRecursive(existingElementData, subRequests); if (modifiedElementData != existingElementData) { shard.setElementData(modifiedElementData, elementPos); modified = true; @@ -590,13 +584,13 @@ private ReadData deleteBlocksRecursive( } if (!modified) { - // No nested shard or DataBlock was modified. + // No nested shard or chunk was modified. // This shard remains unchanged. return existingReadData; } if (shardElementSetToNull) { - // At least one DataBlock or nested shard was removed. + // At least one chunk or nested shard was removed. // Check whether this shard becomes empty. if (shard.index().allElementsNull()) { // This shard is empty and should be removed. @@ -640,7 +634,7 @@ private ReadData deleteBlocksRecursive( // do not want to write DataBlocks that are completely outside the // dataset (even if the "big DataBlock" covers this area.) // - // This works for writing. For reading we'll have to decide what to return, + // This works for writing. For reading, we'll have to decide what to return, // though... Potentially, there is no valid block at the border, so we // cannot determine where to put the border just from the data. We need to // rely on external input or heuristics. @@ -652,14 +646,14 @@ private ReadData deleteBlocksRecursive( // easily revisit and change this heuristic. @Override - public DataBlock readShard( + public DataBlock readBlock( final PositionValueAccess pva, final long[] shardGridPosition, final int level ) throws N5IOException { if (level == 0) { - return readBlock(pva, shardGridPosition); + return readChunk(pva, shardGridPosition); } final long[] shardPixelPos = grid.pixelPosition(shardGridPosition, level); @@ -671,10 +665,10 @@ public DataBlock readShard( for (int d = 0; d < n; ++d) { shardSizeInPixels[d] = Math.min(defaultShardSize[d], (int) (datasetSize[d] - shardPixelPos[d])); } - return readShardInternal(pva, shardGridPosition, shardSizeInPixels, level); + return readBlockInternal(pva, shardGridPosition, shardSizeInPixels, level); } - private DataBlock readShardInternal( + private DataBlock readBlockInternal( final PositionValueAccess pva, final long[] shardGridPosition, final int[] shardSizeInPixels, // expected size of this shard in pixels @@ -689,60 +683,60 @@ private DataBlock readShardInternal( } } - // level-0 block-grid position of the min block in the shard + // level-0 block-grid position of the min chunk in the shard final long[] gridMin = grid.absolutePosition(shardGridPosition, level, 0); - // level-0 block-grid position of the max block in the shard that we need to read. + // level-0 block-grid position of the max chunk in the shard that we need to read. // (the shard might go beyond the dataset border, and we don't need to read anything there) final long[] gridMax = new long[n]; - final long[] datasetSizeInBlocks = grid.getDatasetSizeInBlocks(); - final int[] blockSize = grid.getBlockSize(0); + final long[] datasetSizeInChunks = grid.getDatasetSizeInChunks(); + final int[] chunkSize = grid.getBlockSize(0); for (int d = 0; d < n; ++d) { - final int shardSizeInBlocks = (shardSizeInPixels[d] + blockSize[d] - 1) / blockSize[d]; - final int gridSize = Math.min(shardSizeInBlocks, (int) (datasetSizeInBlocks[d] - gridMin[d])); + final int shardSizeInChunks = (shardSizeInPixels[d] + chunkSize[d] - 1) / chunkSize[d]; + final int gridSize = Math.min(shardSizeInChunks, (int) (datasetSizeInChunks[d] - gridMin[d])); gridMax[d] = gridMin[d] + gridSize - 1; } - // read all blocks in (gridMin, gridMax) and filter out missing blocks - final List blockPositions = Region.gridPositions(gridMin, gridMax); - final List> blocks = readBlocks(pva, blockPositions) + // read all chunks in (gridMin, gridMax) and filter out missing chunks + final List chunkPositions = Region.gridPositions(gridMin, gridMax); + final List> chunks = readChunks(pva, chunkPositions) .stream().filter(Objects::nonNull).collect(Collectors.toList()); - if (blocks.isEmpty()) { + if (chunks.isEmpty()) { return null; } - // allocate shard and copy data from blocks - final DataBlock shard = DataBlockFactory.of(blocks.get(0).getData()).createDataBlock(shardSizeInPixels, shardGridPosition); + // allocate shard and copy data from chunks + final DataBlock shard = DataBlockFactory.of(chunks.get(0).getData()).createDataBlock(shardSizeInPixels, shardGridPosition); final long[] shardPixelPos = grid.pixelPosition(shardGridPosition, level); - final long[] blockPixelPos = new long[n]; + final long[] chunkPixelPos = new long[n]; final int[] srcPos = new int[n]; final int[] destPos = new int[n]; final int[] size = new int[n]; - for (final DataBlock block : blocks) { - // copy block data that overlaps the shard - grid.pixelPosition(block.getGridPosition(), 0, blockPixelPos); - final int[] bsize = block.getSize(); + for (final DataBlock chunk : chunks) { + // copy chunk data that overlaps the shard + grid.pixelPosition(chunk.getGridPosition(), 0, chunkPixelPos); + final int[] bsize = chunk.getSize(); for (int d = 0; d < n; d++) { - destPos[d] = (int) (blockPixelPos[d] - shardPixelPos[d]); + destPos[d] = (int) (chunkPixelPos[d] - shardPixelPos[d]); size[d] = Math.min(bsize[d], shardSizeInPixels[d] - destPos[d]); } - SubArrayCopy.copy(block.getData(), bsize, srcPos, shard.getData(), shardSizeInPixels, destPos, size); + SubArrayCopy.copy(chunk.getData(), bsize, srcPos, shard.getData(), shardSizeInPixels, destPos, size); } return shard; } // - // -- writeShard ---------------------------------------------------------- + // -- writeBlock ---------------------------------------------------------- @Override - public void writeShard( + public void writeBlock( final PositionValueAccess pva, final DataBlock dataBlock, final int level ) throws N5IOException { if (level == 0) { - writeBlock(pva, dataBlock); + writeChunk(pva, dataBlock); return; } @@ -750,49 +744,49 @@ public void writeShard( final DataBlockFactory blockFactory = DataBlockFactory.of(shardData); final int n = grid.numDimensions(); - final int[] blockSize = grid.getBlockSize(0); // size of a standard (non-truncated) DataBlock - final long[] datasetBlockSize = grid.getDatasetSizeInBlocks(); + final int[] chunkSize = grid.getBlockSize(0); // size of a standard (non-truncated) chunk + final long[] datasetChunkSize = grid.getDatasetSizeInChunks(); final long[] shardPixelMin = grid.pixelPosition(dataBlock.getGridPosition(), level); final int[] shardPixelSize = dataBlock.getSize(); - // the max block + 1 in the shard, if it isn't truncated by the dataset border - final long[] shardBlockTo = new long[n]; - Arrays.setAll(shardBlockTo, d -> (shardPixelMin[d] + shardPixelSize[d] + blockSize[d] - 1) / blockSize[d]); + // the max chunk + 1 in the shard, if it isn't truncated by the dataset border + final long[] shardChunkTo = new long[n]; + Arrays.setAll(shardChunkTo, d -> (shardPixelMin[d] + shardPixelSize[d] + chunkSize[d] - 1) / chunkSize[d]); - // level 0 DataBlock positions of all DataBlocks we want to extract + // level 0 grid positions of all chunks we want to extract final long[] gridMin = grid.absolutePosition(dataBlock.getGridPosition(), level, 0); final long[] gridMax = new long[n]; - Arrays.setAll(gridMax, d -> Math.min(shardBlockTo[d], datasetBlockSize[d]) - 1); - final List blockPositions = Region.gridPositions(gridMin, gridMax); + Arrays.setAll(gridMax, d -> Math.min(shardChunkTo[d], datasetChunkSize[d]) - 1); + final List chunkPositions = Region.gridPositions(gridMin, gridMax); // Max pixel coordinates + 1, of the region we want to copy. This should // always be shardPixelMin + shardPixelSize, except at the dataset - // border, where we truncate to the smallest multiple of blockSize still + // border, where we truncate to the smallest multiple of chunkSize still // overlapping the dataset. final long[] regionBound = new long[n]; - Arrays.setAll(regionBound, d -> Math.min(shardPixelMin[d] + shardPixelSize[d], datasetBlockSize[d] * blockSize[d])); + Arrays.setAll(regionBound, d -> Math.min(shardPixelMin[d] + shardPixelSize[d], datasetChunkSize[d] * chunkSize[d])); - final List> blocks = new ArrayList<>(blockPositions.size()); + final List> chunks = new ArrayList<>(chunkPositions.size()); final int[] srcPos = new int[n]; final int[] destPos = new int[n]; final int[] destSize = new int[n]; - for ( long[] blockPos : blockPositions) { - final long[] pixelMin = grid.pixelPosition(blockPos, 0); + for ( long[] chunkPos : chunkPositions) { + final long[] pixelMin = grid.pixelPosition(chunkPos, 0); for (int d = 0; d < n; d++) { srcPos[d] = (int) (pixelMin[d] - shardPixelMin[d]); - destSize[d] = Math.min(blockSize[d], (int) (regionBound[d] - pixelMin[d])); + destSize[d] = Math.min(chunkSize[d], (int) (regionBound[d] - pixelMin[d])); } - // This extracting DataBlocks will not work if num_array_elements != num_block_elements. + // This extracting chunks will not work if num_array_elements != num_block_elements. // But we'll deal with that later if it becomes a problem... - final DataBlock block = blockFactory.createDataBlock(destSize, blockPos); - SubArrayCopy.copy(shardData, shardPixelSize, srcPos, block.getData(), destSize, destPos, destSize); - blocks.add(block); + final DataBlock chunk = blockFactory.createDataBlock(destSize, chunkPos); + SubArrayCopy.copy(shardData, shardPixelSize, srcPos, chunk.getData(), destSize, destPos, destSize); + chunks.add(chunk); } - writeBlocks(pva, blocks); + writeChunks(pva, chunks); } // @@ -824,73 +818,72 @@ private RawShard getRawShard( } /** - * A request to read or write a (level-0) DataBlock at a given {@link #position}. + * A request to read or write a chunk (level-0 DataBlock) at a given {@link #position}. *

- * Write requests are constructed with {@link #position} and {@link #block}. + * Write requests are constructed with {@link #position} and {@link #chunk}. *

* Read requests are constructed with only a {@link #position}, and - * initially {@link #block block=null}. When the DataBlock is read, it will - * be put into {@link #block}. + * initially {@link #chunk chunk=null}. When the DataBlock is read, it will + * be put into {@link #chunk}. *

- * {@code DataBlockRequest} are used for reading/writing a list of blocks - * with {@link #readBlocks} and {@link #writeBlocks}. The {@link #index} - * field is the position in the list of positions/blocks to read/write. For + * {@code ChunkRequest} are used for reading/writing a list of chunks + * with {@link #readChunks} and {@link #writeChunks}. The {@link #index} + * field is the position in the list of positions/chunks to read/write. For * processing, requests are re-ordered such that all requests from the same * (sub-)shard are grouped together. The {@link #index} field is used to - * re-establish the order of results (only important for {@link - * #readBlocks}). + * re-establish the order of results (only important for {@link #readChunks}). * * @param * type of the data contained in the DataBlock */ - private static final class DataBlockRequest { + private static final class ChunkRequest { final NestedPosition position; final int index; - DataBlock block; + DataBlock chunk; // read request - DataBlockRequest(final NestedPosition position, final int index) { + ChunkRequest(final NestedPosition position, final int index) { this.position = position; this.index = index; - this.block = null; + this.chunk = null; } // write request - DataBlockRequest(final NestedPosition position, final DataBlock block) { + ChunkRequest(final NestedPosition position, final DataBlock chunk) { this.position = position; this.index = -1; - this.block = block; + this.chunk = chunk; } @Override public String toString() { - return "DataBlockRequest{position=" + position + ", index=" + index + '}'; + return "ChunkRequest{position=" + position + ", index=" + index + '}'; } } /** - * A list of {@code DataBlockRequest}, ordered by {@code NestedPosition}. + * A list of {@code ChunkRequest}, ordered by {@code NestedPosition}. * All requests lie in the same shard at the given {@code level}. *

- * {@code DataBlockRequests} should be constructed using {@link + * {@code ChunkRequests} should be constructed using {@link * #createReadRequests} (for reading) or {@link #createWriteRequests} (for * writing). *

- * When recursing into nested shard levels, {@code DataBlockRequests} should - * be {@link #split} to partition into sub-{@code DataBlockRequests} that + * When recursing into nested shard levels, {@code ChunkRequests} should + * be {@link #split} to partition into sub-{@code ChunkRequests} that * each cover one shard. * * @param * type of the data contained in the DataBlocks */ - private static final class DataBlockRequests implements Iterable> { + private static final class ChunkRequests implements Iterable> { private final NestedGrid grid; - private final List> requests; + private final List> requests; private final int level; - private DataBlockRequests(final List> requests, final int level, final NestedGrid grid) { + private ChunkRequests(final List> requests, final int level, final NestedGrid grid) { this.requests = requests; this.level = level; this.grid = grid; @@ -907,12 +900,12 @@ private DataBlockRequests(final List> requests, final int le * If {@code n} duplicates occur in {@code requests}, the resulting list * will have {@code 2*n} elements. */ - public List> removeDuplicates() { - List> duplicates = new ArrayList<>(); - DataBlockRequest previous = null; - final ListIterator> iter = requests.listIterator(); + public List> removeDuplicates() { + List> duplicates = new ArrayList<>(); + ChunkRequest previous = null; + final ListIterator> iter = requests.listIterator(); while (iter.hasNext()) { - final DataBlockRequest current = iter.next(); + final ChunkRequest current = iter.next(); if (previous != null) { if (previous.position.equals(current.position)) { iter.remove(); @@ -927,15 +920,15 @@ public List> removeDuplicates() { } @Override - public Iterator> iterator() { + public Iterator> iterator() { return requests.iterator(); } /** - * All blocks contained in this {@code DataBlockRequests} are in the + * All chunks contained in this {@code ChunkRequests} are in the * same shard at this nesting level. *

- * Use {@link #split()} to partition into {@code DataBlockRequests} with + * Use {@link #split()} to partition into {@code ChunkRequests} with * nesting level {@link #level()}{@code -1}. * * @return nesting level @@ -945,7 +938,7 @@ public int level() { } /** - * Position on the shard grid at the level of this DataBlockRequests + * Position on the shard grid at the level of this ChunkRequests * (of the one shard containing all the requested blocks). */ public long[] gridPosition() { @@ -953,7 +946,7 @@ public long[] gridPosition() { } /** - * Relative grid position at the level of this DataBlockRequests, + * Relative grid position at the level of this ChunkRequests, * that is, relative offset within containing the (level+1) element. */ public long[] relativeGridPosition() { @@ -969,9 +962,9 @@ private NestedPosition position() { /** * Split into sub-requests, grouping by same position at nesting level {@link #level()}{@code -1}. */ - public List> split() { + public List> split() { final int subLevel = level - 1; - final List> subRequests = new ArrayList<>(); + final List> subRequests = new ArrayList<>(); for (int i = 0; i < requests.size(); ) { final long[] ilpos = requests.get(i).position.absolute(subLevel); int j = i + 1; @@ -981,14 +974,14 @@ public List> split() { break; } } - subRequests.add(new DataBlockRequests<>(requests.subList(i, j), subLevel, grid)); + subRequests.add(new ChunkRequests<>(requests.subList(i, j), subLevel, grid)); i = j; } return subRequests; } /** - * Returns {@code true} if this {@code DataBlockRequests} completely + * Returns {@code true} if this {@code ChunkRequests} completely * fills its containing shard at nesting level {@link #level()}. * (This can be used to avoid reading a shard that will be completely * overwritten). @@ -998,7 +991,7 @@ public List> split() { */ public boolean coversShard() { final long[] gridMin = grid.absolutePosition(position().absolute(level), level, 0); - final long[] datasetSize = grid.getDatasetSizeInBlocks(); // in units of DataBlocks + final long[] datasetSize = grid.getDatasetSizeInChunks(); // in units of DataBlocks final int[] defaultShardSize = grid.relativeToBaseBlockSize(level); // in units of DataBlocks int numElements = 1; for (int d = 0; d < defaultShardSize.length; d++) { @@ -1008,65 +1001,65 @@ public boolean coversShard() { } /** - * Extract {@link DataBlockRequest#block DataBlock}s from the requests, - * in the order of {@link DataBlockRequest#index indices}. + * Extract {@link ChunkRequest#chunk chunk}s from the requests, + * in the order of {@link ChunkRequest#index indices}. *

- * (This is used in {@link #readBlocks} to collect DataBlocks in the + * (This is used in {@link #readChunks} to collect chunks in the * requested order.) */ - public List> blocks(final List> duplicates) { + public List> chunks(final List> duplicates) { final int size = requests.size() + duplicates.size() / 2; final DataBlock[] blocks = new DataBlock[size]; - requests.forEach(r -> blocks[r.index] = r.block); + requests.forEach(r -> blocks[r.index] = r.chunk); for (int i = 0; i < duplicates.size(); i += 2) { - final DataBlockRequest a = duplicates.get(i * 2); - final DataBlockRequest b = duplicates.get(i * 2 + 1); - blocks[a.index] = b.block; + final ChunkRequest a = duplicates.get(i * 2); + final ChunkRequest b = duplicates.get(i * 2 + 1); + blocks[a.index] = b.chunk; } return Arrays.asList(blocks); } } /** - * Construct {@code DataBlockRequests} from a list of level-0 grid positions + * Construct {@code ChunkRequests} from a list of level-0 grid positions * for reading. *

- * The nesting level ot the returned {@code DataBlockRequests} is {@code + * The nesting level of the returned {@code ChunkRequests} is {@code * grid.numLevels()}, that is level of the highest-order shard + 1. This * implies that the requests are not guaranteed to be in the same shard (at - * any level. {@link DataBlockRequests#split() Splitting} the {@code - * DataBlockRequests} once will return a list of {@code DataBlockRequests} - * that each contain blocks from one highest-order shard. + * any level. {@link ChunkRequests#split() Splitting} the {@code + * ChunkRequests} once will return a list of {@code ChunkRequests} + * that each contain chunks from one highest-order shard. */ - private DataBlockRequests createReadRequests(final List gridPositions) { - final List> requests = new ArrayList<>(gridPositions.size()); + private ChunkRequests createReadRequests(final List gridPositions) { + final List> requests = new ArrayList<>(gridPositions.size()); for (int i = 0; i < gridPositions.size(); i++) { final NestedPosition pos = grid.nestedPosition(gridPositions.get(i)); - requests.add(new DataBlockRequest<>(pos, i)); + requests.add(new ChunkRequest<>(pos, i)); } requests.sort(Comparator.comparing(r -> r.position)); - return new DataBlockRequests<>(requests, grid.numLevels(), grid); + return new ChunkRequests<>(requests, grid.numLevels(), grid); } /** - * Construct {@code DataBlockRequests} from a list of level-0 DataBlocks for - * writing. + * Construct {@code ChunkRequests} from a list of chunks (level-0 + * DataBlocks) for writing. *

- * The nesting level ot the returned {@code DataBlockRequests} is {@code + * The nesting level ot the returned {@code ChunkRequests} is {@code * grid.numLevels()}, that is level of the highest-order shard + 1. This * implies that the requests are not guaranteed to be in the same shard (at - * any level. {@link DataBlockRequests#split() Splitting} the {@code - * DataBlockRequests} once will return a list of {@code DataBlockRequests} - * that each contain blocks from one highest-order shard. + * any level. {@link ChunkRequests#split() Splitting} the {@code + * ChunkRequests} once will return a list of {@code ChunkRequests} + * that each contain chunks from one highest-order shard. */ - private DataBlockRequests createWriteRequests(final List> dataBlocks) { - final List> requests = new ArrayList<>(dataBlocks.size()); + private ChunkRequests createWriteRequests(final List> dataBlocks) { + final List> requests = new ArrayList<>(dataBlocks.size()); for (final DataBlock dataBlock : dataBlocks) { final NestedPosition pos = grid.nestedPosition(dataBlock.getGridPosition()); - requests.add(new DataBlockRequest<>(pos, dataBlock)); + requests.add(new ChunkRequest<>(pos, dataBlock)); } requests.sort(Comparator.comparing(r -> r.position)); - return new DataBlockRequests<>(requests, grid.numLevels(), grid); + return new ChunkRequests<>(requests, grid.numLevels(), grid); } /** @@ -1074,7 +1067,7 @@ private DataBlockRequests createWriteRequests(final List> dataBl * array type and the number of elements in a block corresponds to the * {@link DataBlock#getSize()}. *

- * This is used by {@link #readShard} and {@link #writeShard} which + * This is used by {@link #readBlock} and {@link #writeBlock} which * internally need to allocate new DataBlocks to split or merge a shard. */ private interface DataBlockFactory { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java index ff3e6e164..feaa91cce 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Nesting.java @@ -37,14 +37,15 @@ * sharded N5 datasets. *

* This class provides classes for representing and navigating nested/sharded - * dataset layouts for the N5 API. In a sharded dataset, data blocks are grouped - * into higher-level containers called shards, which can themselves be nested - * within parent shards, creating a multi-level hierarchy. + * dataset layouts for the N5 API. In a sharded dataset, level-0 data blocks + * (called chunks) are grouped into higher-level containers called + * shards, which can themselves be nested within parent shards, + * creating a multi-level hierarchy. *

- * Nesting levels: + * Nesting levels: *

    - *
  • Level 0: The finest granularity - individual data blocks - * containing actual pixel data + *
  • Level 0: The finest granularity - individual chunks (level-0 data + * blocks) containing actual pixel data *
  • Level 1: First-level shards, which contain multiple level-0 data * blocks *
  • Level 2+: Higher-level shards (if they exist), which contain @@ -62,7 +63,7 @@ * {@code NestedGrid} at a particular nesting level, providing coordinate * transformations between levels. *
- * + * */ public class Nesting { @@ -85,8 +86,8 @@ protected NestedPosition(final NestedGrid grid, final long[] position, final int /** * Get the nesting level of this position. *

- * Positions with {@code level=0} refer to DataBlocks, positions with - * {@code level=1} refer to first-level shards (containing DataBlocks), + * Positions with {@code level=0} refer to chunks, positions with + * {@code level=1} refer to first-level shards (containing chunks), * and so on. * * @return nesting level @@ -220,11 +221,11 @@ public int hashCode() { /** * A nested grid of blocks used to coordinate the relationships of shards - * and the blocks / chunks they contain. + * and the blocks (chunks / sub-shards) they contain. *

* The nesting depth ({@link #numLevels()}) of the {@code NestedGrid} is 1 * for non-sharded datasets, 2 for simple sharded datasets (where shards - * contain datablocks), and ≥3 for nested sharded datasets. + * contain chunks), and ≥3 for nested sharded datasets. *

* Positions with {@code level=0} refer to the DataBlock grid, positions * with {@code level=1} refer to first-level Shard grid, and so on. @@ -255,9 +256,9 @@ public static class NestedGrid { private final long[] datasetSize; /** - * dimensions of the dataset in (level-0) blocks. + * dimensions of the dataset in level-0 blocks. */ - private final long[] datasetSizeInBlocks; + private final long[] datasetSizeInChunks; /** * {@code blockSizes[l][d]} is the block size at level {@code l} in dimension {@code d}. @@ -321,10 +322,10 @@ public NestedGrid(int[][] blockSizes, long[] datasetSize) { } if (datasetSize == null) { - datasetSizeInBlocks = null; + datasetSizeInChunks = null; } else { - datasetSizeInBlocks = new long[numDimensions]; - Arrays.setAll(datasetSizeInBlocks, d -> (datasetSize[d] + blockSizes[0][d] - 1) / blockSizes[0][d]); + datasetSizeInChunks = new long[numDimensions]; + Arrays.setAll(datasetSizeInChunks, d -> (datasetSize[d] + blockSizes[0][d] - 1) / blockSizes[0][d]); } } @@ -337,7 +338,7 @@ public NestedGrid(int[][] blockSizes) { * level} grid {@code position}. *

* Note that {@code position} is in units of grid elements at {@code - * level}. Positions with {@code level=0} refer to the DataBlock grid, + * level}. Positions with {@code level=0} refer to the Chunk grid, * positions with {@code level=1} refer to first-level Shard grid, and * so on. *

@@ -356,18 +357,18 @@ public NestedPosition nestedPosition(final long[] position, final int level) { } /** - * Create a {@code NestedPosition} at the specified block grid {@code + * Create a {@code NestedPosition} at the specified chunk grid {@code * position} (that is, at nesting level 0). *

- * Note that {@code position} is in units of DataBlocks. + * Note that {@code position} is in units of chunks. *

* The returned {@code NestedPosition} will have * {@link NestedPosition#level() level()==0}. * * @param position - * position at level 0 (block grid) + * position at level 0 (chunk grid) * - * @return a NestedPosition representation of the specified block grid position + * @return a NestedPosition representation of the specified chunk grid position */ public NestedPosition nestedPosition(final long[] position) { return nestedPosition(position, 0); @@ -429,12 +430,12 @@ public long[] pixelPosition( } /** - * Get the maximum pixel position in the shard/block at the given {@code - * sourcePos} grid position at {@code sourceLevel}. + * Get the maximum pixel position in the block (chunk/shard) at the + * given {@code sourcePos} grid position at {@code sourceLevel}. *

* Note that this does not take into account {@link #getDatasetSize() * dataset dimensions}. That is, it is always assumed that the - * shard/block has the default size. + * chunk/shard has the default size. * * @param sourcePos * a grid position at {@code sourceLevel} @@ -454,12 +455,12 @@ public void maxPixelPosition( } /** - * Get the maximum pixel position in the shard/block at the given {@code - * sourcePos} grid position at {@code sourceLevel}. + * Get the maximum pixel position in the block (chunk/shard) at the + * given {@code sourcePos} grid position at {@code sourceLevel}. *

* Note that this does not take into account {@link #getDatasetSize() * dataset dimensions}. That is, it is always assumed that the - * shard/block has the default size. + * chunk/shard has the default size. * * @param sourcePos * a grid position at {@code sourceLevel} @@ -483,7 +484,7 @@ public long[] maxPixelPosition( *

* For example, this can be used to compute the coordinates on the shard * grid ({@code targetLevel==1}) of the shard containing a given - * datablock ({@code sourcePos} at {@code sourceLevel==0}). + * chunk ({@code sourcePos} at {@code sourceLevel==0}). * * @param sourcePos * a grid position at {@code sourceLevel} @@ -512,7 +513,7 @@ public void absolutePosition( *

* For example, this can be used to compute the coordinates on the shard * grid ({@code targetLevel==1}) of the shard containing a given - * datablock ({@code sourcePos} at {@code sourceLevel==0}). + * chunk ({@code sourcePos} at {@code sourceLevel==0}). * * @param sourcePos * the source position j @@ -537,11 +538,11 @@ public long[] absolutePosition( /** * Get the absolute grid position at {@code targetLevel} for the given - * {@code sourcePos} block grid position (level 0). + * {@code sourcePos} chunk grid position (level 0). *

* For example, this can be used to compute the coordinates on the shard * grid ({@code targetLevel==1}) of the shard containing a given - * datablock ({@code sourcePos}. + * chunk ({@code sourcePos}. * * @param sourcePos * the source position j @@ -564,8 +565,8 @@ public long[] absolutePosition( * numLevels} or the dataset for {@code targetLevel+1 == numLevels}.) *

* For example, this can be used to compute the grid coordinates {@code - * targetLevel==0} of a given datablock ({@code sourcePos} at {@code - * sourceLevel==0}) withing a shard (containing element at level {@code + * targetLevel==0} of a given chunk ({@code sourcePos} at {@code + * sourceLevel==0}) within a shard (containing element at level {@code * targetLevel+1==1}). *

* @@ -601,18 +602,12 @@ public long[] relativePosition( return targetPos; } - public long[] relativePosition( - final long[] sourcePos, - final int targetLevel) { - return relativePosition(sourcePos, 0, targetLevel); - } - /** * Get size of a block at the given {@code level} relative to {@code * level-1} (that is, in units of {@code level-1} blocks). *

* For example {@code relativeBlockSize(1)} returns the number of - * datablocks in a (non-nested) shard. + * chunks in a (non-nested) shard. */ public int[] relativeBlockSize(final int level) { return relativeToAdjacent[level]; @@ -620,10 +615,10 @@ public int[] relativeBlockSize(final int level) { /** * Get size of a block at the given {@code level} relative to level {@code - * 0} (that is, in units of {@code level-0} blocks). + * 0} (that is, in units of chunks). *

* For example {@code relativeToBaseBlockSize(1)} returns the number of - * datablocks in a (non-nested) shard. + * chunks in a (non-nested) shard. */ public int[] relativeToBaseBlockSize(final int level) { return relativeToBase[level]; @@ -642,15 +637,15 @@ public long[] getDatasetSize() { } /** - * Get the size of the dataset in units of DataBlocks. + * Get the size of the dataset in units of chunks. *

* This might return {@code null}, if this {@code NestedGrid} was not * constructed with dataset dimensions. * - * @return size of the dataset in pixels + * @return size of the dataset in chunks */ - public long[] getDatasetSizeInBlocks() { - return datasetSizeInBlocks; + public long[] getDatasetSizeInChunks() { + return datasetSizeInChunks; } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java index 80035b958..a0e87b1ad 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java @@ -39,33 +39,33 @@ /** * Wrap a KeyValueAccess and a dataset URI to be able to get/set values (ReadData) by {@code long[]} key - * indicating the position of a block or shard. + * indicating the position of a top-level DataBlock (a top-level shard for sharded datasets, + * a chunk for non-sharded datasets). */ public interface PositionValueAccess { /** - * Gets the {@link VolatileReadData} for the DataBlock (or shard) at the - * given position in the block (or shard) grid. + * Gets the {@link VolatileReadData} for the DataBlock at the given position + * in the block grid. *

* If the requested key does not exist, either {@code null} is returned or a * lazy {@code VolatileReadData} that will throw {@code N5NoSuchKeyException} * when trying to materialize. * * @param key - * The position of the data block or shard - * @return ReadData for the given key or {@code null} if the key doesn't - * exist + * The position of the block + * @return ReadData for the given key or {@code null} if the key doesn't exist * @throws N5Exception.N5IOException * if an error occurs while reading */ VolatileReadData get(long[] key) throws N5Exception.N5IOException; /** - * Write the {@code data} for a DataBlock (or shard) to the given position - * in the block grid. + * Write the {@code data} for a DataBlock to the given position in the block + * grid. * * @param key - * The grid position of the DataBlock (or shard) to write + * The grid position of the DataBlock to write * @param data * The data to write * @@ -106,7 +106,7 @@ class KvaPositionValueAccess implements PositionValueAccess { } /** - * Constructs the absolute path for a data block (or shard) at a given grid + * Constructs the absolute path for a DataBlock at a given grid * position. * * @param gridPosition diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java index 01d7e961b..419ee1ccb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java @@ -28,6 +28,11 @@ */ package org.janelia.saalfeldlab.n5.shard; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.segment.Segment; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; @@ -86,4 +91,16 @@ public void setElementData(final ReadData data, final long[] pos) { final Segment segment = data == null ? null : SegmentedReadData.wrap(data).segments().get(0); index.set(segment, pos); } + + public void prefetch(List positions) { + + final List ranges = new ArrayList<>(positions.size()); + for (long[] pos : positions) { + final Segment seg = index.get(pos); + if (seg != null) + ranges.add(sourceData.location(seg)); + } + sourceData.prefetch(ranges); + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java index e23640333..21c7f63f1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardCodec.java @@ -45,7 +45,7 @@ public class RawShardCodec implements BlockCodec { /** - * Number of elements (DataBlocks, nested shards) in each dimension per shard. + * Number of elements (chunks, nested shards) in each dimension per shard. */ private final int[] size; private final IndexLocation indexLocation; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java index f129c9791..b2124851f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShardDataBlock.java @@ -46,7 +46,7 @@ public class RawShardDataBlock implements DataBlock { } // TODO: should this be the number of elements in the Shard (number of - // sub-shards / datablock) along each dimension, or the number of + // sub-shards / chunks) along each dimension, or the number of // pixels alon each dimension? @Override public int[] getSize() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java index 38a267ba1..ecfaa9570 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/Region.java @@ -36,7 +36,7 @@ /** * Bounds, in pixel coordinates, of a region in a dataset. *

- * Provides methods to find which blocks and shards are contained in the + * Provides methods to find which chunks and shards are contained in the * region, iterate sub-NestedPositions, etc. */ public class Region { @@ -63,12 +63,12 @@ public class Region { private final long[] size; /** - * {@code NestedPosition} of the block containing the min pixel position. + * {@code NestedPosition} of the chunk containing the min pixel position. */ private final Nesting.NestedPosition minPos; /** - * {@code NestedPosition} of the block containing the max pixel position. + * {@code NestedPosition} of the chunk containing the max pixel position. */ private final Nesting.NestedPosition maxPos; @@ -79,37 +79,38 @@ public Region(final long[] min, final long[] size, final NestedGrid grid) { this.datasetDimensions = grid.getDatasetSize(); final int n = min.length; - final int[] blockSize = grid.getBlockSize(0); + final int[] chunkSize = grid.getBlockSize(0); - final long[] minBlock = new long[n]; - Arrays.setAll(minBlock, d -> min[d] / blockSize[d]); - minPos = grid.nestedPosition(minBlock); + final long[] minChunk = new long[n]; + Arrays.setAll(minChunk, d -> min[d] / chunkSize[d]); + minPos = grid.nestedPosition(minChunk); - final long[] maxBlock = new long[n]; - Arrays.setAll(maxBlock, d -> (min[d] + size[d] - 1) / blockSize[d]); - maxPos = grid.nestedPosition(maxBlock); + final long[] maxChunk = new long[n]; + Arrays.setAll(maxChunk, d -> (min[d] + size[d] - 1) / chunkSize[d]); + maxPos = grid.nestedPosition(maxChunk); } /** - * Get the {@code NestedPosition} of the minimum DataBlock touched by the region. + * Get the {@code NestedPosition} of the minimum chunk touched by the region. */ public Nesting.NestedPosition minPos() { return minPos; } /** - * Get the {@code NestedPosition} of the maximum DataBlock touched by the region. + * Get the {@code NestedPosition} of the maximum chunk touched by the region. */ public Nesting.NestedPosition maxPos() { return maxPos; } /** - * Check whether the Shard or DataBlock corresponding to the given - * position is fully contained inside the region. - * (The {@link Nesting.NestedPosition#level() level} of {@code position} is used - * to determine whether it refers to a DataBlock or a (potentially - * nested) Shard. + * Check whether the shard or chunk corresponding to the given position is + * fully contained inside the region. + *

+ * The {@link Nesting.NestedPosition#level() level} of {@code position} is + * used to determine whether it refers to a chunk or a (potentially nested) + * shard. * * @param position * the NestedPosition to check diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java index fb324881a..8fa019c20 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/ShardCodecInfo.java @@ -37,8 +37,8 @@ public interface ShardCodecInfo extends BlockCodecInfo { /** - * Chunk size of each shard element (either nested shard or DataBlock) - * + * Size in pixels of each shard element (either nested shard or chunk) + * * @return the size of each shard element */ int[] getInnerBlockSize(); @@ -64,14 +64,14 @@ public interface ShardCodecInfo extends BlockCodecInfo { /** * BlockCodec for shard index - * + * * @return the BlockCodecInfo for this shard's index */ BlockCodecInfo getIndexBlockCodecInfo(); /** * Deterministic-size DataCodecs for index BlockCodec - * + * * @return the collection of DataCodecInfos for this shard's index */ DataCodecInfo[] getIndexDataCodecInfos(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index dc12d3943..c240cac43 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -74,6 +74,9 @@ /** * Abstract base class for testing N5 functionality. * Subclasses are expected to provide a specific N5 implementation to be tested by defining the {@link #createN5Writer()} method. + *

+ * This class does not create sharded datasets. Its tests generally call read/writeBlock which are equivalent to read/writeChunk + * for the cases being tested here. The test {@link #testReadChunkVsBlock} checks that the equivalence between these methods holds. * * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> * @author Igor Pisarev <pisarevi@janelia.hhmi.org> @@ -162,7 +165,7 @@ protected N5Writer createN5Writer(final String location) throws IOException, URI return createN5Writer(location, new GsonBuilder()); } - /* Tests that overide this should enusre that the `N5Writer` created will remove its container on close() */ + /* Tests that override this should ensure that the `N5Writer` created will remove its container on close() */ protected abstract N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException, URISyntaxException; protected N5Reader createN5Reader(final String location) throws IOException, URISyntaxException { @@ -243,6 +246,7 @@ public void testCreateDataset() { } assertArrayEquals(dimensions, info.getDimensions()); assertArrayEquals(blockSize, info.getBlockSize()); + assertArrayEquals("blockSize == chunkSize when not sharded", blockSize, info.getChunkSize()); assertEquals(DataType.UINT64, info.getDataType()); } @@ -254,8 +258,8 @@ public void testBlocksLargerThanDimensions() { final int[] largeBlockSize = new int[]{5, 7, 10}; try (final N5Writer n5 = createTempN5Writer()) { - n5.createDataset(datasetName, smallDimensions, largeBlockSize, DataType.UINT8, new RawCompression()); - final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + final DatasetAttributes attributes = n5.createDataset( + datasetName, smallDimensions, largeBlockSize, DataType.UINT8, new RawCompression()); // Create a block that is larger than the dataset dimensions final int numElements = largeBlockSize[0] * largeBlockSize[1] * largeBlockSize[2]; @@ -330,8 +334,6 @@ public void testUnalignedBlocksTruncatedAtEnd() { } } - - @Test public void testWriteReadByteBlock() { @@ -475,32 +477,35 @@ public void testWriteReadDoubleBlock() { } @Test - public void testWriteReadShard() { - - // test that writeShard behaves the same as writeBlock - // for unsharded datasets + public void testReadChunkVsBlock() { + // test that readBlock behaves the same as readChunk for unsharded datasets for (final Compression compression : getCompressions()) { try (final N5Writer n5 = createTempN5Writer()) { + final short[] shortData1 = new short[shortBlock.length]; + for( int i = 0; i < shortBlock.length; i++) + shortData1[i] = (short)(2 * shortBlock[i] + 3); + n5.createDataset(datasetName, dimensions, blockSize, DataType.INT16, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); - final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock); + final ShortArrayDataBlock dataBlock0 = new ShortArrayDataBlock(blockSize, new long[]{0, 0, 0}, shortBlock); + final ShortArrayDataBlock dataBlock1 = new ShortArrayDataBlock(blockSize, new long[]{1, 0, 0}, shortData1); - n5.writeShard(datasetName, attributes, dataBlock); - - // read with readShard - final DataBlock loadedShard = n5.readShard(datasetName, attributes, 0, 0, 0); - assertArrayEquals(shortBlock, (short[])loadedShard.getData()); + n5.writeChunk(datasetName, attributes, dataBlock0); + n5.writeBlock(datasetName, attributes, dataBlock1); // read with readBlock - final DataBlock loadedDataBlock = n5.readShard(datasetName, attributes, 0, 0, 0); - assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData()); + assertArrayEquals(shortBlock, (short[])n5.readBlock(datasetName, attributes, 0, 0, 0).getData()); + assertArrayEquals(shortData1, (short[])n5.readBlock(datasetName, attributes, 1, 0, 0).getData()); + + // read with readChunk + assertArrayEquals(shortBlock, (short[])n5.readChunk(datasetName, attributes, 0, 0, 0).getData()); + assertArrayEquals(shortData1, (short[])n5.readChunk(datasetName, attributes, 1, 0, 0).getData()); } } } - @Test public void testMode1WriteReadByteBlock() { @@ -766,6 +771,21 @@ public void testAttributes() { } } + @Test + public void testDatasetAttributes() { + + final String dset = ""; + final String key = "user-attr"; + final String value = "value"; + try (final N5Writer n5 = createTempN5Writer()) { + + n5.setAttribute(dset, key, value); + n5.setDatasetAttributes(dset, new DatasetAttributes(new long[]{5}, new int[]{5}, DataType.INT32)); + assertNotNull(n5.getDatasetAttributes(dset)); + assertEquals(value, n5.getAttribute(dset, key, String.class)); + } + } + @Test public void testNullAttributes() throws URISyntaxException, IOException { @@ -1302,15 +1322,22 @@ public void testDelete() { n5.writeBlock(datasetName, attributes, dataBlock); // block should exist at position1 but not at position2 - final DataBlock readBlock = n5.readBlock(datasetName, attributes, position1); + final DataBlock readBlock = n5.readChunk(datasetName, attributes, position1); assertNotNull(readBlock); assertTrue(readBlock instanceof ByteArrayDataBlock); assertArrayEquals(byteBlock, ((ByteArrayDataBlock)readBlock).getData()); - assertTrue("deleting existing block should return true", n5.deleteBlock(datasetName, position1)); + assertTrue("deleting existing chunk should return true", n5.deleteChunk(datasetName, position1)); + assertFalse("deleting non-existing chunk should return false", n5.deleteChunk(datasetName, position1)); + assertFalse("deleting non-existing chunk should return false", n5.deleteChunk(datasetName, position2)); + + // for an unsharded dataset, deleteChunk and deleteBlock behave identically assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position1)); assertFalse("deleting non-existing block should return false", n5.deleteBlock(datasetName, position2)); + n5.writeChunk(datasetName, attributes, dataBlock); + assertTrue("deleting existing block should return true", n5.deleteBlock(datasetName, position1)); + // no block should exist anymore assertNull(n5.readBlock(datasetName, attributes, position1)); assertNull(n5.readBlock(datasetName, attributes, position2)); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java index 8725baf34..7e57c6ae0 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/DatasetAttributesTest.java @@ -46,7 +46,7 @@ public class DatasetAttributesTest { /** - * Test that validateBlockShardSizes method accepts valid shard and block size combinations. + * Test that validateBlockShardSizes method accepts valid shard and chunk size combinations. */ @Test public void testValidateBlockShardSizesValid() { @@ -54,49 +54,53 @@ public void testValidateBlockShardSizesValid() { // Test case 1: shard size equals block size long[] dimensions = new long[]{100, 200, 300}; int[] shardSize = new int[]{64, 64, 64}; - int[] blockSize = new int[]{64, 64, 64}; + int[] chunkSize = new int[]{64, 64, 64}; DataType dataType = DataType.UINT8; // This should not throw any exception - DatasetAttributes attrs = shardDatasetAttributes(dimensions, shardSize, blockSize, dataType); - assertEquals(blockSize, attrs.getBlockSize()); + DatasetAttributes attrs = shardDatasetAttributes(dimensions, shardSize, chunkSize, dataType); + assertEquals(chunkSize, attrs.getChunkSize()); + assertEquals(shardSize, attrs.getBlockSize()); NestedGrid grid = attrs.getNestedBlockGrid(); - assertEquals(blockSize, grid.getBlockSize(0)); + assertEquals(chunkSize, grid.getBlockSize(0)); assertEquals(shardSize, grid.getBlockSize(1)); // Test case 2: shard size is a multiple of block size shardSize = new int[]{128}; - blockSize = new int[]{64}; - attrs = shardDatasetAttributes(new long[]{128}, shardSize, blockSize, dataType); - assertEquals(blockSize, attrs.getBlockSize()); + chunkSize = new int[]{64}; + attrs = shardDatasetAttributes(new long[]{128}, shardSize, chunkSize, dataType); + assertEquals(chunkSize, attrs.getChunkSize()); + assertEquals(shardSize, attrs.getBlockSize()); grid = attrs.getNestedBlockGrid(); - assertEquals(blockSize, grid.getBlockSize(0)); + assertEquals(chunkSize, grid.getBlockSize(0)); assertEquals(shardSize, grid.getBlockSize(1)); // Test case 3: different multiples per dimension shardSize = new int[]{128, 256, 32, 2}; - blockSize = new int[]{32, 64, 32, 1}; - attrs = shardDatasetAttributes(new long[]{128, 128, 128, 128}, shardSize, blockSize, dataType ); - assertEquals(blockSize, attrs.getBlockSize()); + chunkSize = new int[]{32, 64, 32, 1}; + attrs = shardDatasetAttributes(new long[]{128, 128, 128, 128}, shardSize, chunkSize, dataType ); + assertEquals(chunkSize, attrs.getChunkSize()); + assertEquals(shardSize, attrs.getBlockSize()); grid = attrs.getNestedBlockGrid(); - assertEquals(blockSize, grid.getBlockSize(0)); + assertEquals(chunkSize, grid.getBlockSize(0)); assertEquals(shardSize, grid.getBlockSize(1)); // Test case 4: large multiples shardSize = new int[]{1024, 2048, 512}; - blockSize = new int[]{32, 64, 16}; - attrs = shardDatasetAttributes(dimensions, shardSize, blockSize, dataType); - assertEquals(blockSize, attrs.getBlockSize()); + chunkSize = new int[]{32, 64, 16}; + attrs = shardDatasetAttributes(dimensions, shardSize, chunkSize, dataType); + assertEquals(chunkSize, attrs.getChunkSize()); + assertEquals(shardSize, attrs.getBlockSize()); grid = attrs.getNestedBlockGrid(); - assertEquals(blockSize, grid.getBlockSize(0)); + assertEquals(chunkSize, grid.getBlockSize(0)); assertEquals(shardSize, grid.getBlockSize(1)); } private static DatasetAttributes shardDatasetAttributes( - long[] dimensions, int[] shardSize, int[] blockSize, DataType dataType) { + long[] dimensions, int[] shardSize, int[] chunkSize, DataType dataType) { DefaultShardCodecInfo blockCodecInfo = new DefaultShardCodecInfo( - blockSize, + chunkSize, new N5BlockCodecInfo(), new DataCodecInfo[]{new RawCompression()}, new RawBlockCodecInfo(), diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java index 546a4072f..198aeae71 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java @@ -141,7 +141,7 @@ public void testDocExample() { n5.createDataset(compressedDatasetName, new long[]{1, 2, 3}, new int[]{1, 2, 3}, DataType.UINT16, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(compressedDatasetName); final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(new int[]{1, 2, 3}, new long[]{0, 0, 0}, dataBlockData); - n5.writeBlock(compressedDatasetName, attributes, dataBlock); + n5.writeChunk(compressedDatasetName, attributes, dataBlock); } catch (final N5Exception e) { fail(e.getMessage()); } @@ -165,7 +165,7 @@ public void benchmarkWritingSpeed() { for (int y = 0; y < nBlocks; ++y) for (int x = 0; x < nBlocks; ++x) { final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(new int[]{64, 64, 64}, new long[]{x, y, z}, data); - n5.writeBlock(compressedDatasetName, attributes, dataBlock); + n5.writeChunk(compressedDatasetName, attributes, dataBlock); } } catch (final N5Exception e) { fail(e.getMessage()); @@ -255,7 +255,7 @@ public void benchmarkParallelWritingSpeed() { exec.submit( () -> { final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(new int[]{64, 64, 64}, new long[]{fx, fy, fz}, data); - n5.writeBlock(compressedDatasetName, attributes, dataBlock); + n5.writeChunk(compressedDatasetName, attributes, dataBlock); return true; })); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java index ecf7643a7..0b959fb5c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5ReadBenchmark.java @@ -114,7 +114,7 @@ public void setup() { n5.createDataset(datasetName, dimensions, blockSize, DataType.FLOAT64, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final DoubleArrayDataBlock dataBlock = new DoubleArrayDataBlock(blockSize, new long[] {0, 0, 0}, doubleBlock); - n5.writeBlock(datasetName, attributes, dataBlock); + n5.writeChunk(datasetName, attributes, dataBlock); } n5 = new N5FSReader(basePath, new GsonBuilder()); @@ -129,7 +129,7 @@ public void setup() { @BenchmarkMode( Mode.AverageTime ) @OutputTimeUnit( TimeUnit.MILLISECONDS ) public void bench() { - n5.readBlock(datasetName, attributes, 0, 0, 0); + n5.readChunk(datasetName, attributes, 0, 0, 0); } public static void main( final String... args ) throws RunnerException, IOException diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java index bb579c813..17d421750 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java @@ -35,7 +35,6 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.file.Files; import java.util.Arrays; @@ -76,7 +75,7 @@ public void backwardReadHelper(final String base, final String dsetPath) throws byte value = 0; long[] p = new long[2]; - DataBlock b00 = n5.readBlock(dsetPath, attrs, p); + DataBlock b00 = n5.readChunk(dsetPath, attrs, p); assertNotNull(b00); assertArrayEquals(new int[]{5,4}, b00.getSize()); assertArrayEquals(expectedData(20, value), b00.getData()); @@ -84,7 +83,7 @@ public void backwardReadHelper(final String base, final String dsetPath) throws p[0] = 1; p[1] = 0; value++; - DataBlock b10 = n5.readBlock(dsetPath, attrs, p); + DataBlock b10 = n5.readChunk(dsetPath, attrs, p); assertNotNull(b10); assertArrayEquals(new int[]{2,4}, b10.getSize()); assertArrayEquals(expectedData(8, value), b10.getData()); @@ -92,7 +91,7 @@ public void backwardReadHelper(final String base, final String dsetPath) throws p[0] = 0; p[1] = 1; value++; - DataBlock b01 = n5.readBlock(dsetPath, attrs, p); + DataBlock b01 = n5.readChunk(dsetPath, attrs, p); assertNotNull(b01); assertArrayEquals(new int[]{5,1}, b01.getSize()); assertArrayEquals(expectedData(5, value), b01.getData()); @@ -100,7 +99,7 @@ public void backwardReadHelper(final String base, final String dsetPath) throws p[0] = 1; p[1] = 1; value++; - DataBlock b11 = n5.readBlock(dsetPath, attrs, p); + DataBlock b11 = n5.readChunk(dsetPath, attrs, p); assertNotNull(b11); assertArrayEquals(new int[]{2,1}, b11.getSize()); assertArrayEquals(expectedData(2, value), b11.getData()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java index 032534b1e..c40b67f26 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CreateSampleData.java @@ -62,28 +62,28 @@ public static N5FSWriter createSampleData(String baseDir, String dataset, Compre byte val = 0; long[] pos = new long[]{0, 0}; - n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + n5.writeChunk(dsetPath, attrs, createDataBlock(blkSize, pos, val)); pos[0] = 1; pos[1] = 0; blkSize[0] = 2; blkSize[1] = 4; val++; - n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + n5.writeChunk(dsetPath, attrs, createDataBlock(blkSize, pos, val)); pos[0] = 0; pos[1] = 1; blkSize[0] = 5; blkSize[1] = 1; val++; - n5.writeBlock(dsetPath, attrs, createDataBlock(blkSize, pos, val)); + n5.writeChunk(dsetPath, attrs, createDataBlock(blkSize, pos, val)); pos[0] = 1; pos[1] = 1; blkSize[0] = 2; blkSize[1] = 1; val++; - n5.writeBlock(dsetPath, attrs, createDataBlock( blkSize, pos, val )); + n5.writeChunk(dsetPath, attrs, createDataBlock( blkSize, pos, val )); return n5; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java index ffa35156f..ec4dad3a2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java @@ -30,7 +30,6 @@ import java.io.File; import java.io.IOException; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java index 4ce148c50..73b6a2f46 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java @@ -29,8 +29,6 @@ package org.janelia.saalfeldlab.n5.benchmarks; import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -40,7 +38,6 @@ import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java index 1e37b496b..f8b8555dc 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/codec/BlockCodecTests.java @@ -49,7 +49,6 @@ import org.janelia.saalfeldlab.n5.ShortArrayDataBlock; import org.janelia.saalfeldlab.n5.codec.BytesCodecTests.BitShiftBytesCodec; import org.janelia.saalfeldlab.n5.shard.DatasetAccess; -import org.janelia.saalfeldlab.n5.shard.DatasetAccessTest; import org.janelia.saalfeldlab.n5.shard.PositionValueAccess; import org.janelia.saalfeldlab.n5.shard.TestPositionValueAccess; import org.junit.Test; @@ -146,7 +145,7 @@ public void testEmptyBlock() throws Exception { final int[] blockSize = {0, 0}; final long[] gridPosition = {0, 0}; final N5BlockCodecInfo blockCodecInfo = new N5BlockCodecInfo(); - final DatasetAccessTest.TestDatasetAttributes attributes = new DatasetAccessTest.TestDatasetAttributes( + final TestDatasetAttributes attributes = new TestDatasetAttributes( new long[]{64, 64}, new int[]{8, 8}, DataType.UINT8, @@ -154,13 +153,13 @@ public void testEmptyBlock() throws Exception { new RawCompression()); final PositionValueAccess store = new TestPositionValueAccess(); - DatasetAccess access = attributes.datasetAccess(); + DatasetAccess access = attributes.getDatasetAccess(); // Test encode/decode final ByteArrayDataBlock emptyBlock = new ByteArrayDataBlock(blockSize, gridPosition, new byte[0]); - access.writeBlock(store, emptyBlock); - final DataBlock decoded = access.readBlock(store, gridPosition); + access.writeChunk(store, emptyBlock); + final DataBlock decoded = access.readChunk(store, gridPosition); assertEquals("Empty block should have 0 elements", 0, decoded.getNumElements()); } @@ -315,5 +314,18 @@ private static void assertDataEquals(DataBlock expected, DataBlock actual) } } + public static class TestDatasetAttributes extends DatasetAttributes { + + public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType dataType, BlockCodecInfo blockCodecInfo, + DataCodecInfo... dataCodecInfos) { + + super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); + } + + @Override // to make this accessible for the test + protected DatasetAccess getDatasetAccess() { + return super.getDatasetAccess(); + } + } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java index 8572e1cee..fd151a265 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java @@ -30,17 +30,14 @@ import org.apache.commons.io.IOUtils; import org.janelia.saalfeldlab.n5.HttpKeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.junit.Test; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.charset.Charset; -import java.util.function.Function; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java index fe13f0d68..4f217c097 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpReaderFsWriter.java @@ -56,7 +56,7 @@ public class HttpReaderFsWriter implements GsonKeyValueN5Writer { private final GsonKeyValueN5Reader reader; public HttpReaderFsWriter(final W writer, final R reader) { - + this.writer = writer; this.reader = reader; @@ -116,9 +116,9 @@ public HttpRead return reader.getDatasetAttributes(pathName); } - @Override public DataBlock readBlock(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { + @Override public DataBlock readChunk(String pathName, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { - return reader.readBlock(pathName, getConvertedDatasetAttributes(datasetAttributes), gridPosition); + return reader.readChunk(pathName, getConvertedDatasetAttributes(datasetAttributes), gridPosition); } @Override public T readSerializedBlock(String dataset, DatasetAttributes attributes, long... gridPosition) throws N5Exception, ClassNotFoundException { @@ -240,7 +240,7 @@ public HttpRead writer.setDatasetAttributes(datasetPath, datasetAttributes); } - + @Override public DatasetAttributes getConvertedDatasetAttributes(DatasetAttributes datasetAttributes) { return writer.getConvertedDatasetAttributes(datasetAttributes); @@ -272,22 +272,22 @@ public HttpRead return convertedDatasetAttributes; } - @Override public void writeBlock(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { - writer.writeBlock(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); + @Override public void writeChunk(String datasetPath, DatasetAttributes datasetAttributes, DataBlock chunk) throws N5Exception { + writer.writeChunk(datasetPath, getConvertedDatasetAttributes(datasetAttributes), chunk); } - @Override public void writeShard(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { - writer.writeShard(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); + @Override public void writeBlock(String datasetPath, DatasetAttributes datasetAttributes, DataBlock dataBlock) throws N5Exception { + writer.writeBlock(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlock); } - @Override public boolean deleteBlock(String datasetPath, long... gridPosition) throws N5Exception { + @Override public boolean deleteChunk(String datasetPath, long... gridPosition) throws N5Exception { - return writer.deleteBlock(datasetPath, gridPosition); + return writer.deleteChunk(datasetPath, gridPosition); } - @Override public boolean deleteBlocks(String datasetPath, DatasetAttributes datasetAttributes, List gridPositions) throws N5Exception { + @Override public boolean deleteChunks(String datasetPath, DatasetAttributes datasetAttributes, List gridPositions) throws N5Exception { - return writer.deleteBlocks(datasetPath, datasetAttributes, gridPositions); + return writer.deleteChunks(datasetPath, datasetAttributes, gridPositions); } @Override public void writeSerializedBlock(Serializable object, String datasetPath, DatasetAttributes datasetAttributes, long... gridPosition) throws N5Exception { @@ -315,8 +315,8 @@ public HttpRead writer.writeAttributes(normalGroupPath, attributes); } - @Override public void writeBlocks(String datasetPath, DatasetAttributes datasetAttributes, DataBlock... dataBlocks) throws N5Exception { + @Override public void writeChunks(String datasetPath, DatasetAttributes datasetAttributes, DataBlock... chunks) throws N5Exception { - writer.writeBlocks(datasetPath, getConvertedDatasetAttributes(datasetAttributes), dataBlocks); + writer.writeChunks(datasetPath, getConvertedDatasetAttributes(datasetAttributes), chunks); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java index 0c4f4a876..a2e18b17d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java @@ -5,13 +5,14 @@ import org.janelia.saalfeldlab.n5.readdata.LazyRead; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; -import org.janelia.saalfeldlab.n5.shard.ShardTest; +import org.janelia.saalfeldlab.n5.readdata.prefetch.AggregatingPrefetchLazyRead; public class TrackingKeyValueAccess extends DelegateKeyValueAccess { public int numMaterializeCalls = 0; public int numIsFileCalls = 0; public long totalBytesRead = 0; + public boolean aggregate = false; public TrackingKeyValueAccess(final KeyValueAccess kva) { super(kva); @@ -25,15 +26,20 @@ public boolean isFile(String normalPath) { @Override public VolatileReadData createReadData(final String normalPath) { -// throw new N5NoSuchKeyException("Test No Such Key"); - return VolatileReadData.from(new TrackingVolatileReadData(kva.createReadData(normalPath))); + + final VolatileReadData volatileReadData = kva.createReadData(normalPath); + final TrackingLazyRead trackingLazyRead = new TrackingLazyRead(volatileReadData); + LazyRead lazyRead = trackingLazyRead; + if (aggregate) + lazyRead = new AggregatingPrefetchLazyRead(trackingLazyRead); + return VolatileReadData.from( lazyRead ); } - private class TrackingVolatileReadData implements LazyRead { + private class TrackingLazyRead implements LazyRead { private final VolatileReadData readData; - TrackingVolatileReadData(final VolatileReadData readData) { + TrackingLazyRead(final VolatileReadData readData) { this.readData = readData; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java new file mode 100644 index 000000000..3499220b9 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java @@ -0,0 +1,58 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.junit.Test; + +public class RangeTests { + + @Test + public void testAggregate() { + + List nonOverlapping = Stream.of(Range.at(0, 5), Range.at(12, 2)).collect(Collectors.toList()); + assertSame(nonOverlapping, Range.aggregate(nonOverlapping)); + + List nonOverlappingRev = Stream.of(Range.at(12, 2), Range.at(0, 5)).collect(Collectors.toList()); + assertSame(nonOverlappingRev, Range.aggregate(nonOverlappingRev)); + + /** + * 0 1 2 3 4 5 + * x x x x x - + * - x - - - - + */ + List containing = Stream.of(Range.at(0, 5), Range.at(1, 1)).collect(Collectors.toList()); + assertEquals(Collections.singletonList(Range.at(0, 5)), Range.aggregate(containing)); + + /** + * 0 1 2 3 4 5 6 + * x x x x x - - + * - - x x x x x + */ + List overlapping = Stream.of(Range.at(0, 5), Range.at(2, 5)).collect(Collectors.toList()); + assertEquals(Collections.singletonList(Range.at(0, 7)), Range.aggregate(overlapping)); + + /** + * 0 1 2 3 4 5 + * x x x x x - + * - - - - - x + */ + List adjacent = Stream.of(Range.at(0, 5), Range.at(5, 1)).collect(Collectors.toList()); + assertEquals(Collections.singletonList(Range.at(0, 6)), Range.aggregate(adjacent)); + + /** + * 0 1 2 3 4 5 + * - - - x x - + * x x - - - - + * - - - - - x + */ + List three = Stream.of(Range.at(3, 2), Range.at(0, 2), Range.at(5, 1)).collect(Collectors.toList()); + assertEquals(Stream.of(Range.at(0, 2), Range.at(3, 3)).collect(Collectors.toList()), Range.aggregate(three)); + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index c4f3fa51b..78522ff3e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -38,7 +38,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.file.FileSystems; import java.util.Arrays; import java.util.function.IntUnaryOperator; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java new file mode 100644 index 000000000..fcac9a5e3 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java @@ -0,0 +1,233 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.Range; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.junit.Test; + +public class SliceTrackingLazyReadTests { + + @Test + public void testDefaultSliceTracking() throws N5IOException { + + /** + * 1. Create sample ReadData from byte[] + * 2. Create a DummyLazy Read + * 3. Make a DefaultSliceTrackingLazyRead + * 4. Create a list of ranges + * 5. Call prefetch + * 6. Ensure the correct number of materialize calls were made (always 1 for DefaultSliceTrackingLazyRead) + * 7. Verify the stored slices contain the correct range + */ + + // 1-2. Create a DummyLazyRead with 64 bytes + DummyLazyRead dummyLazyRead = createDummyLazyRead(64); + + // 3. Make a testable DefaultSliceTrackingLazyRead + TestableDefaultSliceTracker sliceTracking = new TestableDefaultSliceTracker(dummyLazyRead); + + // 4. Create a list of ranges (two non-overlapping ranges with a gap) + List ranges = Arrays.asList( + Range.at(10, 5), // offset 10, length 5 (bytes 10-14) + Range.at(50, 10) // offset 50, length 10 (bytes 50-59) + ); + + // 5. Call prefetch + sliceTracking.prefetch(ranges); + + // 6. Ensure exactly 1 materialize call was made + // DefaultSliceTrackingLazyRead creates a single large slice covering all ranges + assertEquals("DefaultSliceTrackingLazyRead should make exactly 1 materialize call", + 1, dummyLazyRead.getNumMaterializeCalls()); + + // 7. Verify the stored slice covers the entire range from offset 10 with length 50 + assertStoredSlices(sliceTracking, Arrays.asList( + Range.at(10, 50) // Single slice covering offset 10-59 + )); + + } + + @Test + public void testAggregatingSliceTracking() throws N5IOException { + + /** + * 1. Create sample ReadData from byte[] + * 2. Create a DummyLazyRead + * 3. Make an AggregatingSliceTrackingLazyRead + * 4. Create a list of ranges + * 5. Call prefetch + * 6. Ensure the correct number of materialize calls were made (one per aggregated range) + * 7. Verify the stored slices contain the correct ranges + */ + + // 1-2. Create a DummyLazyRead with 64 bytes + DummyLazyRead dummyLazyRead = createDummyLazyRead(64); + + // 3. Make a testable AggregatingSliceTrackingLazyRead + TestableAggregatingSliceTracker sliceTracking = new TestableAggregatingSliceTracker(dummyLazyRead); + + /* + * Non-adjacent ranges + */ + // 4. Create a list of ranges (two non-overlapping ranges with a gap) + List ranges = Arrays.asList( + Range.at(10, 5), // offset 10, length 5 (bytes 10-14) + Range.at(50, 10) // offset 50, length 10 (bytes 50-59) + ); + + // 5. Call prefetch + sliceTracking.prefetch(ranges); + + // 6. Ensure exactly 2 materialize calls were made + // AggregatingSliceTrackingLazyRead aggregates overlapping/adjacent ranges + // Since these ranges are not adjacent or overlapping, it makes 2 separate calls + assertEquals("AggregatingSliceTrackingLazyRead should make 2 materialize calls for non-adjacent ranges", + 2, dummyLazyRead.getNumMaterializeCalls()); + + // 7. Verify the stored slices contain two separate ranges + assertStoredSlices(sliceTracking, Arrays.asList( + Range.at(10, 5), // First slice + Range.at(50, 10) // Second slice + )); + + /* + * Adjacent ranges + */ + + // new sliceTracking instance to clear slices + sliceTracking = new TestableAggregatingSliceTracker(dummyLazyRead); + dummyLazyRead.resetNumMaterializeCalls(); + + // 4. Create a list of three contiguous ranges + List adjacentRanges = Arrays.asList( + Range.at(10, 5), // offset 10, length 5 (bytes 10-14) + Range.at(15, 10), // offset 15, length 10 (bytes 15-24) + Range.at(25, 5) // offset 25, length 5 (bytes 25-29) + ); + + // 5. Call prefetch + sliceTracking.prefetch(adjacentRanges); + + // 6. Ensure exactly 1 materialize call was made + // AggregatingSliceTrackingLazyRead should aggregate these three contiguous ranges + // into a single range from offset 10 to 30, with length 20 + assertEquals("AggregatingSliceTrackingLazyRead should make 1 materialize call for contiguous ranges", + 1, dummyLazyRead.getNumMaterializeCalls()); + + // 7. Verify the stored slices now contain three ranges total: + // the two from the first prefetch plus one aggregated range from the second prefetch + assertStoredSlices(sliceTracking, Arrays.asList( + Range.at(10, 20) // Aggregated range + )); + + } + + private static DummyLazyRead createDummyLazyRead(int size) { + byte[] data = new byte[size]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)i; + } + return new DummyLazyRead(ReadData.from(data)); + } + + /** + * Helper method to verify that stored slices match expected ranges. + * + * @param sliceTracking the SliceTrackingLazyRead instance + * @param expectedRanges the expected ranges stored in slices + */ + private static void assertStoredSlices(TestableSliceTracker sliceTracking, List expectedRanges) { + // Access protected slices field via a test helper + List actualSlices = sliceTracking.getSlices(); + + assertEquals("Number of stored slices should match", expectedRanges.size(), actualSlices.size()); + + for (int i = 0; i < expectedRanges.size(); i++) { + Range expected = expectedRanges.get(i); + Range actual = actualSlices.get(i); + assertEquals("Slice " + i + " offset should match", expected.offset(), actual.offset()); + assertEquals("Slice " + i + " length should match", expected.length(), actual.length()); + } + } + + /** + * Testable wrapper for DefaultSliceTrackingLazyRead that exposes slices. + */ + static class TestableDefaultSliceTracker extends EnclosingPrefetchLazyRead implements TestableSliceTracker { + public TestableDefaultSliceTracker(LazyRead delegate) { + super(delegate); + } + + @Override + public List getSlices() { + return java.util.Collections.unmodifiableList(slices); + } + } + + /** + * Testable wrapper for AggregatingSliceTrackingLazyRead that exposes slices. + */ + static class TestableAggregatingSliceTracker extends AggregatingPrefetchLazyRead implements TestableSliceTracker { + public TestableAggregatingSliceTracker(LazyRead delegate) { + super(delegate); + } + + @Override + public List getSlices() { + return java.util.Collections.unmodifiableList(slices); + } + } + + /** + * Interface for testable slice trackers. + */ + interface TestableSliceTracker { + List getSlices(); + } + + static class DummyLazyRead implements LazyRead { + + private ReadData data; + private int numMaterializeCalls = 0; + + public DummyLazyRead( ReadData data ) { + this.data = data; + } + + @Override + public void close() throws IOException { + // no op + } + + @Override + public ReadData materialize(long offset, long length) throws N5IOException { + + numMaterializeCalls++; + return data.slice(offset, length).materialize(); + } + + @Override + public long size() throws N5IOException { + + return data.length(); + } + + public int getNumMaterializeCalls() { + + return numMaterializeCalls; + } + + public void resetNumMaterializeCalls() { + + numMaterializeCalls = 0; + } + + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java new file mode 100644 index 000000000..0ce2b7442 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java @@ -0,0 +1,134 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.util.ArrayList; +import java.util.List; +import org.janelia.saalfeldlab.n5.readdata.Range; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + + +public class SlicesTest { + + private List createSlices(final long[] offsets, final long[] lengths) { + final List slices = new ArrayList<>(); + for (int i = 0; i < offsets.length; ++i) { + slices.add(Range.at(offsets[i], lengths[i])); + } + return slices; + } + + @Test + public void testFindContaining() { + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,6) [-----------] + // (6,4) [---------] + // (8,6) [-----------] + + final List slices = createSlices( + new long[] {2, 6, 8}, + new long[] {6, 4, 6}); + Range slice; + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (1,1) [-] + slice = Slices.findContainingSlice(slices, 1, 1); + assertEquals(null, slice); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,1) [-] + slice = Slices.findContainingSlice(slices, 2, 1); + assertEquals(2, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,6) [-----------] + slice = Slices.findContainingSlice(slices, 2, 6); + assertEquals(2, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,7) [-------------] + slice = Slices.findContainingSlice(slices, 2, 7); + assertEquals(null, slice); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (6,4) [-------] + slice = Slices.findContainingSlice(slices, 6, 4); + assertEquals(6, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (8,2) [---] + slice = Slices.findContainingSlice(slices, 8, 2); + assertEquals(8, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (12,2) [---] + slice = Slices.findContainingSlice(slices, 12, 2); + assertEquals(8, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (12,3) [-----] + slice = Slices.findContainingSlice(slices, 12, 3); + assertEquals(null, slice); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (14,1) [-] + slice = Slices.findContainingSlice(slices, 14, 1); + assertEquals(null, slice); + } + + + @Test + public void testAddSlice() { + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,6) [-----------] + // (6,4) [---------] + // (8,6) [-----------] + final List initial = createSlices( + new long[] {2, 6, 8}, + new long[] {6, 4, 6}); + List slices; + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, Range.at(0, 1)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (0,1) [-] + // (2,6) [-----------] + // (6,4) [---------] + // (8,6) [-----------] + assertEquals(createSlices( + new long[] {0, 2, 6, 8}, + new long[] {1, 6, 4, 6}), slices); + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, Range.at(0, 16)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (0,16)[-------------------------------] + assertEquals(createSlices( + new long[] {0}, + new long[] {16}), slices); + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, Range.at(2, 8)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,8) [-----------------] + // (8,6) [-----------] + assertEquals(createSlices( + new long[] {2, 8}, + new long[] {8, 6}), slices); + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, Range.at(1, 10)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (1,10) [---------------------] + // (8,6) [-----------] + assertEquals(createSlices( + new long[] {1, 8}, + new long[] {10, 6}), slices); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java index 1d3becba7..29c587c9e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/DatasetAccessTest.java @@ -93,7 +93,7 @@ public void setup() { new RawCompression() ); - datasetAccess = attributes.datasetAccess(); + datasetAccess = attributes.getDatasetAccess(); } @@ -103,20 +103,20 @@ public void testWriteReadIndividual() { final PositionValueAccess store = new TestPositionValueAccess(); // write some blocks, filled with constant values - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 0, 0}, 1)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 0, 0}, 2)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {0, 1, 0}, 3)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {1, 1, 0}, 4)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {3, 2, 1}, 5)); - datasetAccess.writeBlock(store, createDataBlock(dataBlockSize, new long[] {8, 4, 1}, 6)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {0, 0, 0}, 1)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {1, 0, 0}, 2)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {0, 1, 0}, 3)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {1, 1, 0}, 4)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {3, 2, 1}, 5)); + datasetAccess.writeChunk(store, createDataBlock(dataBlockSize, new long[] {8, 4, 1}, 6)); // verify that the written blocks can be read back with the correct values - checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), true, 1); - checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); - checkBlock(datasetAccess.readBlock(store, new long[] {0, 1, 0}), true, 3); - checkBlock(datasetAccess.readBlock(store, new long[] {1, 1, 0}), true, 4); - checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5); - checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6); + checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), true, 1); + checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); + checkBlock(datasetAccess.readChunk(store, new long[] {0, 1, 0}), true, 3); + checkBlock(datasetAccess.readChunk(store, new long[] {1, 1, 0}), true, 4); + checkBlock(datasetAccess.readChunk(store, new long[] {3, 2, 1}), true, 5); + checkBlock(datasetAccess.readChunk(store, new long[] {8, 4, 1}), true, 6); } @Test @@ -132,13 +132,13 @@ public void testWriteReadBulk() { for (int i = 0; i < writeGridPositions.size(); i++) { writeBlocks.add(createDataBlock(dataBlockSize, writeGridPositions.get(i), 1 + i)); } - datasetAccess.writeBlocks(store, writeBlocks); + datasetAccess.writeChunks(store, writeBlocks); // verify that the written blocks can be read back with the correct values final List readGridPositions = Arrays.asList(new long[][] { {1, 0, 0}, {0, 0, 0}, {0, 1, 0}, {2, 4, 2}, {3, 2, 1}, {8, 4, 1} }); - final List> readBlocks = datasetAccess.readBlocks(store, readGridPositions); + final List> readBlocks = datasetAccess.readChunks(store, readGridPositions); checkBlock(readBlocks.get(0), true, 2); checkBlock(readBlocks.get(1), true, 1); checkBlock(readBlocks.get(2), true, 3); @@ -160,20 +160,20 @@ public void testDeleteBlock() { for (int i = 0; i < writeGridPositions.size(); i++) { writeBlocks.add(createDataBlock(dataBlockSize, writeGridPositions.get(i), 1 + i)); } - datasetAccess.writeBlocks(store, writeBlocks); + datasetAccess.writeChunks(store, writeBlocks); // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) - datasetAccess.deleteBlock(store, new long[] {0, 0, 0}); - checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); - checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); + datasetAccess.deleteChunk(store, new long[] {0, 0, 0}); + checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), false, 1); + checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); // if a shard becomes empty the corresponding key should be deleted assertTrue(keyExists(store, new long[] {1, 0, 0})); - datasetAccess.deleteBlock(store, new long[] {8, 4, 1}); + datasetAccess.deleteChunk(store, new long[] {8, 4, 1}); assertFalse(keyExists(store, new long[] {1, 0, 0})); // deleting a non-existent block should not fail - datasetAccess.deleteBlock(store, new long[] {0, 0, 8}); + datasetAccess.deleteChunk(store, new long[] {0, 0, 8}); } private boolean keyExists(final PositionValueAccess store, final long[] key) { @@ -200,20 +200,20 @@ public void testDeleteBlocks() { for (int i = 0; i < writeGridPositions.size(); i++) { writeBlocks.add(createDataBlock(dataBlockSize, writeGridPositions.get(i), 1 + i)); } - datasetAccess.writeBlocks(store, writeBlocks); + datasetAccess.writeChunks(store, writeBlocks); // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) - datasetAccess.deleteBlocks(store, Arrays.asList(new long[][] {{0, 0, 0}, {4, 2, 2}, {3, 2, 1}})); - checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); - checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); + datasetAccess.deleteChunks(store, Arrays.asList(new long[][] {{0, 0, 0}, {4, 2, 2}, {3, 2, 1}})); + checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), false, 1); + checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2); // if a shard becomes empty the corresponding key should be deleted assertTrue(keyExists(store, new long[] {1, 0, 0})); - datasetAccess.deleteBlocks(store, Arrays.asList(new long[][] {{8, 4, 1}})); + datasetAccess.deleteChunks(store, Arrays.asList(new long[][] {{8, 4, 1}})); assertFalse(keyExists(store, new long[] {1, 0, 0})); // deleting a non-existent block should not fail - datasetAccess.deleteBlocks(store, Arrays.asList(new long[] {0, 0, 8})); + datasetAccess.deleteChunks(store, Arrays.asList(new long[] {0, 0, 8})); } private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { @@ -243,12 +243,10 @@ public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType d super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); } - public DatasetAccess datasetAccess() { - - // to make this accessible for the test - return createDatasetAccess(); + @Override // to make this accessible for the test + protected DatasetAccess getDatasetAccess() { + return super.getDatasetAccess(); } - } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java index 8d1f4411d..18cdbaf10 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/NestedGridTest.java @@ -78,7 +78,7 @@ public void testAbsolutePositionChunkSize() { } private static long relPosition1D(final NestedGrid grid, final int sourcePos, final int targetLevel) { - return grid.relativePosition(new long[] {sourcePos}, targetLevel)[0]; + return grid.relativePosition(new long[] {sourcePos}, 0, targetLevel)[0]; } @Test diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index d881df45f..36bf19240 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -40,6 +40,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -58,6 +59,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -127,15 +129,15 @@ public void removeTempWriters() { tempN5Factory.removeTempWriters(); } - private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] blockSize) { + private DatasetAttributes getTestAttributes(long[] dimensions, int[] shardSize, int[] chunkSize) { - return getTestAttributes(DataType.UINT8, dimensions, shardSize, blockSize); + return getTestAttributes(DataType.UINT8, dimensions, shardSize, chunkSize); } - private DatasetAttributes getTestAttributes(DataType dataType, long[] dimensions, int[] shardSize, int[] blockSize) { + private DatasetAttributes getTestAttributes(DataType dataType, long[] dimensions, int[] shardSize, int[] chunkSize) { DefaultShardCodecInfo blockCodec = new DefaultShardCodecInfo( - blockSize, + chunkSize, new N5BlockCodecInfo(), new DataCodecInfo[]{new RawCompression()}, new RawBlockCodecInfo(), @@ -155,7 +157,7 @@ protected DatasetAttributes getTestAttributes() { } @Test - public void writeReadBlocksTest() { + public void writeReadChunksTest() { final N5Writer writer = tempN5Factory.createTempN5Writer(); @@ -169,29 +171,29 @@ public void writeReadBlocksTest() { writer.remove(dataset); writer.createDataset(dataset, datasetAttributes); - final int[] blockSize = datasetAttributes.getBlockSize(); - final int numElements = blockSize[0] * blockSize[1]; + final int[] chunkSize = datasetAttributes.getBlockSize(); + final int numElements = chunkSize[0] * chunkSize[1]; final byte[] data = new byte[numElements]; for (int i = 0; i < data.length; i++) { data[i] = (byte)((100) + (10) + i); } - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), - new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + new ByteArrayDataBlock(chunkSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{0, 1}, data), + new ByteArrayDataBlock(chunkSize, new long[]{1, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{1, 1}, data), /* shard (1, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{4, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{5, 0}, data), /* shard (2, 2) */ - new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) + new ByteArrayDataBlock(chunkSize, new long[]{11, 11}, data) ); final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); @@ -212,10 +214,10 @@ public void writeReadBlocksTest() { ensureKeysExist(kva, writer.getURI(), dataset, datasetAttributes, keys); ensureKeysDoNotExist(kva, writer.getURI(), dataset, datasetAttributes, someUnusedKeys); - final long[][] blockIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; - for (long[] blockIndex : blockIndices) { - final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); - Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + final long[][] chunkIndices = new long[][]{{0, 0}, {0, 1}, {1, 0}, {1, 1}, {4, 0}, {5, 0}, {11, 11}}; + for (long[] chunkIndex : chunkIndices) { + final DataBlock chunk = writer.readChunk(dataset, datasetAttributes, chunkIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])chunk.getData()); } final byte[] data2 = new byte[numElements]; @@ -223,19 +225,19 @@ public void writeReadBlocksTest() { data2[i] = (byte)(10 + i); } - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data2), - new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data2), + new ByteArrayDataBlock(chunkSize, new long[]{0, 0}, data2), + new ByteArrayDataBlock(chunkSize, new long[]{1, 1}, data2), /* shard (0, 1) */ - new ByteArrayDataBlock(blockSize, new long[]{0, 4}, data2), - new ByteArrayDataBlock(blockSize, new long[]{0, 5}, data2), + new ByteArrayDataBlock(chunkSize, new long[]{0, 4}, data2), + new ByteArrayDataBlock(chunkSize, new long[]{0, 5}, data2), /* shard (2, 2) */ - new ByteArrayDataBlock(blockSize, new long[]{10, 10}, data2)); + new ByteArrayDataBlock(chunkSize, new long[]{10, 10}, data2)); long[][] keys2 = new long[][]{ {0, 0}, @@ -253,21 +255,21 @@ public void writeReadBlocksTest() { ensureKeysExist(kva, writer.getURI(), dataset, datasetAttributes, keys2); ensureKeysDoNotExist(kva, writer.getURI(), dataset, datasetAttributes, someUnusedKeys2); - final long[][] oldBlockIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; - for (long[] blockIndex : oldBlockIndices) { - final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); - Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + final long[][] oldChunkIndices = new long[][]{{0, 1}, {1, 0}, {4, 0}, {5, 0}, {11, 11}}; + for (long[] chunkIndex : oldChunkIndices) { + final DataBlock chunk = writer.readChunk(dataset, datasetAttributes, chunkIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])chunk.getData()); } - final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; - final List newBlockIndexList = Arrays.asList(newBlockIndices); - final List> readBlocks = writer.readBlocks(dataset, datasetAttributes, newBlockIndexList); - for (int i = 0; i < newBlockIndices.length; i++) { - final long[] blockIndex = newBlockIndices[i]; - final DataBlock block = writer.readBlock(dataset, datasetAttributes, blockIndex); - Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])block.getData()); - final DataBlock blockFromReadBlocks = readBlocks.get(i); - Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])blockFromReadBlocks.getData()); + final long[][] newChunkIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; + final List newChunkIndexList = Arrays.asList(newChunkIndices); + final List> readChunks = writer.readChunks(dataset, datasetAttributes, newChunkIndexList); + for (int i = 0; i < newChunkIndices.length; i++) { + final long[] chunkIndex = newChunkIndices[i]; + final DataBlock chunk = writer.readChunk(dataset, datasetAttributes, chunkIndex); + Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])chunk.getData()); + final DataBlock chunkFromReadChunks = readChunks.get(i); + Assert.assertArrayEquals("Read from shard doesn't match", data2, (byte[])chunkFromReadChunks.getData()); } } @@ -295,7 +297,7 @@ public void writeShardDataSizeTest() { // note: this test depends on the use of raw compression final N5Writer writer = tempN5Factory.createTempN5Writer(); - int numBlocksPerShard = 16; + int numChunksPerShard = 16; final int n5HeaderSizeBytes = 12; // 2 + 2 + 4*2 final DatasetAttributes attrs = getTestAttributes( new long[]{24, 24}, @@ -310,8 +312,8 @@ public void writeShardDataSizeTest() { final KeyValueAccess kva = ((N5KeyValueWriter)writer).getKeyValueAccess(); - final int[] blockSize = datasetAttributes.getBlockSize(); - final int numElements = blockSize[0] * blockSize[1]; + final int[] chunkSize = datasetAttributes.getChunkSize(); + final int numElements = chunkSize[0] * chunkSize[1]; final byte[] data = new byte[numElements]; for (int i = 0; i < data.length; i++) { @@ -319,42 +321,42 @@ public void writeShardDataSizeTest() { } /* - * No blocks or shards exist. - * Calling readBlocks should return a list that is the same length as the requested grid positions, + * No chunks or shards exist. + * Calling readChunks should return a list that is the same length as the requested grid positions, * and every entry should be null. */ - final long[][] newBlockIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; - final List> readBlocks = writer.readBlocks(dataset, datasetAttributes, Arrays.asList(newBlockIndices)); - assertEquals(newBlockIndices.length, readBlocks.size()); - assertTrue("readBlocks for empty shard: all blocks null", readBlocks.stream().allMatch(blk -> blk == null)); + final long[][] newChunkIndices = new long[][]{{0, 0}, {1, 1}, {0, 4}, {0, 5}, {10, 10}}; + final List> readChunks = writer.readChunks(dataset, datasetAttributes, Arrays.asList(newChunkIndices)); + assertEquals(newChunkIndices.length, readChunks.size()); + assertTrue("readChunks for empty shard: all chunks null", readChunks.stream().allMatch(Objects::isNull)); /* - * Now write blocks + * Now write chunks */ - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), - new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + new ByteArrayDataBlock(chunkSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{0, 1}, data), + new ByteArrayDataBlock(chunkSize, new long[]{1, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{1, 1}, data), /* shard (1, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{4, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{5, 0}, data), /* shard (2, 2) */ - new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) + new ByteArrayDataBlock(chunkSize, new long[]{11, 11}, data) ); - final int indexSizeBytes = numBlocksPerShard * 16; // 8 bytes per long * - final int blockDataSizeBytes = numElements + n5HeaderSizeBytes; + final int indexSizeBytes = numChunksPerShard * 16; // 8 bytes per long * + final int chunkDataSizeBytes = numElements + n5HeaderSizeBytes; - // shard 0,0 has 4 blocks so should be this size: - long shard00SizeBytes = indexSizeBytes + 4 * blockDataSizeBytes; - long shard10SizeBytes = indexSizeBytes + 2 * blockDataSizeBytes; - long shard22SizeBytes = indexSizeBytes + 1 * blockDataSizeBytes; + // shard 0,0 has 4 chunks so should be this size: + long shard00SizeBytes = indexSizeBytes + 4 * chunkDataSizeBytes; + long shard10SizeBytes = indexSizeBytes + 2 * chunkDataSizeBytes; + long shard22SizeBytes = indexSizeBytes + 1 * chunkDataSizeBytes; final String[][] keys = new String[][]{ {dataset, "0", "0"}, @@ -377,7 +379,7 @@ public void writeShardDataSizeTest() { } @Test - public void writeReadBlockTest() { + public void writeReadChunkTest() { final GsonKeyValueN5Writer writer = (GsonKeyValueN5Writer)tempN5Factory.createTempN5Writer(); final DatasetAttributes datasetAttributes = getTestAttributes(); @@ -386,33 +388,33 @@ public void writeReadBlockTest() { writer.remove(dataset); writer.createDataset(dataset, datasetAttributes); - final int[] blockSize = datasetAttributes.getBlockSize(); + final int[] chunkSize = datasetAttributes.getChunkSize(); final DataType dataType = datasetAttributes.getDataType(); final int numElements = 2 * 2; - final HashMap writtenBlocks = new HashMap<>(); + final HashMap writtenChunks = new HashMap<>(); for (int idx1 = 1; idx1 >= 0; idx1--) { for (int idx2 = 1; idx2 >= 0; idx2--) { final long[] gridPosition = {idx1, idx2}; - final DataBlock dataBlock = (DataBlock)dataType.createDataBlock(blockSize, gridPosition, numElements); - byte[] data = dataBlock.getData(); + final DataBlock chunk = (DataBlock)dataType.createDataBlock(chunkSize, gridPosition, numElements); + byte[] data = chunk.getData(); for (int i = 0; i < data.length; i++) { data[i] = (byte)((idx1 * 100) + (idx2 * 10) + i); } - writer.writeBlock(dataset, datasetAttributes, dataBlock); + writer.writeChunk(dataset, datasetAttributes, chunk); - final DataBlock block = writer.readBlock(dataset, datasetAttributes, dataBlock.getGridPosition().clone()); - Assert.assertArrayEquals("Read from shard doesn't match", data, (byte[])block.getData()); + final DataBlock readChunk = writer.readChunk(dataset, datasetAttributes, chunk.getGridPosition().clone()); + Assert.assertArrayEquals("Read from shard doesn't match", data, readChunk.getData()); - for (Map.Entry entry : writtenBlocks.entrySet()) { + for (Map.Entry entry : writtenChunks.entrySet()) { final long[] otherGridPosition = entry.getKey(); final byte[] otherData = entry.getValue(); - final DataBlock otherBlock = writer.readBlock(dataset, datasetAttributes, otherGridPosition); - Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, (byte[])otherBlock.getData()); + final DataBlock otherChunk = writer.readChunk(dataset, datasetAttributes, otherGridPosition); + Assert.assertArrayEquals("Read prior write from shard no loner matches", otherData, otherChunk.getData()); } - writtenBlocks.put(gridPosition, data); + writtenChunks.put(gridPosition, data); } } } @@ -425,23 +427,22 @@ public void writeReadShardTest() { final int[] shardSize = new int[] {4,4}; final int shardN = 16; - final int[] blockSize = new int[] {2,2}; - final int blockN = 4; + final int[] chunkSize = new int[] {2,2}; final String dataset = "writeReadShard"; - DatasetAttributes attrs = getTestAttributes(DataType.INT32, new long[]{8, 8}, shardSize, blockSize); + DatasetAttributes attrs = getTestAttributes(DataType.INT32, new long[]{8, 8}, shardSize, chunkSize); final int[] shardData = range(shardN); IntArrayDataBlock shard = new IntArrayDataBlock(shardSize, new long[]{0, 0}, shardData); - n5.writeShard(dataset, attrs, shard); - DataBlock readShard = n5.readShard(dataset, attrs, 0, 0); - assertArrayEquals(shardData, readShard.getData()); + n5.writeBlock(dataset, attrs, shard); + DataBlock readBlock = n5.readBlock(dataset, attrs, 0, 0); + assertArrayEquals(shardData, readBlock.getData()); /** * The 4x4 shard at (0,0) - * and the 2x2 blocks it contains + * and the 2x2 chunks it contains * * * 0 1 | 2 3 @@ -451,15 +452,15 @@ public void writeReadShardTest() { * 12 13 | 14 15 */ - assertArrayEquals(new int[]{0, 1, 4, 5}, (int[])n5.readBlock(dataset, attrs, 0, 0).getData()); - assertArrayEquals(new int[]{2, 3, 6, 7}, (int[])n5.readBlock(dataset, attrs, 1, 0).getData()); - assertArrayEquals(new int[]{8, 9, 12, 13}, (int[])n5.readBlock(dataset, attrs, 0, 1).getData()); - assertArrayEquals(new int[]{10, 11, 14, 15}, (int[])n5.readBlock(dataset, attrs, 1, 1).getData()); + assertArrayEquals(new int[]{0, 1, 4, 5}, (int[])n5.readChunk(dataset, attrs, 0, 0).getData()); + assertArrayEquals(new int[]{2, 3, 6, 7}, (int[])n5.readChunk(dataset, attrs, 1, 0).getData()); + assertArrayEquals(new int[]{8, 9, 12, 13}, (int[])n5.readChunk(dataset, attrs, 0, 1).getData()); + assertArrayEquals(new int[]{10, 11, 14, 15}, (int[])n5.readChunk(dataset, attrs, 1, 1).getData()); - n5.deleteBlock(dataset, attrs, new long[]{1, 1}); + n5.deleteChunk(dataset, attrs, new long[]{1, 1}); /** - * After deleting block (1,1) + * After deleting chunk (1,1) * * 0 1 | 2 3 * 4 5 | 6 7 @@ -467,20 +468,34 @@ public void writeReadShardTest() { * 8 9 | 0 0 * 12 13 | 0 0 */ - final DataBlock partlyEmptyShard = n5.readShard(dataset, attrs, 0, 0); + final DataBlock partlyEmptyShard = n5.readBlock(dataset, attrs, 0, 0); assertArrayEquals(new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 12, 13, 0, 0}, partlyEmptyShard.getData()); - // Delete the rest of the blocks - n5.deleteBlocks(dataset, attrs, + // Delete the rest of the chunks + n5.deleteChunks(dataset, attrs, Stream.of( new long[] {0,0}, new long[] {1,0}, new long[] {0,1}).collect(Collectors.toList())); - assertNull(n5.readShard(dataset, attrs, 0, 0)); + assertNull(n5.readBlock(dataset, attrs, 0, 0)); + + + // write the shard again + n5.writeBlock(dataset, attrs, shard); + + // delete the chard + // ensure it returns true because the shard exists + assertTrue(n5.deleteBlock(dataset, attrs, shard.getGridPosition())); + + // ensure it returns false when the shard does not exist + assertFalse(n5.deleteBlock(dataset, attrs, shard.getGridPosition())); + + // readBlock must return null for the deleted shard + assertNull(n5.readBlock(dataset, attrs, shard.getGridPosition())); } } /** - * Checks how many read calls to the backend are performed for a particular readBlocks + * Checks how many read calls to the backend are performed for a particular readChunks * call. At this time (Nov 4 2025), one read for the index, and one read per block are performed. */ public void numReadsTest() { @@ -497,34 +512,33 @@ public void numReadsTest() { writer.remove(dataset); writer.createDataset(dataset, datasetAttributes); - final int[] blockSize = datasetAttributes.getBlockSize(); - final int numElements = blockSize[0] * blockSize[1]; + final int[] chunkSize = datasetAttributes.getChunkSize(); + final int numElements = chunkSize[0] * chunkSize[1]; final byte[] data = new byte[numElements]; for (int i = 0; i < data.length; i++) { data[i] = (byte)((100) + (10) + i); } - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, /* shard (0, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{0, 1}, data), - new ByteArrayDataBlock(blockSize, new long[]{1, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{1, 1}, data), + new ByteArrayDataBlock(chunkSize, new long[]{0, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{0, 1}, data), + new ByteArrayDataBlock(chunkSize, new long[]{1, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{1, 1}, data), /* shard (1, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), - new ByteArrayDataBlock(blockSize, new long[]{5, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{4, 0}, data), + new ByteArrayDataBlock(chunkSize, new long[]{5, 0}, data), /* shard (2, 2) */ - new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) + new ByteArrayDataBlock(chunkSize, new long[]{11, 11}, data) ); - writer.resetNumMaterializeCalls(); - writer.readBlocks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0})); - System.out.println(writer.getNumMaterializeCalls()); + writer.resetNumMaterializeCalls(); + writer.readChunks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0})); ArrayList ptList = new ArrayList<>(); ptList.add(new long[] {0, 0}); @@ -532,10 +546,8 @@ public void numReadsTest() { ptList.add(new long[] {1, 0}); ptList.add(new long[] {1, 1}); - writer.resetNumMaterializeCalls(); - writer.readBlocks(dataset, datasetAttributes, ptList); - System.out.println(writer.getNumMaterializeCalls()); - System.out.println(""); + writer.resetNumMaterializeCalls(); + writer.readChunks(dataset, datasetAttributes, ptList); } @Test @@ -553,8 +565,8 @@ public void shardExistsTest() { writer.remove(dataset); DatasetAttributes attrs = writer.createDataset(dataset, datasetAttributes); - final int[] blockSize = datasetAttributes.getBlockSize(); - final int numElements = blockSize[0] * blockSize[1]; + final int[] chunkSize = datasetAttributes.getChunkSize(); + final int numElements = chunkSize[0] * chunkSize[1]; final byte[] data = new byte[numElements]; for (int i = 0; i < data.length; i++) { @@ -562,19 +574,19 @@ public void shardExistsTest() { } /* write blocks to shards (0,0), (1,0), and (2,2) */ - writer.writeBlocks( + writer.writeChunks( dataset, attrs, - new ByteArrayDataBlock(blockSize, new long[]{0, 0}, data), /* shard (0, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{4, 0}, data), /* shard (1, 0) */ - new ByteArrayDataBlock(blockSize, new long[]{11, 11}, data) /* shard (2, 2) */ + new ByteArrayDataBlock(chunkSize, new long[]{0, 0}, data), /* shard (0, 0) */ + new ByteArrayDataBlock(chunkSize, new long[]{4, 0}, data), /* shard (1, 0) */ + new ByteArrayDataBlock(chunkSize, new long[]{11, 11}, data) /* shard (2, 2) */ ); TrackingN5Writer trackingWriter = ((TrackingN5Writer) writer); Predicate assertShardExistsTracking = (gridPosition) -> { trackingWriter.resetAllTracking(); - final boolean exists = writer.shardExists(dataset, attrs, gridPosition); + final boolean exists = writer.blockExists(dataset, attrs, gridPosition); assertEquals("isFileCheck incremented", 1, trackingWriter.getNumIsFileCalls()); assertEquals("No Bytes Read", 0, trackingWriter.getTotalBytesRead()); return exists; @@ -613,8 +625,8 @@ public void testPartialReadAggregationBehavior() { writer.remove(dataset); DatasetAttributes attrs = writer.createDataset(dataset, datasetAttributes); - final int[] blockSize = attrs.getBlockSize(); - final int numElements = blockSize[0] * blockSize[1]; + final int[] chunkSize = attrs.getChunkSize(); + final int numElements = chunkSize[0] * chunkSize[1]; final byte[] data = new byte[numElements]; for (int i = 0; i < data.length; i++) { @@ -629,26 +641,41 @@ public void testPartialReadAggregationBehavior() { ptList.add(new long[] {1,1}); /* write blocks to shard (0,0) */ - writer.writeBlocks( + writer.writeChunks( dataset, datasetAttributes, - new ByteArrayDataBlock(blockSize, ptList.get(0), data), - new ByteArrayDataBlock(blockSize, ptList.get(1), data), - new ByteArrayDataBlock(blockSize, ptList.get(2), data), - new ByteArrayDataBlock(blockSize, ptList.get(3), data) + new ByteArrayDataBlock(chunkSize, ptList.get(0), data), + new ByteArrayDataBlock(chunkSize, ptList.get(1), data), + new ByteArrayDataBlock(chunkSize, ptList.get(2), data), + new ByteArrayDataBlock(chunkSize, ptList.get(3), data) ); writer.resetNumMaterializeCalls(); - writer.readBlocks(dataset, datasetAttributes, ptList); + writer.readChunks(dataset, datasetAttributes, ptList); + + // one for the index, one for the four blocks (aggregated) + assertEquals(2, writer.getNumMaterializeCalls()); + + writer.resetNumMaterializeCalls(); + writer.readBlock(dataset, datasetAttributes, new long[] {0,0}); + // one for the index, one for the four blocks (aggregated) + assertEquals(2, writer.getNumMaterializeCalls()); + + + /** + * Aggregate read calls + */ + writer.tkva.aggregate = true; + writer.resetNumMaterializeCalls(); + writer.readChunks(dataset, datasetAttributes, ptList); - // TODO change this if and when we implement aggregation of read calls - // one for the index, one for each of the four blocks - assertEquals(5, writer.getNumMaterializeCalls()); + // one for the index, one that covers ALL the blocks) + assertEquals(2, writer.getNumMaterializeCalls()); writer.resetNumMaterializeCalls(); - writer.readShard(dataset, datasetAttributes, new long[] {0,0}); - // one for the index, one for each of the four blocks - assertEquals(5, writer.getNumMaterializeCalls()); + writer.readBlock(dataset, datasetAttributes, new long[] {0,0}); + // one for the index, one that covers ALL the blocks + assertEquals(2, writer.getNumMaterializeCalls()); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java index 957b8e5dd..7ac8f7db4 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/TestPositionValueAccess.java @@ -41,6 +41,12 @@ import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; +/** + * Implementation of {@link PositionValueAccess} for tests. + *

+ * Instead of writing to a {@code KeyValueAccess} backend, here, data is just + * stored in a {@code Map} as {@code byte[]} arrays. + */ public class TestPositionValueAccess implements PositionValueAccess { private final Map map = new HashMap<>(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java index 89e9a4990..90f979e61 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java @@ -38,6 +38,7 @@ import org.janelia.saalfeldlab.n5.codec.DataCodecInfo; import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo; import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.janelia.saalfeldlab.n5.N5Writer.DataBlockSupplier; import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation; import org.junit.Test; @@ -45,122 +46,97 @@ public class WriteRegionTest { -// #...............................#...............................#...............................#...............................# -// $.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$ -// |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...| -// 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96 - - public static void main(String[] args) { - -// int[] datablockSize = {3, 3, 3}; -// int[] level1ShardSize = {6, 6, 6}; -// int[] level2ShardSize = {24, 24, 24}; - int[] datablockSize = {3}; - int[] level1ShardSize = {6}; - int[] level2ShardSize = {24}; - final long[] datasetDimensions = {96}; + @Test + public void testWriteRegion() { - // DataBlocks are 3x3x3 - // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) - // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) + int[] chunkSize = {3}; + final long[] datasetDimensions = {15}; final BlockCodecInfo c0 = new N5BlockCodecInfo(); - final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, - c0, - new DataCodecInfo[] {new RawCompression()}, - new RawBlockCodecInfo(), - new DataCodecInfo[] {new RawCompression()}, - IndexLocation.END - ); - final ShardCodecInfo c2 = new DefaultShardCodecInfo( - level1ShardSize, - c1, - new DataCodecInfo[] {new RawCompression()}, - new RawBlockCodecInfo(), - new DataCodecInfo[] {new RawCompression()}, - IndexLocation.START - ); TestDatasetAttributes attributes = new TestDatasetAttributes( datasetDimensions, - level2ShardSize, + chunkSize, DataType.INT8, - c2, + c0, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); + DataBlockSupplier chunks = (gridPos, existing) -> { + return createDataBlock(chunkSize, gridPos.clone(), (byte) gridPos[0]); + }; + + DataBlockSupplier chunks255 = (gridPos, existing) -> { + return createDataBlock(chunkSize, gridPos.clone(), (byte)255); + }; + + DataBlockSupplier chunksDelete = (gridPos, existing) -> { + return null; + }; - // --------------------------------------------------------------- - // Some "tests" - // TODO: Turn into unit tests - // --------------------------------------------------------------- - // write some blocks, filled with constant values - final int[] dataBlockSize = c1.getInnerBlockSize(); -// final long[] datasetDimensions = {100, 100, 100}; -// final long[] regionMin = {9,9,9}; -// final long[] regionSize = {15,15,15}; + // write one chunk at grid Position 1 + datasetAccess.writeRegion(store, + new long[] {3}, + new long[] {3}, + chunks, + false); - // #...............................#...............................#...............................#...............................# - // $.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$ - // |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...| - // 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96 + // Chunks + // |...|...|...|...|...| + // Pixels indexes + // 0 3 6 9 12 15- + + checkChunk(datasetAccess.readChunk(store, new long[] {0}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {1}), true, 1); + checkChunk(datasetAccess.readChunk(store, new long[] {2}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {3}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {4}), false, 0); + + // write two chunks at grid positions 2 and 3 + datasetAccess.writeRegion(store, + new long[]{6}, + new long[]{6}, + chunks, + false); - final long[] regionMin = {93}; - final long[] regionSize = {1}; - DataBlockSupplier blocks = (gridPos, existing) -> { -// System.out.println("BlockSupplier.get(" + Arrays.toString(gridPos) + ", " + existing + ")"); - return createDataBlock(dataBlockSize, gridPos.clone(), (byte) gridPos[0]); - }; + checkChunk(datasetAccess.readChunk(store, new long[] {0}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {1}), true, 1); + checkChunk(datasetAccess.readChunk(store, new long[] {2}), true, 2); + checkChunk(datasetAccess.readChunk(store, new long[] {3}), true, 3); + checkChunk(datasetAccess.readChunk(store, new long[] {4}), false, 0); + + // delete two chunks at grid positions 3 and 4 datasetAccess.writeRegion(store, - regionMin, - regionSize, - blocks, + new long[]{9}, + new long[]{6}, + chunksDelete, false); - // verify that the written blocks can be read back with the correct values -// checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), true, 1); -// checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); -// checkBlock(datasetAccess.readBlock(store, new long[] {0, 1, 0}), true, 3); -// checkBlock(datasetAccess.readBlock(store, new long[] {1, 1, 0}), true, 4); -// checkBlock(datasetAccess.readBlock(store, new long[] {3, 2, 1}), true, 5); -// checkBlock(datasetAccess.readBlock(store, new long[] {8, 4, 1}), true, 6); - - // verify that deleting a block removes it from the shard (while other blocks in the same shard are still present) -// datasetAccess.deleteBlock(store, new long[] {0, 0, 0}); -// checkBlock(datasetAccess.readBlock(store, new long[] {0, 0, 0}), false, 1); -// checkBlock(datasetAccess.readBlock(store, new long[] {1, 0, 0}), true, 2); - - // if a shard becomes empty the corresponding key should be deleted -// if ( store.get(new long[] {1, 0, 0}) == null ) { -// throw new IllegalStateException("expected non-null readData"); -// } -// datasetAccess.deleteBlock(store, new long[] {8, 4, 1}); -// if ( store.get(new long[] {1, 0, 0}) != null ) { -// throw new IllegalStateException("expected null readData"); -// } - - // deleting a non-existent block should not fail -// datasetAccess.deleteBlock(store, new long[] {0, 0, 8}); - - System.out.println("all good"); } - + @Test public void testWriteRegionSharded() { - int[] datablockSize = {3}; - int[] level2ShardSize = {24}; + // Shards + // #...............................#...............................#...............................#...............................# + // Chunks + // |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...| + // Pixels indexes + // 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96 + + int[] chunkSize = {3}; + int[] shardSize = {24}; final long[] datasetDimensions = {96}; + int numChunks = (int)(datasetDimensions[0] / chunkSize[0]); + + // Chunks are size 3 + // Shards are size 24 (contain 8 chunks) - // DataBlocks are 3x3x3 - // Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks) - // Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards) final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -170,33 +146,121 @@ public void testWriteRegionSharded() { TestDatasetAttributes attributes = new TestDatasetAttributes( datasetDimensions, - level2ShardSize, + shardSize, DataType.INT8, c1, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); - final int[] dataBlockSize = c1.getInnerBlockSize(); - final long[] regionMin = {93}; - final long[] regionSize = {1}; + DataBlockSupplier chunks = (gridPos, existing) -> { + return createDataBlock(chunkSize, gridPos.clone(), (byte) gridPos[0]); + }; + + DataBlockSupplier chunks255 = (gridPos, existing) -> { + return createDataBlock(chunkSize, gridPos.clone(), (byte)255); + }; - DataBlockSupplier blocks = (gridPos, existing) -> { - return createDataBlock(dataBlockSize, gridPos.clone(), (byte) gridPos[0]); + DataBlockSupplier chunksDelete = (gridPos, existing) -> { + return null; }; + // write one chunk at grid Position 1 + datasetAccess.writeRegion(store, + new long[] {3}, + new long[] {3}, + chunks, + false); + + checkChunk(datasetAccess.readChunk(store, new long[] {0}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {1}), true, 1); + checkChunk(datasetAccess.readChunk(store, new long[] {2}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {3}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[] {4}), false, 0); + + // only the first shard should exist + checkKey(store, new long[]{0}, true); + checkKey(store, new long[]{1}, false); + checkKey(store, new long[]{2}, false); + checkKey(store, new long[]{3}, false); + + // write two chunks at grid positions 2 and 3 + datasetAccess.writeRegion(store, + new long[]{6}, + new long[]{6}, + chunks, + false); + + checkChunk(datasetAccess.readChunk(store, new long[]{0}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[]{1}), true, 1); + checkChunk(datasetAccess.readChunk(store, new long[]{2}), true, 2); + checkChunk(datasetAccess.readChunk(store, new long[]{3}), true, 3); + checkChunk(datasetAccess.readChunk(store, new long[]{4}), false, 0); + + // delete two chunks at grid positions 3 and 4 + datasetAccess.writeRegion(store, + new long[]{9}, + new long[]{6}, + chunksDelete, + false); + + checkChunk(datasetAccess.readChunk(store, new long[]{0}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[]{1}), true, 1); + checkChunk(datasetAccess.readChunk(store, new long[]{2}), true, 2); + checkChunk(datasetAccess.readChunk(store, new long[]{3}), false, 0); + checkChunk(datasetAccess.readChunk(store, new long[]{4}), false, 0); + + // overwrite all chunks to hold 255 + datasetAccess.writeRegion(store, + new long[]{0}, + new long[]{96}, + chunks255, + false); + + for (int i = 0; i < numChunks; i++) { + checkChunk(datasetAccess.readChunk(store, new long[]{i}), true, 255); + } + + // all shards should exist + checkKey(store, new long[]{0}, true); + checkKey(store, new long[]{1}, true); + checkKey(store, new long[]{2}, true); + checkKey(store, new long[]{3}, true); + + // delete some chunks datasetAccess.writeRegion(store, - regionMin, - regionSize, - blocks, + new long[]{18}, + new long[]{18}, + chunksDelete, false); + checkChunk(datasetAccess.readChunk(store, new long[]{5}), true, 255); + checkChunk(datasetAccess.readChunk(store, new long[]{6}), false, 0); + + // all shards should exist + checkKey(store, new long[]{0}, true); + checkKey(store, new long[]{1}, true); + checkKey(store, new long[]{2}, true); + checkKey(store, new long[]{3}, true); + + // delete more chunks so that shard 1 is empty + datasetAccess.writeRegion(store, + new long[]{36}, + new long[]{15}, + chunksDelete, + false); + + // shard 1 should be gone + checkKey(store, new long[]{0}, true); + checkKey(store, new long[]{1}, false); + checkKey(store, new long[]{2}, true); + checkKey(store, new long[]{3}, true); } - private static void checkBlock(final DataBlock dataBlock, final boolean expectedNonNull, final int expectedFillValue) { + private static void checkChunk(final DataBlock chunk, final boolean expectedNonNull, final int expectedFillValue) { - if (dataBlock == null) { + if (chunk == null) { if (expectedNonNull) { throw new IllegalStateException("expected non-null dataBlock"); } @@ -204,7 +268,7 @@ private static void checkBlock(final DataBlock dataBlock, final boolean if (!expectedNonNull) { throw new IllegalStateException("expected null dataBlock"); } - final byte[] bytes = dataBlock.getData(); + final byte[] bytes = chunk.getData(); for (byte b : bytes) { if (b != (byte) expectedFillValue) { throw new IllegalStateException("expected all values to be " + expectedFillValue); @@ -213,6 +277,17 @@ private static void checkBlock(final DataBlock dataBlock, final boolean } } + private static void checkKey(PositionValueAccess pva, long[] position, final boolean expectedNonNull) { + + try (VolatileReadData val = pva.get(position)) { + + if (expectedNonNull && val == null) + throw new IllegalStateException("expected non-null value"); + else if (!expectedNonNull && val != null) + throw new IllegalStateException("expected null value"); + } + } + private static DataBlock createDataBlock(int[] size, long[] gridPosition, int fillValue) { final byte[] bytes = new byte[DataBlock.getNumElements(size)]; Arrays.fill(bytes, (byte) fillValue); @@ -227,12 +302,10 @@ public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType d super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); } - public DatasetAccess datasetAccess() { - - // to make this accessible for the test - return createDatasetAccess(); + @Override // to make this accessible for the test + protected DatasetAccess getDatasetAccess() { + return super.getDatasetAccess(); } - } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java index 5c900a4a6..d03de5a4a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest.java @@ -58,15 +58,15 @@ public class WriteShardTest { public static void main(String[] args) { - final int[] datablockSize = {3}; + final int[] chunkSize = {3}; final int[] level1ShardSize = {6}; final long[] datasetDimensions = {36}; - // DataBlocks are 3 + // Chunks are 3 // Level 1 shards are 6 final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -81,7 +81,7 @@ public static void main(String[] args) { c1, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); // 0 1 2 3 4 5 6 @@ -100,7 +100,7 @@ public static void main(String[] args) { System.out.println("shard.getSize() = " + Arrays.toString(shard.getSize())); System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); System.out.println(); - datasetAccess.writeShard(store, shard, 1); + datasetAccess.writeBlock(store, shard, 1); System.out.println(); System.out.println(); } @@ -110,7 +110,7 @@ public static void main(String[] args) { System.out.println("shard.getSize() = " + Arrays.toString(shard.getSize())); System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); System.out.println(); - datasetAccess.writeShard(store, shard, 1); + datasetAccess.writeBlock(store, shard, 1); System.out.println(); System.out.println(); } @@ -119,19 +119,19 @@ public static void main(String[] args) { System.out.println("all good"); } - + @Test public void testShardDatasetAccess() { - final int[] datablockSize = {3}; + final int[] chunkSize = {3}; final int[] level1ShardSize = {6}; final long[] datasetDimensions = {36}; - // DataBlocks are 3 + // Chunks are 3 // Level 1 shards are 6 final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -146,7 +146,7 @@ public void testShardDatasetAccess() { c1, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); // 0 1 2 3 4 5 6 @@ -163,14 +163,14 @@ public void testShardDatasetAccess() { @Test public void testWriteNullBlockRemovesShard() throws Exception { - final int[] datablockSize = {3}; + final int[] chunkSize = {3}; final int[] level1ShardSize = {6}; final long[] datasetDimensions = {36}; // Level 1 shards have size 6 (each containing two datablocks of size 3) final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[]{new RawCompression()}, new RawBlockCodecInfo(), @@ -184,7 +184,7 @@ public void testWriteNullBlockRemovesShard() throws Exception { c1, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); final long[] shardKey = {1}; @@ -194,58 +194,58 @@ public void testWriteNullBlockRemovesShard() throws Exception { assertFalse("Shard should not exist at the start of the test", store.exists(shardKey)); - // Write a single block at grid position [3] - // This block is in shard [1] - // ( shard 0 contains blocks 0-1, - // shard 1 contains blocks 2-3 ) - final long[] blockGridPosition = {3}; - final DataBlock block = createDataBlock(datablockSize, blockGridPosition, 100); + // Write a single chunk at grid position [3] + // This chunk is in shard [1] + // ( shard 0 contains chunks 0-1, + // shard 1 contains chunks 2-3 ) + final long[] chunkGridPosition = {3}; + final DataBlock chunk = createDataBlock(chunkSize, chunkGridPosition, 100); - datasetAccess.writeBlock(store, block); + datasetAccess.writeChunk(store, chunk); // Verify the shard exists - assertTrue("Shard should exist after writing block", store.exists(shardKey)); + assertTrue("Shard should exist after writing chunk", store.exists(shardKey)); - // Write a null block at the same location using writeRegion - // This should remove the block and delete the now-empty shard - final long[] regionMin = {9}; // pixel position of block [3] + // Write a null chunk at the same location using writeRegion + // This should remove the chunk and delete the now-empty shard + final long[] regionMin = {9}; // pixel position of chunk [3] final long[] regionSize = {3}; // size of one block datasetAccess.writeRegion( store, regionMin, regionSize, - (gridPosition, existingBlock) -> null, // block supplier returns - // null to indicate - // removal + (gridPosition, + // block supplier returns null to indicate removal + existingBlock) -> null, false); // Verify the shard has been removed - assertFalse("Shard should be removed after writing null block", store.exists(shardKey)); - + assertFalse("Shard should be removed after writing null chunk", store.exists(shardKey)); + /** - * THREE BLOCKS, TWO SHARDS + * THREE CHUNKS, TWO SHARDS */ - // Write two blocks into the same shard, and one block into a second shard - // Shard [1] contains blocks [2] and [3] - // Shard [2] contains block [4] + // Write two chunks into the same shard, and one chunk into a second shard + // Shard [1] contains chunks [2] and [3] + // Shard [2] contains chunk [4] final long[] shardKey1 = {1}; final long[] shardKey2 = {2}; - final DataBlock block1 = createDataBlock(datablockSize, new long[]{2}, 100); - final DataBlock block2 = createDataBlock(datablockSize, new long[]{3}, 200); - final DataBlock block3 = createDataBlock(datablockSize, new long[]{4}, 300); + final DataBlock chunk1 = createDataBlock(chunkSize, new long[]{2}, 100); + final DataBlock chunk2 = createDataBlock(chunkSize, new long[]{3}, 200); + final DataBlock chunk3 = createDataBlock(chunkSize, new long[]{4}, 300); assertFalse("Shard should not exist at the start of the test", store.exists(shardKey1)); assertFalse("Shard should not exist at the start of the test", store.exists(shardKey2)); // write blocks - datasetAccess.writeBlocks(store, Streams.of(block1, block2, block3).collect(Collectors.toList())); + datasetAccess.writeChunks(store, Streams.of(chunk1, chunk2, chunk3).collect(Collectors.toList())); // Verify the shard exists - assertTrue("Shard should exist after writing blocks", store.exists(shardKey1)); - assertTrue("Shard should exist after writing blocks", store.exists(shardKey2)); + assertTrue("Shard should exist after writing chunks", store.exists(shardKey1)); + assertTrue("Shard should exist after writing chunks", store.exists(shardKey2)); // Write a null block at block [3]'s location datasetAccess.writeRegion( @@ -255,26 +255,26 @@ public void testWriteNullBlockRemovesShard() throws Exception { (gridPosition, existingBlock) -> null, false); - // Verify the shard still exists (because block [2] is still there) - assertTrue("Shard should still exist because it contains another block", store.exists(shardKey1)); + // Verify the shard still exists (because chunk [2] is still there) + assertTrue("Shard should still exist because it contains another chunk", store.exists(shardKey1)); assertTrue("Shard should still exist because was not affected", store.exists(shardKey2)); - // Verify we can still read block [2] - final DataBlock readBlock = datasetAccess.readBlock(store, new long[]{2}); - assertTrue("Block [2] should still be readable", readBlock != null); - assertTrue("Block [2] data should match", Arrays.equals(block1.getData(), readBlock.getData())); + // Verify we can still read chunk [2] + final DataBlock readChunk = datasetAccess.readChunk(store, new long[]{2}); + assertTrue("Block [2] should still be readable", readChunk != null); + assertTrue("Block [2] data should match", Arrays.equals(chunk1.getData(), readChunk.getData())); - // Verify block [3] is gone - final DataBlock readBlock2 = datasetAccess.readBlock(store, new long[]{3}); - assertNull("Block [3] should be null after removal", readBlock2); + // Verify chunk [3] is gone + final DataBlock readChunk2 = datasetAccess.readChunk(store, new long[]{3}); + assertNull("Block [3] should be null after removal", readChunk2); - // Verify block [4] exists - final DataBlock readBlock3 = datasetAccess.readBlock(store, new long[]{4}); - assertTrue("Block [4] should still be readable", readBlock3 != null); - assertTrue("Block [4] data should match", Arrays.equals(block3.getData(), readBlock3.getData())); + // Verify chunk [4] exists + final DataBlock readChunk3 = datasetAccess.readChunk(store, new long[]{4}); + assertTrue("Block [4] should still be readable", readChunk3 != null); + assertTrue("Block [4] data should match", Arrays.equals(chunk3.getData(), readChunk3.getData())); - // Write a null block at block [2]'s location - final long[] regionMin2 = {6}; // pixel position of block [3] + // Write a null chunk at chunk [2]'s location + final long[] regionMin2 = {6}; // pixel position of chunk [3] final long[] regionSize2 = {3}; datasetAccess.writeRegion( @@ -284,7 +284,7 @@ public void testWriteNullBlockRemovesShard() throws Exception { (gridPosition, existingBlock) -> null, false); - assertFalse("Shard should not exist after deleting all blocks", store.exists(shardKey1)); + assertFalse("Shard should not exist after deleting all chunks", store.exists(shardKey1)); assertTrue("Shard should still exist because was not affected", store.exists(shardKey2)); } @@ -302,10 +302,9 @@ public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType d super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); } - public DatasetAccess datasetAccess() { - - // to make this accessible for the test - return createDatasetAccess(); + @Override // to make this accessible for the test + protected DatasetAccess getDatasetAccess() { + return super.getDatasetAccess(); } } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java index 86c0308a2..be1d973dd 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTest2.java @@ -50,15 +50,15 @@ public class WriteShardTest2 { public static void main(String[] args) { - final int[] datablockSize = {3,3}; + final int[] chunkSize = {3,3}; final int[] level1ShardSize = {6,6}; final long[] datasetDimensions = {36, 18}; - // DataBlocks are 3 - // Level 1 shards are 6 + // Chunks are 3x3 + // Level 1 shards are 6x6 final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -73,7 +73,7 @@ public static void main(String[] args) { c1, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); // 0 1 2 3 4 5 6 @@ -90,9 +90,9 @@ public static void main(String[] args) { System.out.println("shard.getGridPosition() = " + Arrays.toString(shard.getGridPosition())); System.out.println("shard.getSize() = " + Arrays.toString(shard.getSize())); System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); - datasetAccess.writeShard(store, shard, 1); + datasetAccess.writeBlock(store, shard, 1); - // we should get these DataBlock values + // we should get these Chunk values // 1, 2, 3, | 4, 5, 6, // 7, 8, 9, | 10, 11, 12, @@ -103,15 +103,15 @@ public static void main(String[] args) { // 31, 32, 33, | 34, 35, 36 System.out.println("{4, 2}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {4, 2}).getData())); + datasetAccess.readChunk(store, new long[] {4, 2}).getData())); System.out.println("{5, 2}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {5, 2}).getData())); + datasetAccess.readChunk(store, new long[] {5, 2}).getData())); System.out.println("{4, 3}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {4, 3}).getData())); + datasetAccess.readChunk(store, new long[] {4, 3}).getData())); System.out.println("{5, 3}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {5, 3}).getData())); + datasetAccess.readChunk(store, new long[] {5, 3}).getData())); - final DataBlock readShard = datasetAccess.readShard(store, new long[] {2, 1}, 1); + final DataBlock readShard = datasetAccess.readBlock(store, new long[] {2, 1}, 1); System.out.println("readShard.getData() = " + Arrays.toString(readShard.getData())); } @@ -129,10 +129,9 @@ public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType d super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); } - public DatasetAccess datasetAccess() { - - // to make this accessible for the test - return createDatasetAccess(); + @Override // to make this accessible for the test + protected DatasetAccess getDatasetAccess() { + return super.getDatasetAccess(); } } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java index 87737668f..d3e537a63 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/WriteShardTestTruncate.java @@ -50,15 +50,15 @@ public class WriteShardTestTruncate { public static void main(String[] args) { - final int[] datablockSize = {3,3}; + final int[] chunkSize = {3,3}; final int[] level1ShardSize = {6,6}; final long[] datasetDimensions = {34, 17}; // 6 x 3 shards, border (2, 1) - // DataBlocks are 3 - // Level 1 shards are 6 + // Chunks are 3x3 + // Level 1 shards are 6x6 final BlockCodecInfo c0 = new N5BlockCodecInfo(); final ShardCodecInfo c1 = new DefaultShardCodecInfo( - datablockSize, + chunkSize, c0, new DataCodecInfo[] {new RawCompression()}, new RawBlockCodecInfo(), @@ -73,7 +73,7 @@ public static void main(String[] args) { c1, new RawCompression()); - final DatasetAccess datasetAccess = attributes.datasetAccess(); + final DatasetAccess datasetAccess = attributes.getDatasetAccess(); final PositionValueAccess store = new TestPositionValueAccess(); // 0 1 2 3 4 5 6 @@ -90,9 +90,9 @@ public static void main(String[] args) { System.out.println("shard.getGridPosition() = " + Arrays.toString(shard.getGridPosition())); System.out.println("shard.getSize() = " + Arrays.toString(shard.getSize())); System.out.println("shard.getData() = " + Arrays.toString(shard.getData())); - datasetAccess.writeShard(store, shard, 1); + datasetAccess.writeBlock(store, shard, 1); - // we should get these DataBlock values + // we should get these chunk values // 1, 2, 3, | 4, 5, 6, // 7, 8, 9, | 10, 11, 12, @@ -103,13 +103,13 @@ public static void main(String[] args) { // 31, 32, 33, | 34, 35, 36 System.out.println("{10, 4}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {10, 4}).getData())); + datasetAccess.readChunk(store, new long[] {10, 4}).getData())); System.out.println("{11, 4}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {11, 4}).getData())); + datasetAccess.readChunk(store, new long[] {11, 4}).getData())); System.out.println("{10, 5}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {10, 5}).getData())); + datasetAccess.readChunk(store, new long[] {10, 5}).getData())); System.out.println("{11, 5}.data = " + Arrays.toString( - datasetAccess.readBlock(store, new long[] {11, 5}).getData())); + datasetAccess.readChunk(store, new long[] {11, 5}).getData())); // 1, 2, 3, | 4 @@ -119,7 +119,7 @@ public static void main(String[] args) { // 19, 20, 21, | 22 // 25, 26, 27, | 28 - final DataBlock readShard = datasetAccess.readShard(store, new long[] {5, 2}, 1); + final DataBlock readShard = datasetAccess.readBlock(store, new long[] {5, 2}, 1); System.out.println("readShard.getData() = " + Arrays.toString(readShard.getData())); } @@ -137,10 +137,9 @@ public TestDatasetAttributes(long[] dimensions, int[] outerBlockSize, DataType d super(dimensions, outerBlockSize, dataType, blockCodecInfo, dataCodecInfos); } - public DatasetAccess datasetAccess() { - - // to make this accessible for the test - return createDatasetAccess(); + @Override // to make this accessible for the test + protected DatasetAccess getDatasetAccess() { + return super.getDatasetAccess(); } } }