diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java
index eff501ba2..06e753366 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java
@@ -28,6 +28,8 @@
*/
package org.janelia.saalfeldlab.n5.readdata;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Comparator;
/**
@@ -61,7 +63,7 @@ default long end() {
}
static boolean equals(final Range r0, final Range r1) {
- if (r0 == null && r1==null) {
+ if (r0 == null && r1 == null) {
return true;
} else if (r0 == null || r1 == null) {
return false;
@@ -119,4 +121,45 @@ public String toString() {
return new DefaultRange(offset, length);
}
+
+ /**
+ * Returns a potentially new collection of {@link Range}s such that
+ * adjacent or overlapping Ranges are combined.
+ *
+ * If the input ranges are non-adjacent the input instance is returned, but if aggregation
+ * occurs, the result will be a new, sorted List.
+ *
+ * @param ranges
+ * a collection of Ranges
+ *
+ * @return collection with adjacent or overlapping ranges merged
+ */
+ static Collection extends Range> aggregate(final Collection extends Range> ranges) {
+
+ if (ranges.size() == 0)
+ return ranges;
+
+ final ArrayList sortedRanges = new ArrayList<>(ranges);
+ sortedRanges.sort(Range.COMPARATOR);
+
+ final ArrayList result = new ArrayList<>();
+ Range lo = null;
+ for (Range hi : sortedRanges) {
+
+ if (lo == null)
+ lo = hi;
+ else if (lo.end() >= hi.offset()) {
+ // merge
+ lo = Range.at(lo.offset(), Math.max(lo.end(), hi.end()) - lo.offset());
+ } else {
+ result.add(lo);
+ lo = hi;
+ }
+ }
+ result.add(lo);
+
+ final boolean wereMerges = ranges.size() != result.size();
+ return wereMerges ? result : ranges;
+ }
+
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java
index 122a399ef..25da8b0b8 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java
@@ -1,7 +1,7 @@
package org.janelia.saalfeldlab.n5.readdata;
-import java.io.InputStream;
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.readdata.prefetch.AggregatingPrefetchLazyRead;
/**
* During its life-time, the content of a {@code VolatileReadData} should not be
@@ -29,7 +29,8 @@ public interface VolatileReadData extends ReadData, AutoCloseable {
* @return a new VolatileReadData
*/
static VolatileReadData from(final LazyRead lazyRead) {
- return new LazyReadData(lazyRead);
+ final LazyRead aggregatingLazyRead = new AggregatingPrefetchLazyRead(lazyRead);
+ return new LazyReadData(aggregatingLazyRead);
}
}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java
new file mode 100644
index 000000000..dcc0097a1
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java
@@ -0,0 +1,44 @@
+package org.janelia.saalfeldlab.n5.readdata.prefetch;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import java.util.List;
+import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.readdata.LazyRead;
+import org.janelia.saalfeldlab.n5.readdata.Range;
+
+/**
+ * A {@link SliceTrackingLazyRead} that implements {@link #prefetch} to
+ * aggregate overlapping / adjacent ranges and then materialize each aggregated
+ * range.
+ */
+public class AggregatingPrefetchLazyRead extends SliceTrackingLazyRead {
+
+ public AggregatingPrefetchLazyRead(final LazyRead delegate) {
+ super(delegate);
+ }
+
+ /**
+ * Indicates that the given slices will be subsequently read.
+ *
+ * This implementation groups overlapping / adjacent {@link Range}s into single read requests.
+ *
+ * @param ranges
+ * slice ranges to prefetch
+ *
+ * @throws N5IOException
+ * if any I/O error occurs
+ */
+ @Override
+ public void prefetch(final Collection extends Range> ranges) throws N5IOException {
+
+ final List filteredRanges = new ArrayList<>(ranges);
+ filteredRanges.removeIf(this::isCovered);
+ final Collection extends Range> aggregatedRanges = Range.aggregate(filteredRanges);
+ for (final Range slice : aggregatedRanges) {
+ materialize(slice.offset(), slice.length());
+ }
+ }
+
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/EnclosingPrefetchLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/EnclosingPrefetchLazyRead.java
new file mode 100644
index 000000000..b0d6fdc7a
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/EnclosingPrefetchLazyRead.java
@@ -0,0 +1,49 @@
+package org.janelia.saalfeldlab.n5.readdata.prefetch;
+
+import java.util.Collection;
+import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.readdata.LazyRead;
+import org.janelia.saalfeldlab.n5.readdata.Range;
+
+/**
+ * A {@link SliceTrackingLazyRead} that implements {@link #prefetch} to
+ * materialize the bounding range of all requested ranges.
+ */
+public class EnclosingPrefetchLazyRead extends SliceTrackingLazyRead {
+
+ public EnclosingPrefetchLazyRead(final LazyRead delegate) {
+ super(delegate);
+ }
+
+ /**
+ * Indicates that the given slices will be subsequently read.
+ * {@code LazyRead} implementations (optionally) may take steps to prepare
+ * for these subsequent slices.
+ *
+ * Minimal implementation: Find offset and length covering all ranges that
+ * are not yet fully covered by existing slices. Then materialize the slice
+ * covering that range.
+ *
+ * @param ranges
+ * slice ranges to prefetch
+ *
+ * @throws N5IOException
+ * if any I/O error occurs
+ */
+ @Override
+ public void prefetch(final Collection extends Range> ranges) throws N5IOException {
+
+ long fromIndex = Long.MAX_VALUE;
+ long toIndex = Long.MIN_VALUE;
+ for (final Range slice : ranges) {
+ if (!isCovered(slice)) {
+ fromIndex = Math.min(fromIndex, slice.offset());
+ toIndex = Math.max(toIndex, slice.end());
+ }
+ }
+
+ if (fromIndex < toIndex) {
+ materialize(fromIndex, toIndex - fromIndex);
+ }
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java
new file mode 100644
index 000000000..5835c473e
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java
@@ -0,0 +1,90 @@
+package org.janelia.saalfeldlab.n5.readdata.prefetch;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.readdata.LazyRead;
+import org.janelia.saalfeldlab.n5.readdata.Range;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+
+/**
+ * A {@link LazyRead} that wraps a delegate {@code LazyRead} and keeps track of
+ * all slices that have been {@link #materialize materialized}.
+ *
+ * When materializing a new slice, we first check whether it is completely
+ * covered by a materialized slice that we already track. If so, then we just
+ * return a slice on the existing materialized slice. If not, we materialize the
+ * slice from the delegate track it.
+ */
+public class SliceTrackingLazyRead implements LazyRead {
+
+ protected static class Slice implements Range {
+
+ // Offset and length in the delegate
+ private final long offset;
+ private final long length;
+
+ // Data of this slice
+ private final ReadData data;
+
+ Slice(final long offset, final long length, final ReadData data) {
+ this.offset = offset;
+ this.length = length;
+ this.data = data;
+ }
+
+ @Override
+ public long offset() {
+ return offset;
+ }
+
+ @Override
+ public long length() {
+ return length;
+ }
+
+ @Override
+ public String toString() {
+ return "{" + offset + ", " + length + '}';
+ }
+ }
+
+ protected final List slices = new ArrayList<>();
+
+ /**
+ * The {@code LazyRead} providing our data.
+ */
+ private final LazyRead delegate;
+
+ public SliceTrackingLazyRead(final LazyRead delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+
+ @Override
+ public ReadData materialize(final long offset, final long length) throws N5IOException {
+ final Slice containing = Slices.findContainingSlice(slices, offset, length);
+ if (containing != null) {
+ return containing.data.slice(offset - containing.offset, length);
+ } else {
+ final ReadData data = delegate.materialize(offset, length);
+ Slices.addSlice(slices, new Slice(offset, length, data));
+ return data;
+ }
+ }
+
+ @Override
+ public long size() throws N5IOException {
+ return delegate.size();
+ }
+
+ protected boolean isCovered(final Range slice) {
+
+ return Slices.findContainingSlice(slices, slice) != null;
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java
new file mode 100644
index 000000000..d3d0dfb11
--- /dev/null
+++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java
@@ -0,0 +1,116 @@
+package org.janelia.saalfeldlab.n5.readdata.prefetch;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import org.janelia.saalfeldlab.n5.readdata.Range;
+
+class Slices {
+
+ private Slices() {
+ // utility class. should not be instantiated.
+ }
+
+ /**
+ * In an ordered list of {@code slices}, find a slice that completely contains the given range.
+ *
+ * Pre-conditions:
+ *
+ * - Slices are ordered by offset.
+ * - If two slices overlap, no slice is fully contained within the other.
+ * (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
+ *
+ *
+ * @param slices
+ * ordered list of slices
+ * @param offset
+ * start of the range to cover
+ * @param length
+ * length of the range to cover
+ *
+ * @return a slice that completely contains the requested range, or {@code null} if no such slice exists
+ */
+ static T findContainingSlice(final List slices, final long offset, final long length) {
+
+ // Find the slice with the largest slice.offset <= offset.
+ final int i = Collections.binarySearch(slices, Range.at(offset, 0), Comparator.comparingLong(Range::offset));
+
+ // Largest index of a slice with slice.offset <= offset.
+ final int index = i < 0 ? -i - 2 : i;
+ if (index < 0) {
+ // We find no overlapping slice, because
+ // slices[0].offset is already too large.
+ return null;
+ }
+
+ final T slice = slices.get(index);
+ if (slice.end() < offset + length) {
+ return null;
+ }
+
+ return slice;
+ }
+
+ /**
+ * In an ordered list of {@code slices}, find a slice that completely contains the given range.
+ *
+ * Pre-conditions:
+ *
+ * - Slices are ordered by offset.
+ * - If two slices overlap, no slice is fully contained within the other.
+ * (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
+ *
+ *
+ * @param slices
+ * ordered list of slices
+ * @param range
+ * range to cover
+ *
+ * @return a slice that completely contains the requested range, or {@code null} if no such slice exists
+ */
+ static T findContainingSlice(final List slices, final Range range) {
+ return findContainingSlice(slices, range.offset(), range.length());
+ }
+
+ /**
+ * Add a new {@code slice} to the {@code slice} list.
+ *
+ * Note, that the new {@code slice} is expected to not be fully contained in
+ * an existing slice!
+ *
+ * Pre/post-conditions:
+ *
+ * - Slices are ordered by offset.
+ * - If two slices overlap, no slice is fully contained within the other.
+ * (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
+ *
+ *
+ * The new {@code slice} will be inserted into the list at the correct position
+ * (such that {@code slices} remains ordered by slice offset), and all existing
+ * slices that are fully contained in the new {@code slice} will be removed.
+ *
+ * @param slices
+ * ordered list of slices
+ * @param slice
+ * slice to be inserted
+ */
+ static void addSlice(final List slices, final T slice) {
+
+ final int i = Collections.binarySearch(slices, slice, Comparator.comparingLong(Range::offset));
+ final int from = i < 0 ? -i - 1 : i;
+
+ int to = from;
+ while (to < slices.size() && slices.get(to).end() <= slice.end()) {
+ ++to;
+ }
+
+ if (from == to) {
+ // empty range: just insert
+ slices.add(from, slice);
+ } else {
+ // overwrite the first element in range, remove the rest
+ slices.set(from, slice);
+ slices.subList(from + 1, to).clear();
+ }
+ }
+}
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java
index 630c30033..66e0dd548 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java
@@ -155,6 +155,15 @@ private void readChunksRecursive(
// TODO: collect all the elementPos that we will need and prefetch
// Probably best to add a prefetch method to RawShard?
+ // Here's an attempt at that.
+ // Don't love that we have to build a list of positions
+ // Consider making DataBlockRequest package private and passing them directly
+ final ArrayList positions = new ArrayList<>();
+ for (final ChunkRequest request : requests) {
+ positions.add(request.position.relative(0));
+ }
+ shard.prefetch(positions);
+
for (final ChunkRequest request : requests) {
final long[] elementPos = request.position.relative(0);
final ReadData elementData = shard.getElementData(elementPos);
@@ -1015,7 +1024,7 @@ public List> chunks(final List> duplicates) {
* Construct {@code ChunkRequests} from a list of level-0 grid positions
* for reading.
*
- * The nesting level ot the returned {@code ChunkRequests} is {@code
+ * The nesting level of the returned {@code ChunkRequests} is {@code
* grid.numLevels()}, that is level of the highest-order shard + 1. This
* implies that the requests are not guaranteed to be in the same shard (at
* any level. {@link ChunkRequests#split() Splitting} the {@code
diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java
index 01d7e961b..419ee1ccb 100644
--- a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java
+++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java
@@ -28,6 +28,11 @@
*/
package org.janelia.saalfeldlab.n5.shard;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.janelia.saalfeldlab.n5.readdata.Range;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.readdata.segment.Segment;
import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData;
@@ -86,4 +91,16 @@ public void setElementData(final ReadData data, final long[] pos) {
final Segment segment = data == null ? null : SegmentedReadData.wrap(data).segments().get(0);
index.set(segment, pos);
}
+
+ public void prefetch(List positions) {
+
+ final List ranges = new ArrayList<>(positions.size());
+ for (long[] pos : positions) {
+ final Segment seg = index.get(pos);
+ if (seg != null)
+ ranges.add(sourceData.location(seg));
+ }
+ sourceData.prefetch(ranges);
+ }
+
}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java
index e5a705422..17d421750 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java
@@ -35,7 +35,6 @@
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
import java.net.URI;
import java.nio.file.Files;
import java.util.Arrays;
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java
index 4ce148c50..73b6a2f46 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java
@@ -29,8 +29,6 @@
package org.janelia.saalfeldlab.n5.benchmarks;
import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
@@ -40,7 +38,6 @@
import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess;
import org.janelia.saalfeldlab.n5.KeyValueAccess;
-import org.janelia.saalfeldlab.n5.LockedChannel;
import org.janelia.saalfeldlab.n5.N5Exception;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java
index 8572e1cee..fd151a265 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java
@@ -30,17 +30,14 @@
import org.apache.commons.io.IOUtils;
import org.janelia.saalfeldlab.n5.HttpKeyValueAccess;
-import org.janelia.saalfeldlab.n5.LockedChannel;
import org.janelia.saalfeldlab.n5.N5Exception;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
import org.junit.Test;
import java.io.IOException;
-import java.io.InputStream;
import java.net.URI;
import java.nio.charset.Charset;
-import java.util.function.Function;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java
index 0c4f4a876..a2e18b17d 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java
@@ -5,13 +5,14 @@
import org.janelia.saalfeldlab.n5.readdata.LazyRead;
import org.janelia.saalfeldlab.n5.readdata.ReadData;
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
-import org.janelia.saalfeldlab.n5.shard.ShardTest;
+import org.janelia.saalfeldlab.n5.readdata.prefetch.AggregatingPrefetchLazyRead;
public class TrackingKeyValueAccess extends DelegateKeyValueAccess {
public int numMaterializeCalls = 0;
public int numIsFileCalls = 0;
public long totalBytesRead = 0;
+ public boolean aggregate = false;
public TrackingKeyValueAccess(final KeyValueAccess kva) {
super(kva);
@@ -25,15 +26,20 @@ public boolean isFile(String normalPath) {
@Override
public VolatileReadData createReadData(final String normalPath) {
-// throw new N5NoSuchKeyException("Test No Such Key");
- return VolatileReadData.from(new TrackingVolatileReadData(kva.createReadData(normalPath)));
+
+ final VolatileReadData volatileReadData = kva.createReadData(normalPath);
+ final TrackingLazyRead trackingLazyRead = new TrackingLazyRead(volatileReadData);
+ LazyRead lazyRead = trackingLazyRead;
+ if (aggregate)
+ lazyRead = new AggregatingPrefetchLazyRead(trackingLazyRead);
+ return VolatileReadData.from( lazyRead );
}
- private class TrackingVolatileReadData implements LazyRead {
+ private class TrackingLazyRead implements LazyRead {
private final VolatileReadData readData;
- TrackingVolatileReadData(final VolatileReadData readData) {
+ TrackingLazyRead(final VolatileReadData readData) {
this.readData = readData;
}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java
new file mode 100644
index 000000000..3499220b9
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java
@@ -0,0 +1,58 @@
+package org.janelia.saalfeldlab.n5.readdata;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+public class RangeTests {
+
+ @Test
+ public void testAggregate() {
+
+ List nonOverlapping = Stream.of(Range.at(0, 5), Range.at(12, 2)).collect(Collectors.toList());
+ assertSame(nonOverlapping, Range.aggregate(nonOverlapping));
+
+ List nonOverlappingRev = Stream.of(Range.at(12, 2), Range.at(0, 5)).collect(Collectors.toList());
+ assertSame(nonOverlappingRev, Range.aggregate(nonOverlappingRev));
+
+ /**
+ * 0 1 2 3 4 5
+ * x x x x x -
+ * - x - - - -
+ */
+ List containing = Stream.of(Range.at(0, 5), Range.at(1, 1)).collect(Collectors.toList());
+ assertEquals(Collections.singletonList(Range.at(0, 5)), Range.aggregate(containing));
+
+ /**
+ * 0 1 2 3 4 5 6
+ * x x x x x - -
+ * - - x x x x x
+ */
+ List overlapping = Stream.of(Range.at(0, 5), Range.at(2, 5)).collect(Collectors.toList());
+ assertEquals(Collections.singletonList(Range.at(0, 7)), Range.aggregate(overlapping));
+
+ /**
+ * 0 1 2 3 4 5
+ * x x x x x -
+ * - - - - - x
+ */
+ List adjacent = Stream.of(Range.at(0, 5), Range.at(5, 1)).collect(Collectors.toList());
+ assertEquals(Collections.singletonList(Range.at(0, 6)), Range.aggregate(adjacent));
+
+ /**
+ * 0 1 2 3 4 5
+ * - - - x x -
+ * x x - - - -
+ * - - - - - x
+ */
+ List three = Stream.of(Range.at(3, 2), Range.at(0, 2), Range.at(5, 1)).collect(Collectors.toList());
+ assertEquals(Stream.of(Range.at(0, 2), Range.at(3, 3)).collect(Collectors.toList()), Range.aggregate(three));
+ }
+
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java
index c4f3fa51b..78522ff3e 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java
@@ -38,7 +38,6 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.nio.file.FileSystems;
import java.util.Arrays;
import java.util.function.IntUnaryOperator;
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java
new file mode 100644
index 000000000..fcac9a5e3
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java
@@ -0,0 +1,233 @@
+package org.janelia.saalfeldlab.n5.readdata.prefetch;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
+import org.janelia.saalfeldlab.n5.readdata.LazyRead;
+import org.janelia.saalfeldlab.n5.readdata.Range;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+import org.junit.Test;
+
+public class SliceTrackingLazyReadTests {
+
+ @Test
+ public void testDefaultSliceTracking() throws N5IOException {
+
+ /**
+ * 1. Create sample ReadData from byte[]
+ * 2. Create a DummyLazy Read
+ * 3. Make a DefaultSliceTrackingLazyRead
+ * 4. Create a list of ranges
+ * 5. Call prefetch
+ * 6. Ensure the correct number of materialize calls were made (always 1 for DefaultSliceTrackingLazyRead)
+ * 7. Verify the stored slices contain the correct range
+ */
+
+ // 1-2. Create a DummyLazyRead with 64 bytes
+ DummyLazyRead dummyLazyRead = createDummyLazyRead(64);
+
+ // 3. Make a testable DefaultSliceTrackingLazyRead
+ TestableDefaultSliceTracker sliceTracking = new TestableDefaultSliceTracker(dummyLazyRead);
+
+ // 4. Create a list of ranges (two non-overlapping ranges with a gap)
+ List ranges = Arrays.asList(
+ Range.at(10, 5), // offset 10, length 5 (bytes 10-14)
+ Range.at(50, 10) // offset 50, length 10 (bytes 50-59)
+ );
+
+ // 5. Call prefetch
+ sliceTracking.prefetch(ranges);
+
+ // 6. Ensure exactly 1 materialize call was made
+ // DefaultSliceTrackingLazyRead creates a single large slice covering all ranges
+ assertEquals("DefaultSliceTrackingLazyRead should make exactly 1 materialize call",
+ 1, dummyLazyRead.getNumMaterializeCalls());
+
+ // 7. Verify the stored slice covers the entire range from offset 10 with length 50
+ assertStoredSlices(sliceTracking, Arrays.asList(
+ Range.at(10, 50) // Single slice covering offset 10-59
+ ));
+
+ }
+
+ @Test
+ public void testAggregatingSliceTracking() throws N5IOException {
+
+ /**
+ * 1. Create sample ReadData from byte[]
+ * 2. Create a DummyLazyRead
+ * 3. Make an AggregatingSliceTrackingLazyRead
+ * 4. Create a list of ranges
+ * 5. Call prefetch
+ * 6. Ensure the correct number of materialize calls were made (one per aggregated range)
+ * 7. Verify the stored slices contain the correct ranges
+ */
+
+ // 1-2. Create a DummyLazyRead with 64 bytes
+ DummyLazyRead dummyLazyRead = createDummyLazyRead(64);
+
+ // 3. Make a testable AggregatingSliceTrackingLazyRead
+ TestableAggregatingSliceTracker sliceTracking = new TestableAggregatingSliceTracker(dummyLazyRead);
+
+ /*
+ * Non-adjacent ranges
+ */
+ // 4. Create a list of ranges (two non-overlapping ranges with a gap)
+ List ranges = Arrays.asList(
+ Range.at(10, 5), // offset 10, length 5 (bytes 10-14)
+ Range.at(50, 10) // offset 50, length 10 (bytes 50-59)
+ );
+
+ // 5. Call prefetch
+ sliceTracking.prefetch(ranges);
+
+ // 6. Ensure exactly 2 materialize calls were made
+ // AggregatingSliceTrackingLazyRead aggregates overlapping/adjacent ranges
+ // Since these ranges are not adjacent or overlapping, it makes 2 separate calls
+ assertEquals("AggregatingSliceTrackingLazyRead should make 2 materialize calls for non-adjacent ranges",
+ 2, dummyLazyRead.getNumMaterializeCalls());
+
+ // 7. Verify the stored slices contain two separate ranges
+ assertStoredSlices(sliceTracking, Arrays.asList(
+ Range.at(10, 5), // First slice
+ Range.at(50, 10) // Second slice
+ ));
+
+ /*
+ * Adjacent ranges
+ */
+
+ // new sliceTracking instance to clear slices
+ sliceTracking = new TestableAggregatingSliceTracker(dummyLazyRead);
+ dummyLazyRead.resetNumMaterializeCalls();
+
+ // 4. Create a list of three contiguous ranges
+ List adjacentRanges = Arrays.asList(
+ Range.at(10, 5), // offset 10, length 5 (bytes 10-14)
+ Range.at(15, 10), // offset 15, length 10 (bytes 15-24)
+ Range.at(25, 5) // offset 25, length 5 (bytes 25-29)
+ );
+
+ // 5. Call prefetch
+ sliceTracking.prefetch(adjacentRanges);
+
+ // 6. Ensure exactly 1 materialize call was made
+ // AggregatingSliceTrackingLazyRead should aggregate these three contiguous ranges
+ // into a single range from offset 10 to 30, with length 20
+ assertEquals("AggregatingSliceTrackingLazyRead should make 1 materialize call for contiguous ranges",
+ 1, dummyLazyRead.getNumMaterializeCalls());
+
+ // 7. Verify the stored slices now contain three ranges total:
+ // the two from the first prefetch plus one aggregated range from the second prefetch
+ assertStoredSlices(sliceTracking, Arrays.asList(
+ Range.at(10, 20) // Aggregated range
+ ));
+
+ }
+
+ private static DummyLazyRead createDummyLazyRead(int size) {
+ byte[] data = new byte[size];
+ for (int i = 0; i < data.length; i++) {
+ data[i] = (byte)i;
+ }
+ return new DummyLazyRead(ReadData.from(data));
+ }
+
+ /**
+ * Helper method to verify that stored slices match expected ranges.
+ *
+ * @param sliceTracking the SliceTrackingLazyRead instance
+ * @param expectedRanges the expected ranges stored in slices
+ */
+ private static void assertStoredSlices(TestableSliceTracker sliceTracking, List expectedRanges) {
+ // Access protected slices field via a test helper
+ List actualSlices = sliceTracking.getSlices();
+
+ assertEquals("Number of stored slices should match", expectedRanges.size(), actualSlices.size());
+
+ for (int i = 0; i < expectedRanges.size(); i++) {
+ Range expected = expectedRanges.get(i);
+ Range actual = actualSlices.get(i);
+ assertEquals("Slice " + i + " offset should match", expected.offset(), actual.offset());
+ assertEquals("Slice " + i + " length should match", expected.length(), actual.length());
+ }
+ }
+
+ /**
+ * Testable wrapper for DefaultSliceTrackingLazyRead that exposes slices.
+ */
+ static class TestableDefaultSliceTracker extends EnclosingPrefetchLazyRead implements TestableSliceTracker {
+ public TestableDefaultSliceTracker(LazyRead delegate) {
+ super(delegate);
+ }
+
+ @Override
+ public List getSlices() {
+ return java.util.Collections.unmodifiableList(slices);
+ }
+ }
+
+ /**
+ * Testable wrapper for AggregatingSliceTrackingLazyRead that exposes slices.
+ */
+ static class TestableAggregatingSliceTracker extends AggregatingPrefetchLazyRead implements TestableSliceTracker {
+ public TestableAggregatingSliceTracker(LazyRead delegate) {
+ super(delegate);
+ }
+
+ @Override
+ public List getSlices() {
+ return java.util.Collections.unmodifiableList(slices);
+ }
+ }
+
+ /**
+ * Interface for testable slice trackers.
+ */
+ interface TestableSliceTracker {
+ List getSlices();
+ }
+
+ static class DummyLazyRead implements LazyRead {
+
+ private ReadData data;
+ private int numMaterializeCalls = 0;
+
+ public DummyLazyRead( ReadData data ) {
+ this.data = data;
+ }
+
+ @Override
+ public void close() throws IOException {
+ // no op
+ }
+
+ @Override
+ public ReadData materialize(long offset, long length) throws N5IOException {
+
+ numMaterializeCalls++;
+ return data.slice(offset, length).materialize();
+ }
+
+ @Override
+ public long size() throws N5IOException {
+
+ return data.length();
+ }
+
+ public int getNumMaterializeCalls() {
+
+ return numMaterializeCalls;
+ }
+
+ public void resetNumMaterializeCalls() {
+
+ numMaterializeCalls = 0;
+ }
+
+ }
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java
new file mode 100644
index 000000000..0ce2b7442
--- /dev/null
+++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java
@@ -0,0 +1,134 @@
+package org.janelia.saalfeldlab.n5.readdata.prefetch;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.janelia.saalfeldlab.n5.readdata.Range;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class SlicesTest {
+
+ private List createSlices(final long[] offsets, final long[] lengths) {
+ final List slices = new ArrayList<>();
+ for (int i = 0; i < offsets.length; ++i) {
+ slices.add(Range.at(offsets[i], lengths[i]));
+ }
+ return slices;
+ }
+
+ @Test
+ public void testFindContaining() {
+
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (2,6) [-----------]
+ // (6,4) [---------]
+ // (8,6) [-----------]
+
+ final List slices = createSlices(
+ new long[] {2, 6, 8},
+ new long[] {6, 4, 6});
+ Range slice;
+
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (1,1) [-]
+ slice = Slices.findContainingSlice(slices, 1, 1);
+ assertEquals(null, slice);
+
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (2,1) [-]
+ slice = Slices.findContainingSlice(slices, 2, 1);
+ assertEquals(2, slice.offset());
+
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (2,6) [-----------]
+ slice = Slices.findContainingSlice(slices, 2, 6);
+ assertEquals(2, slice.offset());
+
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (2,7) [-------------]
+ slice = Slices.findContainingSlice(slices, 2, 7);
+ assertEquals(null, slice);
+
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (6,4) [-------]
+ slice = Slices.findContainingSlice(slices, 6, 4);
+ assertEquals(6, slice.offset());
+
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (8,2) [---]
+ slice = Slices.findContainingSlice(slices, 8, 2);
+ assertEquals(8, slice.offset());
+
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (12,2) [---]
+ slice = Slices.findContainingSlice(slices, 12, 2);
+ assertEquals(8, slice.offset());
+
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (12,3) [-----]
+ slice = Slices.findContainingSlice(slices, 12, 3);
+ assertEquals(null, slice);
+
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (14,1) [-]
+ slice = Slices.findContainingSlice(slices, 14, 1);
+ assertEquals(null, slice);
+ }
+
+
+ @Test
+ public void testAddSlice() {
+
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (2,6) [-----------]
+ // (6,4) [---------]
+ // (8,6) [-----------]
+ final List initial = createSlices(
+ new long[] {2, 6, 8},
+ new long[] {6, 4, 6});
+ List slices;
+
+
+ slices = new ArrayList<>(initial);
+ Slices.addSlice(slices, Range.at(0, 1));
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (0,1) [-]
+ // (2,6) [-----------]
+ // (6,4) [---------]
+ // (8,6) [-----------]
+ assertEquals(createSlices(
+ new long[] {0, 2, 6, 8},
+ new long[] {1, 6, 4, 6}), slices);
+
+
+ slices = new ArrayList<>(initial);
+ Slices.addSlice(slices, Range.at(0, 16));
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (0,16)[-------------------------------]
+ assertEquals(createSlices(
+ new long[] {0},
+ new long[] {16}), slices);
+
+
+ slices = new ArrayList<>(initial);
+ Slices.addSlice(slices, Range.at(2, 8));
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (2,8) [-----------------]
+ // (8,6) [-----------]
+ assertEquals(createSlices(
+ new long[] {2, 8},
+ new long[] {8, 6}), slices);
+
+
+ slices = new ArrayList<>(initial);
+ Slices.addSlice(slices, Range.at(1, 10));
+ // 0 1 2 3 4 5 6 7 8 9 A B C D E F
+ // (1,10) [---------------------]
+ // (8,6) [-----------]
+ assertEquals(createSlices(
+ new long[] {1, 8},
+ new long[] {10, 6}), slices);
+ }
+}
diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java
index 7d30db739..36bf19240 100644
--- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java
+++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java
@@ -537,9 +537,8 @@ public void numReadsTest() {
new ByteArrayDataBlock(chunkSize, new long[]{11, 11}, data)
);
- writer.resetNumMaterializeCalls();
- writer.readChunks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0}));
- System.out.println(writer.getNumMaterializeCalls());
+ writer.resetNumMaterializeCalls();
+ writer.readChunks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0}));
ArrayList ptList = new ArrayList<>();
ptList.add(new long[] {0, 0});
@@ -547,10 +546,8 @@ public void numReadsTest() {
ptList.add(new long[] {1, 0});
ptList.add(new long[] {1, 1});
- writer.resetNumMaterializeCalls();
- writer.readChunks(dataset, datasetAttributes, ptList);
- System.out.println(writer.getNumMaterializeCalls());
- System.out.println("");
+ writer.resetNumMaterializeCalls();
+ writer.readChunks(dataset, datasetAttributes, ptList);
}
@Test
@@ -656,14 +653,29 @@ public void testPartialReadAggregationBehavior() {
writer.resetNumMaterializeCalls();
writer.readChunks(dataset, datasetAttributes, ptList);
- // TODO change this if and when we implement aggregation of read calls
- // one for the index, one for each of the four blocks
- assertEquals(5, writer.getNumMaterializeCalls());
+ // one for the index, one for the four blocks (aggregated)
+ assertEquals(2, writer.getNumMaterializeCalls());
writer.resetNumMaterializeCalls();
writer.readBlock(dataset, datasetAttributes, new long[] {0,0});
- // one for the index, one for each of the four blocks
- assertEquals(5, writer.getNumMaterializeCalls());
+ // one for the index, one for the four blocks (aggregated)
+ assertEquals(2, writer.getNumMaterializeCalls());
+
+
+ /**
+ * Aggregate read calls
+ */
+ writer.tkva.aggregate = true;
+ writer.resetNumMaterializeCalls();
+ writer.readChunks(dataset, datasetAttributes, ptList);
+
+ // one for the index, one that covers ALL the blocks)
+ assertEquals(2, writer.getNumMaterializeCalls());
+
+ writer.resetNumMaterializeCalls();
+ writer.readBlock(dataset, datasetAttributes, new long[] {0,0});
+ // one for the index, one that covers ALL the blocks
+ assertEquals(2, writer.getNumMaterializeCalls());
}
}