diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java index eff501ba2..06e753366 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/Range.java @@ -28,6 +28,8 @@ */ package org.janelia.saalfeldlab.n5.readdata; +import java.util.ArrayList; +import java.util.Collection; import java.util.Comparator; /** @@ -61,7 +63,7 @@ default long end() { } static boolean equals(final Range r0, final Range r1) { - if (r0 == null && r1==null) { + if (r0 == null && r1 == null) { return true; } else if (r0 == null || r1 == null) { return false; @@ -119,4 +121,45 @@ public String toString() { return new DefaultRange(offset, length); } + + /** + * Returns a potentially new collection of {@link Range}s such that + * adjacent or overlapping Ranges are combined. + *

+ * If the input ranges are non-adjacent the input instance is returned, but if aggregation + * occurs, the result will be a new, sorted List. + * + * @param ranges + * a collection of Ranges + * + * @return collection with adjacent or overlapping ranges merged + */ + static Collection aggregate(final Collection ranges) { + + if (ranges.size() == 0) + return ranges; + + final ArrayList sortedRanges = new ArrayList<>(ranges); + sortedRanges.sort(Range.COMPARATOR); + + final ArrayList result = new ArrayList<>(); + Range lo = null; + for (Range hi : sortedRanges) { + + if (lo == null) + lo = hi; + else if (lo.end() >= hi.offset()) { + // merge + lo = Range.at(lo.offset(), Math.max(lo.end(), hi.end()) - lo.offset()); + } else { + result.add(lo); + lo = hi; + } + } + result.add(lo); + + final boolean wereMerges = ranges.size() != result.size(); + return wereMerges ? result : ranges; + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java index 122a399ef..25da8b0b8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/VolatileReadData.java @@ -1,7 +1,7 @@ package org.janelia.saalfeldlab.n5.readdata; -import java.io.InputStream; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.prefetch.AggregatingPrefetchLazyRead; /** * During its life-time, the content of a {@code VolatileReadData} should not be @@ -29,7 +29,8 @@ public interface VolatileReadData extends ReadData, AutoCloseable { * @return a new VolatileReadData */ static VolatileReadData from(final LazyRead lazyRead) { - return new LazyReadData(lazyRead); + final LazyRead aggregatingLazyRead = new AggregatingPrefetchLazyRead(lazyRead); + return new LazyReadData(aggregatingLazyRead); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java new file mode 100644 index 000000000..dcc0097a1 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/AggregatingPrefetchLazyRead.java @@ -0,0 +1,44 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.util.ArrayList; +import java.util.Collection; + +import java.util.List; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.Range; + +/** + * A {@link SliceTrackingLazyRead} that implements {@link #prefetch} to + * aggregate overlapping / adjacent ranges and then materialize each aggregated + * range. + */ +public class AggregatingPrefetchLazyRead extends SliceTrackingLazyRead { + + public AggregatingPrefetchLazyRead(final LazyRead delegate) { + super(delegate); + } + + /** + * Indicates that the given slices will be subsequently read. + *

+ * This implementation groups overlapping / adjacent {@link Range}s into single read requests. + * + * @param ranges + * slice ranges to prefetch + * + * @throws N5IOException + * if any I/O error occurs + */ + @Override + public void prefetch(final Collection ranges) throws N5IOException { + + final List filteredRanges = new ArrayList<>(ranges); + filteredRanges.removeIf(this::isCovered); + final Collection aggregatedRanges = Range.aggregate(filteredRanges); + for (final Range slice : aggregatedRanges) { + materialize(slice.offset(), slice.length()); + } + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/EnclosingPrefetchLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/EnclosingPrefetchLazyRead.java new file mode 100644 index 000000000..b0d6fdc7a --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/EnclosingPrefetchLazyRead.java @@ -0,0 +1,49 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.util.Collection; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.Range; + +/** + * A {@link SliceTrackingLazyRead} that implements {@link #prefetch} to + * materialize the bounding range of all requested ranges. + */ +public class EnclosingPrefetchLazyRead extends SliceTrackingLazyRead { + + public EnclosingPrefetchLazyRead(final LazyRead delegate) { + super(delegate); + } + + /** + * Indicates that the given slices will be subsequently read. + * {@code LazyRead} implementations (optionally) may take steps to prepare + * for these subsequent slices. + *

+ * Minimal implementation: Find offset and length covering all ranges that + * are not yet fully covered by existing slices. Then materialize the slice + * covering that range. + * + * @param ranges + * slice ranges to prefetch + * + * @throws N5IOException + * if any I/O error occurs + */ + @Override + public void prefetch(final Collection ranges) throws N5IOException { + + long fromIndex = Long.MAX_VALUE; + long toIndex = Long.MIN_VALUE; + for (final Range slice : ranges) { + if (!isCovered(slice)) { + fromIndex = Math.min(fromIndex, slice.offset()); + toIndex = Math.max(toIndex, slice.end()); + } + } + + if (fromIndex < toIndex) { + materialize(fromIndex, toIndex - fromIndex); + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java new file mode 100644 index 000000000..5835c473e --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyRead.java @@ -0,0 +1,90 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.Range; +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +/** + * A {@link LazyRead} that wraps a delegate {@code LazyRead} and keeps track of + * all slices that have been {@link #materialize materialized}. + *

+ * When materializing a new slice, we first check whether it is completely + * covered by a materialized slice that we already track. If so, then we just + * return a slice on the existing materialized slice. If not, we materialize the + * slice from the delegate track it. + */ +public class SliceTrackingLazyRead implements LazyRead { + + protected static class Slice implements Range { + + // Offset and length in the delegate + private final long offset; + private final long length; + + // Data of this slice + private final ReadData data; + + Slice(final long offset, final long length, final ReadData data) { + this.offset = offset; + this.length = length; + this.data = data; + } + + @Override + public long offset() { + return offset; + } + + @Override + public long length() { + return length; + } + + @Override + public String toString() { + return "{" + offset + ", " + length + '}'; + } + } + + protected final List slices = new ArrayList<>(); + + /** + * The {@code LazyRead} providing our data. + */ + private final LazyRead delegate; + + public SliceTrackingLazyRead(final LazyRead delegate) { + this.delegate = delegate; + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + @Override + public ReadData materialize(final long offset, final long length) throws N5IOException { + final Slice containing = Slices.findContainingSlice(slices, offset, length); + if (containing != null) { + return containing.data.slice(offset - containing.offset, length); + } else { + final ReadData data = delegate.materialize(offset, length); + Slices.addSlice(slices, new Slice(offset, length, data)); + return data; + } + } + + @Override + public long size() throws N5IOException { + return delegate.size(); + } + + protected boolean isCovered(final Range slice) { + + return Slices.findContainingSlice(slices, slice) != null; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java new file mode 100644 index 000000000..d3d0dfb11 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/readdata/prefetch/Slices.java @@ -0,0 +1,116 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import org.janelia.saalfeldlab.n5.readdata.Range; + +class Slices { + + private Slices() { + // utility class. should not be instantiated. + } + + /** + * In an ordered list of {@code slices}, find a slice that completely contains the given range. + *

+ * Pre-conditions: + *

    + *
  1. Slices are ordered by offset.
  2. + *
  3. If two slices overlap, no slice is fully contained within the other. + * (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
  4. + *
+ * + * @param slices + * ordered list of slices + * @param offset + * start of the range to cover + * @param length + * length of the range to cover + * + * @return a slice that completely contains the requested range, or {@code null} if no such slice exists + */ + static T findContainingSlice(final List slices, final long offset, final long length) { + + // Find the slice with the largest slice.offset <= offset. + final int i = Collections.binarySearch(slices, Range.at(offset, 0), Comparator.comparingLong(Range::offset)); + + // Largest index of a slice with slice.offset <= offset. + final int index = i < 0 ? -i - 2 : i; + if (index < 0) { + // We find no overlapping slice, because + // slices[0].offset is already too large. + return null; + } + + final T slice = slices.get(index); + if (slice.end() < offset + length) { + return null; + } + + return slice; + } + + /** + * In an ordered list of {@code slices}, find a slice that completely contains the given range. + *

+ * Pre-conditions: + *

    + *
  1. Slices are ordered by offset.
  2. + *
  3. If two slices overlap, no slice is fully contained within the other. + * (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
  4. + *
+ * + * @param slices + * ordered list of slices + * @param range + * range to cover + * + * @return a slice that completely contains the requested range, or {@code null} if no such slice exists + */ + static T findContainingSlice(final List slices, final Range range) { + return findContainingSlice(slices, range.offset(), range.length()); + } + + /** + * Add a new {@code slice} to the {@code slice} list. + *

+ * Note, that the new {@code slice} is expected to not be fully contained in + * an existing slice! + *

+ * Pre/post-conditions: + *

    + *
  1. Slices are ordered by offset.
  2. + *
  3. If two slices overlap, no slice is fully contained within the other. + * (Therefore, if {@code a.offset < b.offset} then {@code a.end < b.end}.)
  4. + *
+ *

+ * The new {@code slice} will be inserted into the list at the correct position + * (such that {@code slices} remains ordered by slice offset), and all existing + * slices that are fully contained in the new {@code slice} will be removed. + * + * @param slices + * ordered list of slices + * @param slice + * slice to be inserted + */ + static void addSlice(final List slices, final T slice) { + + final int i = Collections.binarySearch(slices, slice, Comparator.comparingLong(Range::offset)); + final int from = i < 0 ? -i - 1 : i; + + int to = from; + while (to < slices.size() && slices.get(to).end() <= slice.end()) { + ++to; + } + + if (from == to) { + // empty range: just insert + slices.add(from, slice); + } else { + // overwrite the first element in range, remove the rest + slices.set(from, slice); + slices.subList(from + 1, to).clear(); + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java index 630c30033..66e0dd548 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/DefaultDatasetAccess.java @@ -155,6 +155,15 @@ private void readChunksRecursive( // TODO: collect all the elementPos that we will need and prefetch // Probably best to add a prefetch method to RawShard? + // Here's an attempt at that. + // Don't love that we have to build a list of positions + // Consider making DataBlockRequest package private and passing them directly + final ArrayList positions = new ArrayList<>(); + for (final ChunkRequest request : requests) { + positions.add(request.position.relative(0)); + } + shard.prefetch(positions); + for (final ChunkRequest request : requests) { final long[] elementPos = request.position.relative(0); final ReadData elementData = shard.getElementData(elementPos); @@ -1015,7 +1024,7 @@ public List> chunks(final List> duplicates) { * Construct {@code ChunkRequests} from a list of level-0 grid positions * for reading. *

- * The nesting level ot the returned {@code ChunkRequests} is {@code + * The nesting level of the returned {@code ChunkRequests} is {@code * grid.numLevels()}, that is level of the highest-order shard + 1. This * implies that the requests are not guaranteed to be in the same shard (at * any level. {@link ChunkRequests#split() Splitting} the {@code diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java index 01d7e961b..419ee1ccb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/RawShard.java @@ -28,6 +28,11 @@ */ package org.janelia.saalfeldlab.n5.shard; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.janelia.saalfeldlab.n5.readdata.Range; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.segment.Segment; import org.janelia.saalfeldlab.n5.readdata.segment.SegmentedReadData; @@ -86,4 +91,16 @@ public void setElementData(final ReadData data, final long[] pos) { final Segment segment = data == null ? null : SegmentedReadData.wrap(data).segments().get(0); index.set(segment, pos); } + + public void prefetch(List positions) { + + final List ranges = new ArrayList<>(positions.size()); + for (long[] pos : positions) { + final Segment seg = index.get(pos); + if (seg != null) + ranges.add(sourceData.location(seg)); + } + sourceData.prefetch(ranges); + } + } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java index e5a705422..17d421750 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java @@ -35,7 +35,6 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.file.Files; import java.util.Arrays; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java index 4ce148c50..73b6a2f46 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java @@ -29,8 +29,6 @@ package org.janelia.saalfeldlab.n5.benchmarks; import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -40,7 +38,6 @@ import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess; import org.janelia.saalfeldlab.n5.KeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java index 8572e1cee..fd151a265 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java @@ -30,17 +30,14 @@ import org.apache.commons.io.IOUtils; import org.janelia.saalfeldlab.n5.HttpKeyValueAccess; -import org.janelia.saalfeldlab.n5.LockedChannel; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; import org.junit.Test; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.charset.Charset; -import java.util.function.Function; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java index 0c4f4a876..a2e18b17d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java @@ -5,13 +5,14 @@ import org.janelia.saalfeldlab.n5.readdata.LazyRead; import org.janelia.saalfeldlab.n5.readdata.ReadData; import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; -import org.janelia.saalfeldlab.n5.shard.ShardTest; +import org.janelia.saalfeldlab.n5.readdata.prefetch.AggregatingPrefetchLazyRead; public class TrackingKeyValueAccess extends DelegateKeyValueAccess { public int numMaterializeCalls = 0; public int numIsFileCalls = 0; public long totalBytesRead = 0; + public boolean aggregate = false; public TrackingKeyValueAccess(final KeyValueAccess kva) { super(kva); @@ -25,15 +26,20 @@ public boolean isFile(String normalPath) { @Override public VolatileReadData createReadData(final String normalPath) { -// throw new N5NoSuchKeyException("Test No Such Key"); - return VolatileReadData.from(new TrackingVolatileReadData(kva.createReadData(normalPath))); + + final VolatileReadData volatileReadData = kva.createReadData(normalPath); + final TrackingLazyRead trackingLazyRead = new TrackingLazyRead(volatileReadData); + LazyRead lazyRead = trackingLazyRead; + if (aggregate) + lazyRead = new AggregatingPrefetchLazyRead(trackingLazyRead); + return VolatileReadData.from( lazyRead ); } - private class TrackingVolatileReadData implements LazyRead { + private class TrackingLazyRead implements LazyRead { private final VolatileReadData readData; - TrackingVolatileReadData(final VolatileReadData readData) { + TrackingLazyRead(final VolatileReadData readData) { this.readData = readData; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java new file mode 100644 index 000000000..3499220b9 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/RangeTests.java @@ -0,0 +1,58 @@ +package org.janelia.saalfeldlab.n5.readdata; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.junit.Test; + +public class RangeTests { + + @Test + public void testAggregate() { + + List nonOverlapping = Stream.of(Range.at(0, 5), Range.at(12, 2)).collect(Collectors.toList()); + assertSame(nonOverlapping, Range.aggregate(nonOverlapping)); + + List nonOverlappingRev = Stream.of(Range.at(12, 2), Range.at(0, 5)).collect(Collectors.toList()); + assertSame(nonOverlappingRev, Range.aggregate(nonOverlappingRev)); + + /** + * 0 1 2 3 4 5 + * x x x x x - + * - x - - - - + */ + List containing = Stream.of(Range.at(0, 5), Range.at(1, 1)).collect(Collectors.toList()); + assertEquals(Collections.singletonList(Range.at(0, 5)), Range.aggregate(containing)); + + /** + * 0 1 2 3 4 5 6 + * x x x x x - - + * - - x x x x x + */ + List overlapping = Stream.of(Range.at(0, 5), Range.at(2, 5)).collect(Collectors.toList()); + assertEquals(Collections.singletonList(Range.at(0, 7)), Range.aggregate(overlapping)); + + /** + * 0 1 2 3 4 5 + * x x x x x - + * - - - - - x + */ + List adjacent = Stream.of(Range.at(0, 5), Range.at(5, 1)).collect(Collectors.toList()); + assertEquals(Collections.singletonList(Range.at(0, 6)), Range.aggregate(adjacent)); + + /** + * 0 1 2 3 4 5 + * - - - x x - + * x x - - - - + * - - - - - x + */ + List three = Stream.of(Range.at(3, 2), Range.at(0, 2), Range.at(5, 1)).collect(Collectors.toList()); + assertEquals(Stream.of(Range.at(0, 2), Range.at(3, 3)).collect(Collectors.toList()), Range.aggregate(three)); + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index c4f3fa51b..78522ff3e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -38,7 +38,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.file.FileSystems; import java.util.Arrays; import java.util.function.IntUnaryOperator; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java new file mode 100644 index 000000000..fcac9a5e3 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SliceTrackingLazyReadTests.java @@ -0,0 +1,233 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.Range; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.junit.Test; + +public class SliceTrackingLazyReadTests { + + @Test + public void testDefaultSliceTracking() throws N5IOException { + + /** + * 1. Create sample ReadData from byte[] + * 2. Create a DummyLazy Read + * 3. Make a DefaultSliceTrackingLazyRead + * 4. Create a list of ranges + * 5. Call prefetch + * 6. Ensure the correct number of materialize calls were made (always 1 for DefaultSliceTrackingLazyRead) + * 7. Verify the stored slices contain the correct range + */ + + // 1-2. Create a DummyLazyRead with 64 bytes + DummyLazyRead dummyLazyRead = createDummyLazyRead(64); + + // 3. Make a testable DefaultSliceTrackingLazyRead + TestableDefaultSliceTracker sliceTracking = new TestableDefaultSliceTracker(dummyLazyRead); + + // 4. Create a list of ranges (two non-overlapping ranges with a gap) + List ranges = Arrays.asList( + Range.at(10, 5), // offset 10, length 5 (bytes 10-14) + Range.at(50, 10) // offset 50, length 10 (bytes 50-59) + ); + + // 5. Call prefetch + sliceTracking.prefetch(ranges); + + // 6. Ensure exactly 1 materialize call was made + // DefaultSliceTrackingLazyRead creates a single large slice covering all ranges + assertEquals("DefaultSliceTrackingLazyRead should make exactly 1 materialize call", + 1, dummyLazyRead.getNumMaterializeCalls()); + + // 7. Verify the stored slice covers the entire range from offset 10 with length 50 + assertStoredSlices(sliceTracking, Arrays.asList( + Range.at(10, 50) // Single slice covering offset 10-59 + )); + + } + + @Test + public void testAggregatingSliceTracking() throws N5IOException { + + /** + * 1. Create sample ReadData from byte[] + * 2. Create a DummyLazyRead + * 3. Make an AggregatingSliceTrackingLazyRead + * 4. Create a list of ranges + * 5. Call prefetch + * 6. Ensure the correct number of materialize calls were made (one per aggregated range) + * 7. Verify the stored slices contain the correct ranges + */ + + // 1-2. Create a DummyLazyRead with 64 bytes + DummyLazyRead dummyLazyRead = createDummyLazyRead(64); + + // 3. Make a testable AggregatingSliceTrackingLazyRead + TestableAggregatingSliceTracker sliceTracking = new TestableAggregatingSliceTracker(dummyLazyRead); + + /* + * Non-adjacent ranges + */ + // 4. Create a list of ranges (two non-overlapping ranges with a gap) + List ranges = Arrays.asList( + Range.at(10, 5), // offset 10, length 5 (bytes 10-14) + Range.at(50, 10) // offset 50, length 10 (bytes 50-59) + ); + + // 5. Call prefetch + sliceTracking.prefetch(ranges); + + // 6. Ensure exactly 2 materialize calls were made + // AggregatingSliceTrackingLazyRead aggregates overlapping/adjacent ranges + // Since these ranges are not adjacent or overlapping, it makes 2 separate calls + assertEquals("AggregatingSliceTrackingLazyRead should make 2 materialize calls for non-adjacent ranges", + 2, dummyLazyRead.getNumMaterializeCalls()); + + // 7. Verify the stored slices contain two separate ranges + assertStoredSlices(sliceTracking, Arrays.asList( + Range.at(10, 5), // First slice + Range.at(50, 10) // Second slice + )); + + /* + * Adjacent ranges + */ + + // new sliceTracking instance to clear slices + sliceTracking = new TestableAggregatingSliceTracker(dummyLazyRead); + dummyLazyRead.resetNumMaterializeCalls(); + + // 4. Create a list of three contiguous ranges + List adjacentRanges = Arrays.asList( + Range.at(10, 5), // offset 10, length 5 (bytes 10-14) + Range.at(15, 10), // offset 15, length 10 (bytes 15-24) + Range.at(25, 5) // offset 25, length 5 (bytes 25-29) + ); + + // 5. Call prefetch + sliceTracking.prefetch(adjacentRanges); + + // 6. Ensure exactly 1 materialize call was made + // AggregatingSliceTrackingLazyRead should aggregate these three contiguous ranges + // into a single range from offset 10 to 30, with length 20 + assertEquals("AggregatingSliceTrackingLazyRead should make 1 materialize call for contiguous ranges", + 1, dummyLazyRead.getNumMaterializeCalls()); + + // 7. Verify the stored slices now contain three ranges total: + // the two from the first prefetch plus one aggregated range from the second prefetch + assertStoredSlices(sliceTracking, Arrays.asList( + Range.at(10, 20) // Aggregated range + )); + + } + + private static DummyLazyRead createDummyLazyRead(int size) { + byte[] data = new byte[size]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte)i; + } + return new DummyLazyRead(ReadData.from(data)); + } + + /** + * Helper method to verify that stored slices match expected ranges. + * + * @param sliceTracking the SliceTrackingLazyRead instance + * @param expectedRanges the expected ranges stored in slices + */ + private static void assertStoredSlices(TestableSliceTracker sliceTracking, List expectedRanges) { + // Access protected slices field via a test helper + List actualSlices = sliceTracking.getSlices(); + + assertEquals("Number of stored slices should match", expectedRanges.size(), actualSlices.size()); + + for (int i = 0; i < expectedRanges.size(); i++) { + Range expected = expectedRanges.get(i); + Range actual = actualSlices.get(i); + assertEquals("Slice " + i + " offset should match", expected.offset(), actual.offset()); + assertEquals("Slice " + i + " length should match", expected.length(), actual.length()); + } + } + + /** + * Testable wrapper for DefaultSliceTrackingLazyRead that exposes slices. + */ + static class TestableDefaultSliceTracker extends EnclosingPrefetchLazyRead implements TestableSliceTracker { + public TestableDefaultSliceTracker(LazyRead delegate) { + super(delegate); + } + + @Override + public List getSlices() { + return java.util.Collections.unmodifiableList(slices); + } + } + + /** + * Testable wrapper for AggregatingSliceTrackingLazyRead that exposes slices. + */ + static class TestableAggregatingSliceTracker extends AggregatingPrefetchLazyRead implements TestableSliceTracker { + public TestableAggregatingSliceTracker(LazyRead delegate) { + super(delegate); + } + + @Override + public List getSlices() { + return java.util.Collections.unmodifiableList(slices); + } + } + + /** + * Interface for testable slice trackers. + */ + interface TestableSliceTracker { + List getSlices(); + } + + static class DummyLazyRead implements LazyRead { + + private ReadData data; + private int numMaterializeCalls = 0; + + public DummyLazyRead( ReadData data ) { + this.data = data; + } + + @Override + public void close() throws IOException { + // no op + } + + @Override + public ReadData materialize(long offset, long length) throws N5IOException { + + numMaterializeCalls++; + return data.slice(offset, length).materialize(); + } + + @Override + public long size() throws N5IOException { + + return data.length(); + } + + public int getNumMaterializeCalls() { + + return numMaterializeCalls; + } + + public void resetNumMaterializeCalls() { + + numMaterializeCalls = 0; + } + + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java new file mode 100644 index 000000000..0ce2b7442 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/prefetch/SlicesTest.java @@ -0,0 +1,134 @@ +package org.janelia.saalfeldlab.n5.readdata.prefetch; + +import java.util.ArrayList; +import java.util.List; +import org.janelia.saalfeldlab.n5.readdata.Range; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + + +public class SlicesTest { + + private List createSlices(final long[] offsets, final long[] lengths) { + final List slices = new ArrayList<>(); + for (int i = 0; i < offsets.length; ++i) { + slices.add(Range.at(offsets[i], lengths[i])); + } + return slices; + } + + @Test + public void testFindContaining() { + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,6) [-----------] + // (6,4) [---------] + // (8,6) [-----------] + + final List slices = createSlices( + new long[] {2, 6, 8}, + new long[] {6, 4, 6}); + Range slice; + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (1,1) [-] + slice = Slices.findContainingSlice(slices, 1, 1); + assertEquals(null, slice); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,1) [-] + slice = Slices.findContainingSlice(slices, 2, 1); + assertEquals(2, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,6) [-----------] + slice = Slices.findContainingSlice(slices, 2, 6); + assertEquals(2, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,7) [-------------] + slice = Slices.findContainingSlice(slices, 2, 7); + assertEquals(null, slice); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (6,4) [-------] + slice = Slices.findContainingSlice(slices, 6, 4); + assertEquals(6, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (8,2) [---] + slice = Slices.findContainingSlice(slices, 8, 2); + assertEquals(8, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (12,2) [---] + slice = Slices.findContainingSlice(slices, 12, 2); + assertEquals(8, slice.offset()); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (12,3) [-----] + slice = Slices.findContainingSlice(slices, 12, 3); + assertEquals(null, slice); + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (14,1) [-] + slice = Slices.findContainingSlice(slices, 14, 1); + assertEquals(null, slice); + } + + + @Test + public void testAddSlice() { + + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,6) [-----------] + // (6,4) [---------] + // (8,6) [-----------] + final List initial = createSlices( + new long[] {2, 6, 8}, + new long[] {6, 4, 6}); + List slices; + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, Range.at(0, 1)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (0,1) [-] + // (2,6) [-----------] + // (6,4) [---------] + // (8,6) [-----------] + assertEquals(createSlices( + new long[] {0, 2, 6, 8}, + new long[] {1, 6, 4, 6}), slices); + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, Range.at(0, 16)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (0,16)[-------------------------------] + assertEquals(createSlices( + new long[] {0}, + new long[] {16}), slices); + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, Range.at(2, 8)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (2,8) [-----------------] + // (8,6) [-----------] + assertEquals(createSlices( + new long[] {2, 8}, + new long[] {8, 6}), slices); + + + slices = new ArrayList<>(initial); + Slices.addSlice(slices, Range.at(1, 10)); + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + // (1,10) [---------------------] + // (8,6) [-----------] + assertEquals(createSlices( + new long[] {1, 8}, + new long[] {10, 6}), slices); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index 7d30db739..36bf19240 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -537,9 +537,8 @@ public void numReadsTest() { new ByteArrayDataBlock(chunkSize, new long[]{11, 11}, data) ); - writer.resetNumMaterializeCalls(); - writer.readChunks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0})); - System.out.println(writer.getNumMaterializeCalls()); + writer.resetNumMaterializeCalls(); + writer.readChunks(dataset, datasetAttributes, Collections.singletonList(new long[] {0,0})); ArrayList ptList = new ArrayList<>(); ptList.add(new long[] {0, 0}); @@ -547,10 +546,8 @@ public void numReadsTest() { ptList.add(new long[] {1, 0}); ptList.add(new long[] {1, 1}); - writer.resetNumMaterializeCalls(); - writer.readChunks(dataset, datasetAttributes, ptList); - System.out.println(writer.getNumMaterializeCalls()); - System.out.println(""); + writer.resetNumMaterializeCalls(); + writer.readChunks(dataset, datasetAttributes, ptList); } @Test @@ -656,14 +653,29 @@ public void testPartialReadAggregationBehavior() { writer.resetNumMaterializeCalls(); writer.readChunks(dataset, datasetAttributes, ptList); - // TODO change this if and when we implement aggregation of read calls - // one for the index, one for each of the four blocks - assertEquals(5, writer.getNumMaterializeCalls()); + // one for the index, one for the four blocks (aggregated) + assertEquals(2, writer.getNumMaterializeCalls()); writer.resetNumMaterializeCalls(); writer.readBlock(dataset, datasetAttributes, new long[] {0,0}); - // one for the index, one for each of the four blocks - assertEquals(5, writer.getNumMaterializeCalls()); + // one for the index, one for the four blocks (aggregated) + assertEquals(2, writer.getNumMaterializeCalls()); + + + /** + * Aggregate read calls + */ + writer.tkva.aggregate = true; + writer.resetNumMaterializeCalls(); + writer.readChunks(dataset, datasetAttributes, ptList); + + // one for the index, one that covers ALL the blocks) + assertEquals(2, writer.getNumMaterializeCalls()); + + writer.resetNumMaterializeCalls(); + writer.readBlock(dataset, datasetAttributes, new long[] {0,0}); + // one for the index, one that covers ALL the blocks + assertEquals(2, writer.getNumMaterializeCalls()); } }