Skip to content

Commit 896f436

Browse files
bogovicjcmhulbert
bogovicj
authored and
cmhulbert
committed
feat: make SliceTrackingLazyRead abstract, add two implementations
* Default- and Aggregating- * add tests
1 parent 643b2ee commit 896f436

4 files changed

Lines changed: 317 additions & 2 deletions

File tree

‎src/main/java/org/janelia/saalfeldlab/n5/readdata/kva/AggregatingSliceTrackingLazyRead.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package org.janelia.saalfeldlab.n5.readdata.kva;
2+
3+
import java.util.Collection;
4+
5+
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
6+
import org.janelia.saalfeldlab.n5.readdata.Range;
7+
8+
public class AggregatingSliceTrackingLazyRead extends SliceTrackingLazyRead {
9+
10+
public AggregatingSliceTrackingLazyRead(final LazyRead delegate) {
11+
super(delegate);
12+
}
13+
14+
/**
15+
* Indicates that the given slices will be subsequently read.
16+
* <p>
17+
* This implementation groups overlapping / adjacent {@link Range}s into single read requests.
18+
*
19+
* @param ranges
20+
* slice ranges to prefetch
21+
*
22+
* @throws N5IOException
23+
* if any I/O error occurs
24+
*/
25+
@Override
26+
public void prefetch(final Collection<? extends Range> ranges) throws N5IOException {
27+
28+
final Collection<? extends Range> aggregatedRanges = Range.aggregate(ranges);
29+
for (final Range slice : aggregatedRanges) {
30+
materialize(slice.offset(), slice.length());
31+
}
32+
}
33+
34+
}

‎src/main/java/org/janelia/saalfeldlab/n5/readdata/kva/DefaultSliceTrackingLazyRead.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package org.janelia.saalfeldlab.n5.readdata.kva;
2+
3+
import java.util.Collection;
4+
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
5+
import org.janelia.saalfeldlab.n5.readdata.Range;
6+
7+
public class DefaultSliceTrackingLazyRead extends SliceTrackingLazyRead {
8+
9+
public DefaultSliceTrackingLazyRead(final LazyRead delegate) {
10+
super(delegate);
11+
}
12+
13+
/**
14+
* Indicates that the given slices will be subsequently read.
15+
* {@code LazyRead} implementations (optionally) may take steps to prepare
16+
* for these subsequent slices.
17+
* <p>
18+
* Minimal implementation: Find offset and length covering all ranges that
19+
* are not yet fully covered by existing slices. Then materialize the slice
20+
* covering that range.
21+
*
22+
* @param ranges
23+
* slice ranges to prefetch
24+
*
25+
* @throws N5IOException
26+
* if any I/O error occurs
27+
*/
28+
@Override
29+
public void prefetch(final Collection<? extends Range> ranges) throws N5IOException {
30+
31+
long fromIndex = Long.MAX_VALUE;
32+
long toIndex = Long.MIN_VALUE;
33+
for (final Range slice : ranges) {
34+
if (!isCovered(slice)) {
35+
fromIndex = Math.min(fromIndex, slice.offset());
36+
toIndex = Math.max(toIndex, slice.end());
37+
}
38+
}
39+
40+
if (fromIndex < toIndex) {
41+
materialize(fromIndex, toIndex - fromIndex);
42+
}
43+
}
44+
45+
private boolean isCovered(final Range slice) {
46+
47+
return Slices.findContainingSlice(slices, slice) != null;
48+
}
49+
}

‎src/main/java/org/janelia/saalfeldlab/n5/readdata/kva/SliceTrackingLazyRead.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import org.janelia.saalfeldlab.n5.readdata.Range;
99
import org.janelia.saalfeldlab.n5.readdata.ReadData;
1010

11-
public class SliceTrackingLazyRead implements LazyRead {
11+
public abstract class SliceTrackingLazyRead implements LazyRead {
1212

1313
private static class Slice implements Range {
1414

@@ -41,7 +41,7 @@ public String toString() {
4141
}
4242
}
4343

44-
private final List<Slice> slices = new ArrayList<>();
44+
protected final List<Slice> slices = new ArrayList<>();
4545

4646
/**
4747
* The {@code LazyRead} providing our data.

‎src/test/java/org/janelia/saalfeldlab/n5/readdata/kva/SliceTrackingLazyReadTests.java‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
package org.janelia.saalfeldlab.n5.readdata.kva;
2+
3+
import static org.junit.Assert.assertEquals;
4+
5+
import java.io.IOException;
6+
import java.util.Arrays;
7+
import java.util.List;
8+
9+
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
10+
import org.janelia.saalfeldlab.n5.readdata.Range;
11+
import org.janelia.saalfeldlab.n5.readdata.ReadData;
12+
import org.junit.Test;
13+
14+
public class SliceTrackingLazyReadTests {
15+
16+
@Test
17+
public void testDefaultSliceTracking() throws N5IOException {
18+
19+
/**
20+
* 1. Create sample ReadData from byte[]
21+
* 2. Create a DummyLazy Read
22+
* 3. Make a DefaultSliceTrackingLazyRead
23+
* 4. Create a list of ranges
24+
* 5. Call prefetch
25+
* 6. Ensure the correct number of materialize calls were made (always 1 for DefaultSliceTrackingLazyRead)
26+
* 7. Verify the stored slices contain the correct range
27+
*/
28+
29+
// 1-2. Create a DummyLazyRead with 64 bytes
30+
DummyLazyRead dummyLazyRead = createDummyLazyRead(64);
31+
32+
// 3. Make a testable DefaultSliceTrackingLazyRead
33+
TestableDefaultSliceTracker sliceTracking = new TestableDefaultSliceTracker(dummyLazyRead);
34+
35+
// 4. Create a list of ranges (two non-overlapping ranges with a gap)
36+
List<Range> ranges = Arrays.asList(
37+
Range.at(10, 5), // offset 10, length 5 (bytes 10-14)
38+
Range.at(50, 10) // offset 50, length 10 (bytes 50-59)
39+
);
40+
41+
// 5. Call prefetch
42+
sliceTracking.prefetch(ranges);
43+
44+
// 6. Ensure exactly 1 materialize call was made
45+
// DefaultSliceTrackingLazyRead creates a single large slice covering all ranges
46+
assertEquals("DefaultSliceTrackingLazyRead should make exactly 1 materialize call",
47+
1, dummyLazyRead.getNumMaterializeCalls());
48+
49+
// 7. Verify the stored slice covers the entire range from offset 10 with length 50
50+
assertStoredSlices(sliceTracking, Arrays.asList(
51+
Range.at(10, 50) // Single slice covering offset 10-59
52+
));
53+
54+
}
55+
56+
@Test
57+
public void testAggregatingSliceTracking() throws N5IOException {
58+
59+
/**
60+
* 1. Create sample ReadData from byte[]
61+
* 2. Create a DummyLazyRead
62+
* 3. Make an AggregatingSliceTrackingLazyRead
63+
* 4. Create a list of ranges
64+
* 5. Call prefetch
65+
* 6. Ensure the correct number of materialize calls were made (one per aggregated range)
66+
* 7. Verify the stored slices contain the correct ranges
67+
*/
68+
69+
// 1-2. Create a DummyLazyRead with 64 bytes
70+
DummyLazyRead dummyLazyRead = createDummyLazyRead(64);
71+
72+
// 3. Make a testable AggregatingSliceTrackingLazyRead
73+
TestableAggregatingSliceTracker sliceTracking = new TestableAggregatingSliceTracker(dummyLazyRead);
74+
75+
/*
76+
* Non-adjacent ranges
77+
*/
78+
// 4. Create a list of ranges (two non-overlapping ranges with a gap)
79+
List<Range> ranges = Arrays.asList(
80+
Range.at(10, 5), // offset 10, length 5 (bytes 10-14)
81+
Range.at(50, 10) // offset 50, length 10 (bytes 50-59)
82+
);
83+
84+
// 5. Call prefetch
85+
sliceTracking.prefetch(ranges);
86+
87+
// 6. Ensure exactly 2 materialize calls were made
88+
// AggregatingSliceTrackingLazyRead aggregates overlapping/adjacent ranges
89+
// Since these ranges are not adjacent or overlapping, it makes 2 separate calls
90+
assertEquals("AggregatingSliceTrackingLazyRead should make 2 materialize calls for non-adjacent ranges",
91+
2, dummyLazyRead.getNumMaterializeCalls());
92+
93+
// 7. Verify the stored slices contain two separate ranges
94+
assertStoredSlices(sliceTracking, Arrays.asList(
95+
Range.at(10, 5), // First slice
96+
Range.at(50, 10) // Second slice
97+
));
98+
99+
/*
100+
* Adjacent ranges
101+
*/
102+
103+
// new sliceTracking instance to clear slices
104+
sliceTracking = new TestableAggregatingSliceTracker(dummyLazyRead);
105+
dummyLazyRead.resetNumMaterializeCalls();
106+
107+
// 4. Create a list of three contiguous ranges
108+
List<Range> adjacentRanges = Arrays.asList(
109+
Range.at(10, 5), // offset 10, length 5 (bytes 10-14)
110+
Range.at(15, 10), // offset 15, length 10 (bytes 15-24)
111+
Range.at(25, 5) // offset 25, length 5 (bytes 25-29)
112+
);
113+
114+
// 5. Call prefetch
115+
sliceTracking.prefetch(adjacentRanges);
116+
117+
// 6. Ensure exactly 1 materialize call was made
118+
// AggregatingSliceTrackingLazyRead should aggregate these three contiguous ranges
119+
// into a single range from offset 10 to 30, with length 20
120+
assertEquals("AggregatingSliceTrackingLazyRead should make 1 materialize call for contiguous ranges",
121+
1, dummyLazyRead.getNumMaterializeCalls());
122+
123+
// 7. Verify the stored slices now contain three ranges total:
124+
// the two from the first prefetch plus one aggregated range from the second prefetch
125+
assertStoredSlices(sliceTracking, Arrays.asList(
126+
Range.at(10, 20) // Aggregated range
127+
));
128+
129+
}
130+
131+
private static DummyLazyRead createDummyLazyRead(int size) {
132+
byte[] data = new byte[size];
133+
for (int i = 0; i < data.length; i++) {
134+
data[i] = (byte)i;
135+
}
136+
return new DummyLazyRead(ReadData.from(data));
137+
}
138+
139+
/**
140+
* Helper method to verify that stored slices match expected ranges.
141+
*
142+
* @param sliceTracking the SliceTrackingLazyRead instance
143+
* @param expectedRanges the expected ranges stored in slices
144+
*/
145+
private static void assertStoredSlices(TestableSliceTracker sliceTracking, List<Range> expectedRanges) {
146+
// Access protected slices field via a test helper
147+
List<Range> actualSlices = sliceTracking.getSlices();
148+
149+
assertEquals("Number of stored slices should match", expectedRanges.size(), actualSlices.size());
150+
151+
for (int i = 0; i < expectedRanges.size(); i++) {
152+
Range expected = expectedRanges.get(i);
153+
Range actual = actualSlices.get(i);
154+
assertEquals("Slice " + i + " offset should match", expected.offset(), actual.offset());
155+
assertEquals("Slice " + i + " length should match", expected.length(), actual.length());
156+
}
157+
}
158+
159+
/**
160+
* Testable wrapper for DefaultSliceTrackingLazyRead that exposes slices.
161+
*/
162+
static class TestableDefaultSliceTracker extends DefaultSliceTrackingLazyRead implements TestableSliceTracker {
163+
public TestableDefaultSliceTracker(LazyRead delegate) {
164+
super(delegate);
165+
}
166+
167+
@Override
168+
public List<Range> getSlices() {
169+
return java.util.Collections.unmodifiableList(slices);
170+
}
171+
}
172+
173+
/**
174+
* Testable wrapper for AggregatingSliceTrackingLazyRead that exposes slices.
175+
*/
176+
static class TestableAggregatingSliceTracker extends AggregatingSliceTrackingLazyRead implements TestableSliceTracker {
177+
public TestableAggregatingSliceTracker(LazyRead delegate) {
178+
super(delegate);
179+
}
180+
181+
@Override
182+
public List<Range> getSlices() {
183+
return java.util.Collections.unmodifiableList(slices);
184+
}
185+
}
186+
187+
/**
188+
* Interface for testable slice trackers.
189+
*/
190+
interface TestableSliceTracker {
191+
List<Range> getSlices();
192+
}
193+
194+
static class DummyLazyRead implements LazyRead {
195+
196+
private ReadData data;
197+
private int numMaterializeCalls = 0;
198+
199+
public DummyLazyRead( ReadData data ) {
200+
this.data = data;
201+
}
202+
203+
@Override
204+
public void close() throws IOException {
205+
// no op
206+
}
207+
208+
@Override
209+
public ReadData materialize(long offset, long length) throws N5IOException {
210+
211+
numMaterializeCalls++;
212+
return data.slice(offset, length).materialize();
213+
}
214+
215+
@Override
216+
public long size() throws N5IOException {
217+
218+
return data.length();
219+
}
220+
221+
public int getNumMaterializeCalls() {
222+
223+
return numMaterializeCalls;
224+
}
225+
226+
public void resetNumMaterializeCalls() {
227+
228+
numMaterializeCalls = 0;
229+
}
230+
231+
}
232+
}

0 commit comments

Comments
 (0)