Skip to content

Commit 7891acd

Browse files
bogovicjbogovicj
committed
test: flesh out WriteRegionTest
1 parent a9a4612 commit 7891acd

1 file changed

Lines changed: 173 additions & 98 deletions

File tree

‎src/test/java/org/janelia/saalfeldlab/n5/shard/WriteRegionTest.java‎

Lines changed: 173 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -38,126 +38,102 @@
3838
import org.janelia.saalfeldlab.n5.codec.DataCodecInfo;
3939
import org.janelia.saalfeldlab.n5.codec.N5BlockCodecInfo;
4040
import org.janelia.saalfeldlab.n5.codec.RawBlockCodecInfo;
41+
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
4142
import org.janelia.saalfeldlab.n5.N5Writer.DataBlockSupplier;
4243
import org.janelia.saalfeldlab.n5.shard.ShardIndex.IndexLocation;
4344
import org.junit.Test;
4445

4546
public class WriteRegionTest {
4647

4748

48-
// #...............................#...............................#...............................#...............................#
49-
// $.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$
50-
// |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|
51-
// 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96
52-
53-
public static void main(String[] args) {
49+
@Test
50+
public void testWriteRegion() {
5451

55-
// int[] chunkSize = {3, 3, 3};
56-
// int[] level1ShardSize = {6, 6, 6};
57-
// int[] level2ShardSize = {24, 24, 24};
5852
int[] chunkSize = {3};
59-
int[] level1ShardSize = {6};
60-
int[] level2ShardSize = {24};
61-
final long[] datasetDimensions = {96};
62-
63-
// Chunks are 3x3x3
64-
// Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks)
65-
// Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards)
53+
final long[] datasetDimensions = {15};
6654
final BlockCodecInfo c0 = new N5BlockCodecInfo();
67-
final ShardCodecInfo c1 = new DefaultShardCodecInfo(
68-
chunkSize,
69-
c0,
70-
new DataCodecInfo[] {new RawCompression()},
71-
new RawBlockCodecInfo(),
72-
new DataCodecInfo[] {new RawCompression()},
73-
IndexLocation.END
74-
);
75-
final ShardCodecInfo c2 = new DefaultShardCodecInfo(
76-
level1ShardSize,
77-
c1,
78-
new DataCodecInfo[] {new RawCompression()},
79-
new RawBlockCodecInfo(),
80-
new DataCodecInfo[] {new RawCompression()},
81-
IndexLocation.START
82-
);
8355

8456
TestDatasetAttributes attributes = new TestDatasetAttributes(
8557
datasetDimensions,
86-
level2ShardSize,
58+
chunkSize,
8759
DataType.INT8,
88-
c2,
60+
c0,
8961
new RawCompression());
9062

9163
final DatasetAccess<byte[]> datasetAccess = attributes.getDatasetAccess();
9264
final PositionValueAccess store = new TestPositionValueAccess();
65+
DataBlockSupplier<byte[]> chunks = (gridPos, existing) -> {
66+
return createDataBlock(chunkSize, gridPos.clone(), (byte) gridPos[0]);
67+
};
68+
69+
DataBlockSupplier<byte[]> chunks255 = (gridPos, existing) -> {
70+
return createDataBlock(chunkSize, gridPos.clone(), (byte)255);
71+
};
72+
73+
DataBlockSupplier<byte[]> chunksDelete = (gridPos, existing) -> {
74+
return null;
75+
};
9376

94-
// ---------------------------------------------------------------
95-
// Some "tests"
96-
// TODO: Turn into unit tests
97-
// ---------------------------------------------------------------
9877

99-
// write some blocks, filled with constant values
100-
final int[] dataBlockSize = c1.getInnerBlockSize();
101-
// final long[] datasetDimensions = {100, 100, 100};
102-
// final long[] regionMin = {9,9,9};
103-
// final long[] regionSize = {15,15,15};
78+
// write one chunk at grid Position 1
79+
datasetAccess.writeRegion(store,
80+
new long[] {3},
81+
new long[] {3},
82+
chunks,
83+
false);
10484

105-
// #...............................#...............................#...............................#...............................#
106-
// $.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$.......$
107-
// |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|
108-
// 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96
85+
// Chunks
86+
// |...|...|...|...|...|
87+
// Pixels indexes
88+
// 0 3 6 9 12 15-
89+
90+
checkChunk(datasetAccess.readChunk(store, new long[] {0}), false, 0);
91+
checkChunk(datasetAccess.readChunk(store, new long[] {1}), true, 1);
92+
checkChunk(datasetAccess.readChunk(store, new long[] {2}), false, 0);
93+
checkChunk(datasetAccess.readChunk(store, new long[] {3}), false, 0);
94+
checkChunk(datasetAccess.readChunk(store, new long[] {4}), false, 0);
95+
96+
// write two chunks at grid positions 2 and 3
97+
datasetAccess.writeRegion(store,
98+
new long[]{6},
99+
new long[]{6},
100+
chunks,
101+
false);
109102

110-
final long[] regionMin = {93};
111-
final long[] regionSize = {1};
112103

113-
DataBlockSupplier<byte[]> blocks = (gridPos, existing) -> {
114-
// System.out.println("BlockSupplier.get(" + Arrays.toString(gridPos) + ", " + existing + ")");
115-
return createDataBlock(dataBlockSize, gridPos.clone(), (byte) gridPos[0]);
116-
};
104+
checkChunk(datasetAccess.readChunk(store, new long[] {0}), false, 0);
105+
checkChunk(datasetAccess.readChunk(store, new long[] {1}), true, 1);
106+
checkChunk(datasetAccess.readChunk(store, new long[] {2}), true, 2);
107+
checkChunk(datasetAccess.readChunk(store, new long[] {3}), true, 3);
108+
checkChunk(datasetAccess.readChunk(store, new long[] {4}), false, 0);
109+
110+
// delete two chunks at grid positions 3 and 4
117111
datasetAccess.writeRegion(store,
118-
regionMin,
119-
regionSize,
120-
blocks,
112+
new long[]{9},
113+
new long[]{6},
114+
chunksDelete,
121115
false);
122116

123-
// verify that the written blocks can be read back with the correct values
124-
// checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), true, 1);
125-
// checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2);
126-
// checkBlock(datasetAccess.readChunk(store, new long[] {0, 1, 0}), true, 3);
127-
// checkBlock(datasetAccess.readChunk(store, new long[] {1, 1, 0}), true, 4);
128-
// checkBlock(datasetAccess.readChunk(store, new long[] {3, 2, 1}), true, 5);
129-
// checkBlock(datasetAccess.readChunk(store, new long[] {8, 4, 1}), true, 6);
130-
131-
// verify that deleting a block removes it from the shard (while other blocks in the same shard are still present)
132-
// datasetAccess.deleteBlock(store, new long[] {0, 0, 0});
133-
// checkBlock(datasetAccess.readChunk(store, new long[] {0, 0, 0}), false, 1);
134-
// checkBlock(datasetAccess.readChunk(store, new long[] {1, 0, 0}), true, 2);
135-
136-
// if a shard becomes empty the corresponding key should be deleted
137-
// if ( store.get(new long[] {1, 0, 0}) == null ) {
138-
// throw new IllegalStateException("expected non-null readData");
139-
// }
140-
// datasetAccess.deleteChunk(store, new long[] {8, 4, 1});
141-
// if ( store.get(new long[] {1, 0, 0}) != null ) {
142-
// throw new IllegalStateException("expected null readData");
143-
// }
144-
145-
// deleting a non-existent block should not fail
146-
// datasetAccess.deleteChunk(store, new long[] {0, 0, 8});
147-
148-
System.out.println("all good");
149117
}
150118

151119
@Test
152120
public void testWriteRegionSharded() {
153121

122+
// Shards
123+
// #...............................#...............................#...............................#...............................#
124+
// Chunks
125+
// |...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|...|
126+
// Pixels indexes
127+
// 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 75 78 81 84 87 90 93 96
128+
154129
int[] chunkSize = {3};
155-
int[] level2ShardSize = {24};
130+
int[] shardSize = {24};
156131
final long[] datasetDimensions = {96};
132+
int numChunks = (int)(datasetDimensions[0] / chunkSize[0]);
133+
134+
// Chunks are size 3
135+
// Shards are size 24 (contain 8 chunks)
157136

158-
// Chunks are 3x3x3
159-
// Level 1 shards are 6x6x6 (contain 2x2x2 DataBlocks)
160-
// Level 2 shards are 24x24x24 (contain 4x4x4 Level 1 shards)
161137
final BlockCodecInfo c0 = new N5BlockCodecInfo();
162138
final ShardCodecInfo c1 = new DefaultShardCodecInfo(
163139
chunkSize,
@@ -170,41 +146,129 @@ public void testWriteRegionSharded() {
170146

171147
TestDatasetAttributes attributes = new TestDatasetAttributes(
172148
datasetDimensions,
173-
level2ShardSize,
149+
shardSize,
174150
DataType.INT8,
175151
c1,
176152
new RawCompression());
177153

178154
final DatasetAccess<byte[]> datasetAccess = attributes.getDatasetAccess();
179155
final PositionValueAccess store = new TestPositionValueAccess();
180156

181-
final int[] dataBlockSize = c1.getInnerBlockSize();
182-
final long[] regionMin = {93};
183-
final long[] regionSize = {1};
157+
DataBlockSupplier<byte[]> chunks = (gridPos, existing) -> {
158+
return createDataBlock(chunkSize, gridPos.clone(), (byte) gridPos[0]);
159+
};
160+
161+
DataBlockSupplier<byte[]> chunks255 = (gridPos, existing) -> {
162+
return createDataBlock(chunkSize, gridPos.clone(), (byte)255);
163+
};
184164

185-
DataBlockSupplier<byte[]> blocks = (gridPos, existing) -> {
186-
return createDataBlock(dataBlockSize, gridPos.clone(), (byte) gridPos[0]);
165+
DataBlockSupplier<byte[]> chunksDelete = (gridPos, existing) -> {
166+
return null;
187167
};
188168

169+
// write one chunk at grid Position 1
170+
datasetAccess.writeRegion(store,
171+
new long[] {3},
172+
new long[] {3},
173+
chunks,
174+
false);
175+
176+
checkChunk(datasetAccess.readChunk(store, new long[] {0}), false, 0);
177+
checkChunk(datasetAccess.readChunk(store, new long[] {1}), true, 1);
178+
checkChunk(datasetAccess.readChunk(store, new long[] {2}), false, 0);
179+
checkChunk(datasetAccess.readChunk(store, new long[] {3}), false, 0);
180+
checkChunk(datasetAccess.readChunk(store, new long[] {4}), false, 0);
181+
182+
// only the first shard should exist
183+
checkKey(store, new long[]{0}, true);
184+
checkKey(store, new long[]{1}, false);
185+
checkKey(store, new long[]{2}, false);
186+
checkKey(store, new long[]{3}, false);
187+
188+
// write two chunks at grid positions 2 and 3
189+
datasetAccess.writeRegion(store,
190+
new long[]{6},
191+
new long[]{6},
192+
chunks,
193+
false);
194+
195+
checkChunk(datasetAccess.readChunk(store, new long[]{0}), false, 0);
196+
checkChunk(datasetAccess.readChunk(store, new long[]{1}), true, 1);
197+
checkChunk(datasetAccess.readChunk(store, new long[]{2}), true, 2);
198+
checkChunk(datasetAccess.readChunk(store, new long[]{3}), true, 3);
199+
checkChunk(datasetAccess.readChunk(store, new long[]{4}), false, 0);
200+
201+
// delete two chunks at grid positions 3 and 4
202+
datasetAccess.writeRegion(store,
203+
new long[]{9},
204+
new long[]{6},
205+
chunksDelete,
206+
false);
207+
208+
checkChunk(datasetAccess.readChunk(store, new long[]{0}), false, 0);
209+
checkChunk(datasetAccess.readChunk(store, new long[]{1}), true, 1);
210+
checkChunk(datasetAccess.readChunk(store, new long[]{2}), true, 2);
211+
checkChunk(datasetAccess.readChunk(store, new long[]{3}), false, 0);
212+
checkChunk(datasetAccess.readChunk(store, new long[]{4}), false, 0);
213+
214+
// overwrite all chunks to hold 255
189215
datasetAccess.writeRegion(store,
190-
regionMin,
191-
regionSize,
192-
blocks,
216+
new long[]{0},
217+
new long[]{96},
218+
chunks255,
193219
false);
194220

221+
for (int i = 0; i < numChunks; i++) {
222+
checkChunk(datasetAccess.readChunk(store, new long[]{i}), true, 255);
223+
}
224+
225+
// all shards should exist
226+
checkKey(store, new long[]{0}, true);
227+
checkKey(store, new long[]{1}, true);
228+
checkKey(store, new long[]{2}, true);
229+
checkKey(store, new long[]{3}, true);
230+
231+
// delete some chunks
232+
datasetAccess.writeRegion(store,
233+
new long[]{18},
234+
new long[]{18},
235+
chunksDelete,
236+
false);
237+
238+
checkChunk(datasetAccess.readChunk(store, new long[]{5}), true, 255);
239+
checkChunk(datasetAccess.readChunk(store, new long[]{6}), false, 0);
240+
241+
// all shards should exist
242+
checkKey(store, new long[]{0}, true);
243+
checkKey(store, new long[]{1}, true);
244+
checkKey(store, new long[]{2}, true);
245+
checkKey(store, new long[]{3}, true);
246+
247+
// delete more chunks so that shard 1 is empty
248+
datasetAccess.writeRegion(store,
249+
new long[]{36},
250+
new long[]{15},
251+
chunksDelete,
252+
false);
253+
254+
// shard 1 should be gone
255+
checkKey(store, new long[]{0}, true);
256+
checkKey(store, new long[]{1}, false);
257+
checkKey(store, new long[]{2}, true);
258+
checkKey(store, new long[]{3}, true);
195259
}
196260

197-
private static void checkBlock(final DataBlock<byte[]> dataBlock, final boolean expectedNonNull, final int expectedFillValue) {
261+
private static void checkChunk(final DataBlock<byte[]> chunk, final boolean expectedNonNull, final int expectedFillValue) {
198262

199-
if (dataBlock == null) {
263+
if (chunk == null) {
200264
if (expectedNonNull) {
201265
throw new IllegalStateException("expected non-null dataBlock");
202266
}
203267
} else {
204268
if (!expectedNonNull) {
205269
throw new IllegalStateException("expected null dataBlock");
206270
}
207-
final byte[] bytes = dataBlock.getData();
271+
final byte[] bytes = chunk.getData();
208272
for (byte b : bytes) {
209273
if (b != (byte) expectedFillValue) {
210274
throw new IllegalStateException("expected all values to be " + expectedFillValue);
@@ -213,6 +277,17 @@ private static void checkBlock(final DataBlock<byte[]> dataBlock, final boolean
213277
}
214278
}
215279

280+
private static void checkKey(PositionValueAccess pva, long[] position, final boolean expectedNonNull) {
281+
282+
try (VolatileReadData val = pva.get(position)) {
283+
284+
if (expectedNonNull && val == null)
285+
throw new IllegalStateException("expected non-null value");
286+
else if (!expectedNonNull && val != null)
287+
throw new IllegalStateException("expected null value");
288+
}
289+
}
290+
216291
private static DataBlock<byte[]> createDataBlock(int[] size, long[] gridPosition, int fillValue) {
217292
final byte[] bytes = new byte[DataBlock.getNumElements(size)];
218293
Arrays.fill(bytes, (byte) fillValue);

0 commit comments

Comments
 (0)