Skip to content

Commit cd1fbb9

Browse files
chris-allanchris-allan
committed
Expand benchmark test to include read performance
1 parent 3135da2 commit cd1fbb9

1 file changed

Lines changed: 101 additions & 12 deletions

File tree

‎src/test/java/org/janelia/saalfeldlab/n5/N5BenchmarkTest.java‎

Lines changed: 101 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.util.concurrent.Executors;
2727
import java.util.concurrent.Future;
2828

29+
import org.junit.Assert;
2930
import org.junit.Before;
3031
import org.junit.Rule;
3132
import org.junit.Test;
@@ -56,11 +57,11 @@ public class N5BenchmarkTest {
5657
@Rule
5758
public TemporaryFolder temp = new TemporaryFolder();
5859

59-
private static String datasetName = "/dataset";
60+
private static final String datasetName = "/dataset";
6061

61-
private static N5Writer n5;
62+
private N5Writer n5;
6263

63-
private static short[] data;
64+
private short[] data;
6465

6566
private static final Compression[] compressions = {
6667
new RawCompression(),
@@ -194,10 +195,10 @@ public void benchmarkParallelWritingSpeed() {
194195

195196
final ExecutorService exec = Executors.newFixedThreadPool(i);
196197
final ArrayList<Future<Boolean>> futures = new ArrayList<>();
197-
long t;
198+
long start, read, write;
198199

199200
for (final Compression compression : compressions) {
200-
t = System.currentTimeMillis();
201+
start = System.currentTimeMillis();
201202
try {
202203
final String compressedDatasetName = datasetName + "." + compression.getType();
203204
n5.createDataset(compressedDatasetName, new long[]{64 * nBlocks, 64 * nBlocks, 64 * nBlocks}, new int[]{64, 64, 64}, DataType.UINT16, compression);
@@ -220,16 +221,38 @@ public void benchmarkParallelWritingSpeed() {
220221
}
221222
for (final Future<Boolean> f : futures)
222223
f.get();
224+
write = System.currentTimeMillis();
223225

224-
System.out.println(String.format("%d : %s : %fs", i, compression.getType(), 0.001 * (System.currentTimeMillis() - t)));
226+
for (int z = 0; z < nBlocks; ++z) {
227+
final int fz = z;
228+
for (int y = 0; y < nBlocks; ++y) {
229+
final int fy = y;
230+
for (int x = 0; x < nBlocks; ++x) {
231+
final int fx = x;
232+
futures.add(
233+
exec.submit(
234+
() -> {
235+
final ShortArrayDataBlock dataBlock = (ShortArrayDataBlock) n5.readBlock(compressedDatasetName, attributes, new long[]{fx, fy, fz});
236+
Assert.assertArrayEquals(new int[]{64, 64, 64}, dataBlock.size);
237+
return true;
238+
}));
239+
}
240+
}
241+
}
242+
for (final Future<Boolean> f : futures)
243+
f.get();
244+
read = System.currentTimeMillis();
245+
246+
System.out.println(String.format("%d : %s write : %fs", i, compression.getType(), 0.001 * (write - start)));
247+
System.out.println(String.format("%d : %s read : %fs", i, compression.getType(), 0.001 * (read - write)));
225248
} catch (final IOException | InterruptedException | ExecutionException e) {
226249
fail(e.getMessage());
227250
}
228251
}
229252

230253
/* TIF blocks */
231254
futures.clear();
232-
t = System.currentTimeMillis();
255+
start = System.currentTimeMillis();
233256
try {
234257
for (int z = 0; z < nBlocks; ++z) {
235258
final int fz = z;
@@ -249,15 +272,37 @@ public void benchmarkParallelWritingSpeed() {
249272
}
250273
for (final Future<Boolean> f : futures)
251274
f.get();
275+
write = System.currentTimeMillis();
252276

253-
System.out.println(String.format("%d : tif : %fs", i, 0.001 * (System.currentTimeMillis() - t)));
277+
for (int z = 0; z < nBlocks; ++z) {
278+
final int fz = z;
279+
for (int y = 0; y < nBlocks; ++y) {
280+
final int fy = y;
281+
for (int x = 0; x < nBlocks; ++x) {
282+
final int fx = x;
283+
futures.add(
284+
exec.submit(
285+
() -> {
286+
final ImagePlus impBlock = new Opener().openTiff(temp.getRoot().toPath().resolve(fx + "-" + fy + "-" + fz + ".tif").toString(), 1);
287+
Assert.assertArrayEquals(new int[]{64, 64 * 64, 1, 1, 1}, impBlock.getDimensions());
288+
return true;
289+
}));
290+
}
291+
}
292+
}
293+
for (final Future<Boolean> f : futures)
294+
f.get();
295+
read = System.currentTimeMillis();
296+
297+
System.out.println(String.format("%d : tif write : %fs", i, 0.001 * (write - start)));
298+
System.out.println(String.format("%d : tif read : %fs", i, 0.001 * (read - write)));
254299
} catch (final InterruptedException | ExecutionException e) {
255300
fail(e.getMessage());
256301
}
257302

258303
/* HDF5 raw */
259304
futures.clear();
260-
t = System.currentTimeMillis();
305+
start = System.currentTimeMillis();
261306
try {
262307
final String hdf5Name = temp.getRoot().toPath().resolve("dataset.h5").toString();
263308
final IHDF5Writer hdf5Writer = HDF5Factory.open( hdf5Name );
@@ -285,16 +330,38 @@ public void benchmarkParallelWritingSpeed() {
285330
}
286331
for (final Future<Boolean> f : futures)
287332
f.get();
333+
write = System.currentTimeMillis();
334+
335+
for (int z = 0; z < nBlocks; ++z) {
336+
final int fz = z;
337+
for (int y = 0; y < nBlocks; ++y) {
338+
final int fy = y;
339+
for (int x = 0; x < nBlocks; ++x) {
340+
final int fx = x;
341+
futures.add(
342+
exec.submit(
343+
() -> {
344+
final MDShortArray targetCell = uint16Writer.readMDArrayBlockWithOffset(datasetName, new int[]{64, 64, 64}, new long[]{64 * fz, 64 * fy, 64 * fx});
345+
Assert.assertArrayEquals(new int[]{64, 64, 64}, targetCell.dimensions());
346+
return true;
347+
}));
348+
}
349+
}
350+
}
351+
for (final Future<Boolean> f : futures)
352+
f.get();
353+
read = System.currentTimeMillis();
288354

289355
hdf5Writer.close();
290-
System.out.println(String.format("%d : hdf5 raw : %fs", i, 0.001 * (System.currentTimeMillis() - t)));
356+
System.out.println(String.format("%d : hdf5 raw write : %fs", i, 0.001 * (write - start)));
357+
System.out.println(String.format("%d : hdf5 raw read : %fs", i, 0.001 * (read - write)));
291358
} catch (final InterruptedException | ExecutionException e) {
292359
fail(e.getMessage());
293360
}
294361

295362
/* HDF5 gzip */
296363
futures.clear();
297-
t = System.currentTimeMillis();
364+
start = System.currentTimeMillis();
298365
try {
299366
final String hdf5Name = temp.getRoot().toPath().resolve("dataset.gz.h5").toString();
300367
final IHDF5Writer hdf5Writer = HDF5Factory.open( hdf5Name );
@@ -322,9 +389,31 @@ public void benchmarkParallelWritingSpeed() {
322389
}
323390
for (final Future<Boolean> f : futures)
324391
f.get();
392+
write = System.currentTimeMillis();
393+
394+
for (int z = 0; z < nBlocks; ++z) {
395+
final int fz = z;
396+
for (int y = 0; y < nBlocks; ++y) {
397+
final int fy = y;
398+
for (int x = 0; x < nBlocks; ++x) {
399+
final int fx = x;
400+
futures.add(
401+
exec.submit(
402+
() -> {
403+
final MDShortArray targetCell = uint16Writer.readMDArrayBlockWithOffset(datasetName, new int[]{64, 64, 64}, new long[]{64 * fz, 64 * fy, 64 * fx});
404+
Assert.assertArrayEquals(new int[]{64, 64, 64}, targetCell.dimensions());
405+
return true;
406+
}));
407+
}
408+
}
409+
}
410+
for (final Future<Boolean> f : futures)
411+
f.get();
412+
read = System.currentTimeMillis();
325413

326414
hdf5Writer.close();
327-
System.out.println(String.format("%d : hdf5 gzip : %fs", i, 0.001 * (System.currentTimeMillis() - t)));
415+
System.out.println(String.format("%d : hdf5 gzip write : %fs", i, 0.001 * (write - start)));
416+
System.out.println(String.format("%d : hdf5 gzip read : %fs", i, 0.001 * (read - write)));
328417
} catch (final InterruptedException | ExecutionException e) {
329418
fail(e.getMessage());
330419
}

0 commit comments

Comments
 (0)