Skip to content

Commit b59b1a7

Browse files
cmhulberttpietzsch
cmhulbert
authored and
tpietzsch
committed
Merge branch 'master' into wip/codecs
2 parents f8ba742 + 3955cdf commit b59b1a7

3 files changed

Lines changed: 213 additions & 2 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,6 @@ Custom compression schemes can be implemented using the annotation discovery mec
132132
133133
## Disclaimer
134134
135-
HDF5 is a great format that provides a wealth of conveniences that I do not want to miss. It's inefficiency for parallel writing, however, limit its applicability for handling of very large n-dimensional data.
135+
HDF5 is a great format that provides a wealth of conveniences that I do not want to miss. Its inefficiency for parallel writing, however, limits its applicability for handling of very large n-dimensional data.
136136
137137
N5 uses the native filesystem of the target platform and JSON files to specify basic and custom meta-data as attributes. It aims at preserving the convenience of HDF5 where possible but doesn't try too hard to be a full replacement.

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
<groupId>org.janelia.saalfeldlab</groupId>
1313
<artifactId>n5</artifactId>
14-
<version>4.0.0-alpha-2-SNAPSHOT</version>
14+
<version>4.0.0-alpha-3-SNAPSHOT</version>
1515

1616
<name>N5</name>
1717
<description>Not HDF5</description>

‎src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
package org.janelia.saalfeldlab.n5.benchmarks;
2+
3+
import java.io.File;
4+
import java.io.IOException;
5+
import java.nio.file.FileSystems;
6+
import java.nio.file.Files;
7+
import java.util.ArrayList;
8+
import java.util.Arrays;
9+
import java.util.Random;
10+
import java.util.concurrent.TimeUnit;
11+
12+
import org.janelia.saalfeldlab.n5.DataBlock;
13+
import org.janelia.saalfeldlab.n5.DataType;
14+
import org.janelia.saalfeldlab.n5.DatasetAttributes;
15+
import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess;
16+
import org.janelia.saalfeldlab.n5.GzipCompression;
17+
import org.janelia.saalfeldlab.n5.N5KeyValueWriter;
18+
import org.janelia.saalfeldlab.n5.N5Writer;
19+
import org.openjdk.jmh.annotations.Benchmark;
20+
import org.openjdk.jmh.annotations.BenchmarkMode;
21+
import org.openjdk.jmh.annotations.Fork;
22+
import org.openjdk.jmh.annotations.Level;
23+
import org.openjdk.jmh.annotations.Measurement;
24+
import org.openjdk.jmh.annotations.Mode;
25+
import org.openjdk.jmh.annotations.OutputTimeUnit;
26+
import org.openjdk.jmh.annotations.Param;
27+
import org.openjdk.jmh.annotations.Scope;
28+
import org.openjdk.jmh.annotations.Setup;
29+
import org.openjdk.jmh.annotations.State;
30+
import org.openjdk.jmh.annotations.TearDown;
31+
import org.openjdk.jmh.annotations.Warmup;
32+
import org.openjdk.jmh.infra.Blackhole;
33+
import org.openjdk.jmh.runner.Runner;
34+
import org.openjdk.jmh.runner.RunnerException;
35+
import org.openjdk.jmh.runner.options.Options;
36+
import org.openjdk.jmh.runner.options.OptionsBuilder;
37+
38+
import com.google.gson.GsonBuilder;
39+
40+
@State(Scope.Benchmark)
41+
@Warmup(iterations = 5, time = 100, timeUnit = TimeUnit.MICROSECONDS)
42+
@Measurement(iterations = 50, time = 100, timeUnit = TimeUnit.MICROSECONDS)
43+
@BenchmarkMode(Mode.AverageTime)
44+
@OutputTimeUnit(TimeUnit.MICROSECONDS)
45+
@Fork(1)
46+
public class N5BlockWriteBenchmarks {
47+
48+
Random random = new Random(7777);
49+
50+
final String writeGroup = "writeGroup";
51+
final String readGroup = "readGroup";
52+
53+
N5Writer n5;
54+
DatasetAttributes dsetAttrs;
55+
ArrayList<DataBlock<?>> blocks;
56+
57+
@Param( value = { "int32" } )
58+
protected String dataType;
59+
60+
@Param( value = { "3" } )
61+
protected int numDimensions;
62+
63+
@Param( value = { "64" } )
64+
protected int blockDim;
65+
66+
@Param( value = { "5" } )
67+
protected int numBlocks;
68+
69+
public static void main( String[] args ) throws RunnerException {
70+
71+
final Options options = new OptionsBuilder().include( N5BlockWriteBenchmarks.class.getSimpleName() + "\\." ).build();
72+
new Runner(options).run();
73+
}
74+
75+
@TearDown(Level.Trial)
76+
public void teardown() {
77+
File d = new File(n5.getURI());
78+
n5.remove();
79+
d.delete();
80+
}
81+
82+
@Setup(Level.Trial)
83+
public void setup() {
84+
85+
File tmpDir;
86+
try {
87+
tmpDir = Files.createTempDirectory("n5-blockWriteBenchmark-").toFile();
88+
FileSystemKeyValueAccess kva = new FileSystemKeyValueAccess(FileSystems.getDefault());
89+
n5 = new N5KeyValueWriter(kva, tmpDir.getAbsolutePath(), new GsonBuilder(), true);
90+
91+
int[] blockSize = new int[numDimensions];
92+
Arrays.fill(blockSize, blockDim);
93+
94+
long[] dims = new long[numDimensions];
95+
Arrays.fill(dims, blockDim);
96+
dims[0] = blockDim * numBlocks;
97+
98+
DataType dtype = DataType.fromString(dataType);
99+
100+
dsetAttrs = new DatasetAttributes(dims, blockSize, dtype, new GzipCompression());
101+
n5.createDataset("", dsetAttrs);
102+
103+
blocks = new ArrayList<>();
104+
for (int i = 0; i < numBlocks; i++) {
105+
long[] p = new long[numDimensions];
106+
p[0] = i;
107+
108+
DataBlock<?> blk = dtype.createDataBlock(blockSize, p);
109+
fillBlock(dtype, blk);
110+
blocks.add(blk);
111+
112+
// write data into the read group
113+
n5.writeBlock(readGroup, dsetAttrs, blk);
114+
}
115+
116+
} catch (final IOException e) {
117+
e.printStackTrace();
118+
}
119+
120+
}
121+
122+
@Benchmark
123+
public void writeBenchmark() throws IOException {
124+
125+
blocks.forEach(blk -> {
126+
n5.writeBlock(writeGroup, dsetAttrs, blk);
127+
});
128+
}
129+
130+
@Benchmark
131+
public void readBenchmark(Blackhole hole) throws IOException {
132+
133+
final long[] p = new long[numDimensions];
134+
for (int i = 0; i < numBlocks; i++) {
135+
p[0] = i;
136+
hole.consume(n5.readBlock(readGroup, dsetAttrs, p));
137+
}
138+
}
139+
140+
private void fillBlock(DataType dtype, DataBlock<?> blk) {
141+
142+
switch (dtype) {
143+
case INT32:
144+
fill((int[])blk.getData());
145+
break;
146+
case FLOAT32:
147+
fill((float[])blk.getData());
148+
break;
149+
case FLOAT64:
150+
fill((double[])blk.getData());
151+
break;
152+
case INT16:
153+
fill((short[])blk.getData());
154+
break;
155+
case INT64:
156+
fill((long[])blk.getData());
157+
break;
158+
case INT8:
159+
fill((byte[])blk.getData());
160+
break;
161+
case OBJECT:
162+
break;
163+
case STRING:
164+
break;
165+
case UINT16:
166+
fill((short[])blk.getData());
167+
break;
168+
case UINT32:
169+
fill((int[])blk.getData());
170+
break;
171+
case UINT64:
172+
fill((long[])blk.getData());
173+
break;
174+
case UINT8:
175+
fill((byte[])blk.getData());
176+
break;
177+
default:
178+
break;
179+
}
180+
}
181+
182+
private void fill(short[] arr) {
183+
for (int i = 0; i < arr.length; i++)
184+
arr[i] = (short)random.nextInt();
185+
}
186+
187+
private void fill(int[] arr) {
188+
for (int i = 0; i < arr.length; i++)
189+
arr[i] = random.nextInt();
190+
}
191+
192+
private void fill(long[] arr) {
193+
for (int i = 0; i < arr.length; i++)
194+
arr[i] = random.nextLong();
195+
}
196+
197+
private void fill(float[] arr) {
198+
for (int i = 0; i < arr.length; i++)
199+
arr[i] = random.nextFloat();
200+
}
201+
202+
private void fill(double[] arr) {
203+
for (int i = 0; i < arr.length; i++)
204+
arr[i] = random.nextDouble();
205+
}
206+
207+
private void fill(byte[] arr) {
208+
random.nextBytes(arr);
209+
}
210+
211+
}

0 commit comments

Comments
 (0)