Skip to content

Commit 14d50df

Browse files
cmhulbertcmhulbert
committed
refactor(test): migrate remaining uses of KVA#lockForReading/Writing. Extract and generify TrackingKeyValueAccess to be more reusable
Signed-off-by: Caleb Hulbert <cmhulbert@gmail.com>
1 parent 0d55709 commit 14d50df

7 files changed

Lines changed: 237 additions & 224 deletions

File tree

‎src/test/java/org/janelia/saalfeldlab/n5/TrackingN5Writer.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package org.janelia.saalfeldlab.n5;
2+
3+
import com.google.gson.GsonBuilder;
4+
import org.janelia.saalfeldlab.n5.kva.TrackingKeyValueAccess;
5+
6+
/**
7+
* An N5Writer that tracks the number of materialize calls performed by
8+
* its underlying key value access.
9+
*/
10+
public class TrackingN5Writer extends N5KeyValueWriter {
11+
12+
public final TrackingKeyValueAccess tkva;
13+
14+
public TrackingN5Writer(String basePath, KeyValueAccess kva) {
15+
16+
super(new TrackingKeyValueAccess(kva), basePath, new GsonBuilder(), false);
17+
this.tkva = (TrackingKeyValueAccess) getKeyValueAccess();
18+
}
19+
20+
public void resetNumMaterializeCalls() {
21+
tkva.numMaterializeCalls = 0;
22+
}
23+
24+
public int getNumMaterializeCalls() {
25+
return tkva.numMaterializeCalls;
26+
}
27+
28+
public void resetNumIsFileCalls() {
29+
tkva.numIsFileCalls = 0;
30+
}
31+
32+
public int getNumIsFileCalls() {
33+
return tkva.numIsFileCalls;
34+
}
35+
36+
public void resetTotalBytesRead() {
37+
tkva.totalBytesRead = 0;
38+
}
39+
40+
public long getTotalBytesRead() {
41+
return tkva.totalBytesRead;
42+
}
43+
44+
public void resetAllTracking() {
45+
tkva.numMaterializeCalls = 0;
46+
tkva.numIsFileCalls = 0;
47+
tkva.totalBytesRead = 0;
48+
}
49+
}

‎src/test/java/org/janelia/saalfeldlab/n5/backward/CompatibilityTest.java‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,8 @@
4040
import java.nio.file.Files;
4141
import java.util.Arrays;
4242

43-
import org.janelia.saalfeldlab.n5.DataBlock;
44-
import org.janelia.saalfeldlab.n5.DatasetAttributes;
45-
import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader;
46-
import org.janelia.saalfeldlab.n5.KeyValueAccess;
47-
import org.janelia.saalfeldlab.n5.LockedChannel;
48-
import org.janelia.saalfeldlab.n5.N5FSReader;
49-
import org.janelia.saalfeldlab.n5.N5FSWriter;
50-
import org.janelia.saalfeldlab.n5.RawCompression;
43+
import org.janelia.saalfeldlab.n5.*;
44+
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
5145
import org.junit.Test;
5246

5347
import com.google.gson.JsonElement;
@@ -148,13 +142,10 @@ public void testBlockData() throws IOException {
148142

149143
private byte[] read(KeyValueAccess kva, String path) {
150144

151-
int N = (int)kva.size(path);
152-
byte[] data = new byte[N];
153-
try (LockedChannel ch = kva.lockForReading(path);
154-
InputStream is = ch.newInputStream();) {
155-
156-
is.read(data);
157-
} catch (IOException e) {
145+
byte[] data;
146+
try (VolatileReadData readData = kva.createReadData(path)) {
147+
data = readData.allBytes();
148+
} catch (N5Exception.N5IOException e) {
158149
return null;
159150
}
160151
return data;

‎src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess;
4242
import org.janelia.saalfeldlab.n5.KeyValueAccess;
4343
import org.janelia.saalfeldlab.n5.LockedChannel;
44+
import org.janelia.saalfeldlab.n5.N5Exception;
45+
import org.janelia.saalfeldlab.n5.readdata.ReadData;
4446
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
4547
import org.openjdk.jmh.annotations.Benchmark;
4648
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -125,14 +127,16 @@ protected void write(Path path, int numBytes) {
125127
final byte[] data = new byte[numBytes];
126128
random.nextBytes(data);
127129

128-
System.out.println(path.toAbsolutePath().toString());
130+
System.out.println(path.toAbsolutePath());
129131
System.out.println(numBytes);
130-
try (final LockedChannel ch = kva.lockForWriting(path.toAbsolutePath().toString())) {
131-
final OutputStream os = ch.newOutputStream();
132+
133+
ReadData readData = ReadData.from(os -> {
132134
os.write(data);
133-
os.flush();
134-
os.close();
135-
} catch (final IOException e) {
135+
}).materialize();
136+
137+
try {
138+
kva.write(path.toAbsolutePath().toString(), readData);
139+
} catch (N5Exception.N5IOException e) {
136140
e.printStackTrace();
137141
}
138142
}

‎src/test/java/org/janelia/saalfeldlab/n5/http/HttpKeyValueAccessTest.java‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,15 @@
3232
import org.janelia.saalfeldlab.n5.HttpKeyValueAccess;
3333
import org.janelia.saalfeldlab.n5.LockedChannel;
3434
import org.janelia.saalfeldlab.n5.N5Exception;
35+
import org.janelia.saalfeldlab.n5.readdata.ReadData;
36+
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
3537
import org.junit.Test;
3638

3739
import java.io.IOException;
3840
import java.io.InputStream;
3941
import java.net.URI;
4042
import java.nio.charset.Charset;
43+
import java.util.function.Function;
4144

4245
import static org.junit.Assert.assertEquals;
4346
import static org.junit.Assert.assertThrows;
@@ -58,16 +61,9 @@ public void testExistsRead() {
5861
final String absolutePath = kva.compose(baseUrl, key);
5962
assumeTrue(kva.exists(absolutePath));
6063

61-
try (LockedChannel ch = kva.lockForReading(absolutePath)) {
62-
63-
final InputStream is = ch.newInputStream();
64-
65-
final String attributes = IOUtils.toString(is, Charset.defaultCharset());
66-
assertEquals(expectedAttributes, attributes);
67-
68-
is.close();
69-
70-
} catch (IOException e) {
64+
try (VolatileReadData data = kva.createReadData(absolutePath)) {
65+
IOUtils.toString(data.inputStream(), Charset.defaultCharset());
66+
} catch (IOException e) {
7167
// not correct to fail for an IO exception
7268
e.printStackTrace();
7369
}
@@ -78,7 +74,7 @@ public void testUnsupportedOperations() {
7874

7975
final HttpKeyValueAccess kva = new HttpKeyValueAccess();
8076
assertThrows(N5Exception.class, () -> kva.delete("foo"));
81-
assertThrows(N5Exception.class, () -> kva.lockForWriting("bar"));
77+
assertThrows(N5Exception.class, () -> kva.write("bar", ReadData.from(os -> {})));
8278
}
8379

8480
}

‎src/test/java/org/janelia/saalfeldlab/n5/kva/DelegateKeyValueAccess.java‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
package org.janelia.saalfeldlab.n5.kva;
2+
3+
import org.janelia.saalfeldlab.n5.KeyValueAccess;
4+
import org.janelia.saalfeldlab.n5.N5Exception;
5+
import org.janelia.saalfeldlab.n5.readdata.ReadData;
6+
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
7+
8+
import java.net.URI;
9+
import java.net.URISyntaxException;
10+
11+
public class DelegateKeyValueAccess implements KeyValueAccess {
12+
13+
protected final KeyValueAccess kva;
14+
15+
public DelegateKeyValueAccess(KeyValueAccess kva) { this.kva = kva; }
16+
17+
@Override
18+
public String[] components(String path) {
19+
return kva.components(path);
20+
}
21+
22+
@Override
23+
public String compose(URI uri, String... components) {
24+
return kva.compose(uri, components);
25+
}
26+
27+
@Override
28+
public String compose(String... components) {
29+
return kva.compose(components);
30+
}
31+
32+
@Override
33+
public String parent(String path) {
34+
return kva.parent(path);
35+
}
36+
37+
@Override
38+
public String relativize(String path, String base) {
39+
return kva.relativize(path, base);
40+
}
41+
42+
@Override
43+
public String normalize(String path) {
44+
return kva.normalize(path);
45+
}
46+
47+
@Override
48+
public URI uri(String uriString) throws URISyntaxException {
49+
return kva.uri(uriString);
50+
}
51+
52+
@Override
53+
public boolean exists(String normalPath) {
54+
return kva.exists(normalPath);
55+
}
56+
57+
@Override
58+
public long size(String normalPath) throws N5Exception.N5NoSuchKeyException {
59+
return kva.size(normalPath);
60+
}
61+
62+
@Override
63+
public boolean isDirectory(String normalPath) {
64+
return kva.isDirectory(normalPath);
65+
}
66+
67+
@Override
68+
public boolean isFile(String normalPath) {
69+
return kva.isFile(normalPath);
70+
}
71+
72+
@Override
73+
public VolatileReadData createReadData(String normalPath) throws N5Exception.N5IOException {
74+
return kva.createReadData(normalPath);
75+
}
76+
77+
@Override
78+
public void write(String normalPath, ReadData data) throws N5Exception.N5IOException {
79+
kva.write( normalPath, data);
80+
}
81+
82+
@Override
83+
public String[] listDirectories(String normalPath) throws N5Exception.N5IOException {
84+
return kva.listDirectories(normalPath);
85+
}
86+
87+
@Override
88+
public String[] list(String normalPath) throws N5Exception.N5IOException {
89+
return kva.list(normalPath);
90+
}
91+
92+
@Override
93+
public void createDirectories(String normalPath) throws N5Exception.N5IOException {
94+
kva.createDirectories(normalPath);
95+
96+
}
97+
98+
@Override
99+
public void delete(String normalPath) throws N5Exception.N5IOException {
100+
kva.delete(normalPath);
101+
}
102+
}

‎src/test/java/org/janelia/saalfeldlab/n5/kva/TrackingKeyValueAccess.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package org.janelia.saalfeldlab.n5.kva;
2+
3+
import org.janelia.saalfeldlab.n5.KeyValueAccess;
4+
import org.janelia.saalfeldlab.n5.N5Exception;
5+
import org.janelia.saalfeldlab.n5.readdata.LazyRead;
6+
import org.janelia.saalfeldlab.n5.readdata.ReadData;
7+
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
8+
import org.janelia.saalfeldlab.n5.shard.ShardTest;
9+
10+
public class TrackingKeyValueAccess extends DelegateKeyValueAccess {
11+
12+
public int numMaterializeCalls = 0;
13+
public int numIsFileCalls = 0;
14+
public long totalBytesRead = 0;
15+
16+
public TrackingKeyValueAccess(final KeyValueAccess kva) {
17+
super(kva);
18+
}
19+
20+
@Override
21+
public boolean isFile(String normalPath) {
22+
numIsFileCalls++;
23+
return kva.isFile(normalPath);
24+
}
25+
26+
@Override
27+
public VolatileReadData createReadData(final String normalPath) {
28+
// throw new N5NoSuchKeyException("Test No Such Key");
29+
return VolatileReadData.from(new TrackingVolatileReadData(kva.createReadData(normalPath)));
30+
}
31+
32+
private class TrackingVolatileReadData implements LazyRead {
33+
34+
private final VolatileReadData readData;
35+
36+
TrackingVolatileReadData(final VolatileReadData readData) {
37+
this.readData = readData;
38+
}
39+
40+
@Override
41+
public long size() throws N5Exception.N5IOException {
42+
43+
return readData.requireLength();
44+
}
45+
46+
@Override
47+
public ReadData materialize(final long offset, final long length) {
48+
49+
numMaterializeCalls++;
50+
return readData.slice(offset, length).materialize();
51+
}
52+
53+
@Override
54+
public void close() {
55+
56+
readData.close();
57+
}
58+
}
59+
}

0 commit comments

Comments
 (0)