Skip to content

Commit 85667f2

Browse files
cmhulbertcmhulbert
authored
Merge pull request #38 from saalfeldlab/n5-deps
N5 deps
2 parents 3179478 + 8fffdb4 commit 85667f2

8 files changed

Lines changed: 188 additions & 10 deletions

File tree

‎.github/workflows/build-main.yml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,17 @@ jobs:
1212
runs-on: ubuntu-latest
1313

1414
steps:
15-
- uses: actions/checkout@v2
15+
- uses: actions/checkout@v4
1616
- name: Install blosc
1717
run: |
1818
sudo apt-get update
1919
sudo apt-get install -y libblosc1
20+
- name: Setup Python
21+
uses: actions/setup-python@v5
22+
with:
23+
python-version: '3.10'
2024
- name: Set up Java
21-
uses: actions/setup-java@v3
25+
uses: actions/setup-java@v4
2226
with:
2327
java-version: '8'
2428
distribution: 'zulu'

‎.github/workflows/build-pr.yml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,17 @@ jobs:
1010
runs-on: ubuntu-latest
1111

1212
steps:
13-
- uses: actions/checkout@v2
13+
- uses: actions/checkout@v4
1414
- name: Install blosc
1515
run: |
1616
sudo apt-get update
1717
sudo apt-get install -y libblosc1
18+
- name: Setup Python
19+
uses: actions/setup-python@v5
20+
with:
21+
python-version: '3.10'
1822
- name: Set up Java
19-
uses: actions/setup-java@v3
23+
uses: actions/setup-java@v4
2024
with:
2125
java-version: '8'
2226
distribution: 'zulu'

‎.github/workflows/platform-test.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ jobs:
1919
runs-on: ${{ matrix.os }}
2020

2121
steps:
22-
- uses: actions/checkout@v2
22+
- uses: actions/checkout@v4
2323
- name: Setup Python
24-
uses: actions/setup-python@v4
24+
uses: actions/setup-python@v5
2525
with:
2626
python-version: '3.10'
2727
- name: Install blosc (Windows)
@@ -35,7 +35,7 @@ jobs:
3535
pip install blosc --no-input --target src/test/resources
3636
mv src/test/resources/lib64/* src/test/resources
3737
- name: Set up Java
38-
uses: actions/setup-java@v2
38+
uses: actions/setup-java@v4
3939
with:
4040
java-version: '8'
4141
distribution: 'zulu'

‎pom.xml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,10 +111,10 @@
111111
<!-- NB: Deploy releases to the SciJava Maven repository. -->
112112
<releaseProfiles>sign,deploy-to-scijava</releaseProfiles>
113113

114-
<n5.version>3.4.1</n5.version>
114+
<n5.version>3.5.0</n5.version>
115115
<n5-ij.version>4.3.0</n5-ij.version>
116-
<n5-aws-s3.version>4.2.2</n5-aws-s3.version>
117-
<n5-google-cloud.version>5.0.0</n5-google-cloud.version>
116+
<n5-aws-s3.version>4.3.0</n5-aws-s3.version>
117+
<n5-google-cloud.version>5.1.0</n5-google-cloud.version>
118118
<n5-zarr.version>1.5.1</n5-zarr.version>
119119

120120
<jackson-jq.version>1.0.0-preview.20191208</jackson-jq.version>

‎src/test/java/org/janelia/saalfeldlab/n5/universe/storage/n5/N5GoogleCloudFactoryTest.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
package org.janelia.saalfeldlab.n5.universe.storage.n5;
22

33
import com.google.cloud.storage.Storage;
4+
import org.janelia.saalfeldlab.n5.DataType;
5+
import org.janelia.saalfeldlab.n5.DatasetAttributes;
6+
import org.janelia.saalfeldlab.n5.N5Exception;
7+
import org.janelia.saalfeldlab.n5.N5Reader;
48
import org.janelia.saalfeldlab.n5.N5Writer;
9+
import org.janelia.saalfeldlab.n5.RawCompression;
510
import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess;
611
import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTests;
712
import org.janelia.saalfeldlab.n5.googlecloud.backend.BackendGoogleCloudStorageFactory;
@@ -15,6 +20,9 @@
1520
import java.net.URISyntaxException;
1621

1722
import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempContainerPath;
23+
import static org.junit.Assert.assertArrayEquals;
24+
import static org.junit.Assert.assertEquals;
25+
import static org.junit.Assert.assertThrows;
1826
import static org.junit.Assert.assertTrue;
1927

2028
public abstract class N5GoogleCloudFactoryTest extends N5StorageTests.N5FactoryTest {
@@ -63,6 +71,40 @@ public N5GoogleCloudMockTest() {
6371

6472
N5GoogleCloudFactoryTest.storage = MockGoogleCloudStorageFactory.getOrCreateStorage();
6573
}
74+
75+
@Override public void testPathsWithIllegalUriCharacters() {
76+
77+
// Omit '%' from illegal char test for mock test only. Works fine and is tested correctly against backend tests. See
78+
// n5-google-cloud test for more detailed info
79+
80+
try (N5Writer writer = createTempN5Writer()) {
81+
try (N5Reader reader = createN5Reader(writer.getURI().toString())) {
82+
83+
final String[] illegalChars = {" ", "#"};
84+
for (final String illegalChar : illegalChars) {
85+
final String groupWithIllegalChar = "test" + illegalChar + "group";
86+
assertThrows("list over group should throw prior to create", N5Exception.N5IOException.class, () -> writer.list(groupWithIllegalChar));
87+
writer.createGroup(groupWithIllegalChar);
88+
assertTrue("Newly created group should exist", writer.exists(groupWithIllegalChar));
89+
assertArrayEquals("list over empty group should be empty list", new String[0], writer.list(groupWithIllegalChar));
90+
writer.setAttribute(groupWithIllegalChar, "/a/b/key1", "value1");
91+
final String attrFromWriter = writer.getAttribute(groupWithIllegalChar, "/a/b/key1", String.class);
92+
final String attrFromReader = reader.getAttribute(groupWithIllegalChar, "/a/b/key1", String.class);
93+
assertEquals("value1", attrFromWriter);
94+
assertEquals("value1", attrFromReader);
95+
96+
97+
final String datasetWithIllegalChar = "test" + illegalChar + "dataset";
98+
final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64, new RawCompression());
99+
writer.createDataset(datasetWithIllegalChar, datasetAttributes);
100+
final DatasetAttributes datasetFromWriter = writer.getDatasetAttributes(datasetWithIllegalChar);
101+
final DatasetAttributes datasetFromReader = reader.getDatasetAttributes(datasetWithIllegalChar);
102+
assertDatasetAttributesEquals(datasetAttributes, datasetFromWriter);
103+
assertDatasetAttributesEquals(datasetAttributes, datasetFromReader);
104+
}
105+
}
106+
}
107+
}
66108
}
67109

68110
public static class N5GoogleCloudBackendTest extends N5GoogleCloudFactoryTest {

‎src/test/java/org/janelia/saalfeldlab/n5/universe/storage/n5/N5HttpFactoryTest.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader;
44
import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer;
55
import org.janelia.saalfeldlab.n5.HttpKeyValueAccess;
6+
import org.janelia.saalfeldlab.n5.N5Exception;
67
import org.janelia.saalfeldlab.n5.N5KeyValueWriter;
78
import org.janelia.saalfeldlab.n5.N5Reader;
89
import org.janelia.saalfeldlab.n5.N5Writer;
@@ -13,16 +14,20 @@
1314
import org.junit.After;
1415
import org.junit.AfterClass;
1516
import org.junit.Ignore;
17+
import org.junit.Test;
1618
import org.junit.runner.RunWith;
1719
import org.junit.runners.Parameterized;
1820

1921
import java.io.File;
2022
import java.io.IOException;
2123
import java.net.URI;
24+
import java.net.URISyntaxException;
2225
import java.nio.file.Files;
2326
import java.nio.file.Path;
2427
import java.util.ArrayList;
2528

29+
import static org.junit.Assert.assertNotNull;
30+
import static org.junit.Assert.assertThrows;
2631
import static org.junit.Assert.assertTrue;
2732

2833
@RunWith(RunnerWithHttpServer.class)
@@ -102,6 +107,49 @@ public static void removeClassTempWriters() {
102107
return HttpKeyValueAccess.class;
103108
}
104109

110+
@Test
111+
public void testReaderCreation() {
112+
113+
// non-existent location should fail
114+
final String location = tempN5Location();
115+
assertThrows("Non-existent location throws error", N5Exception.N5IOException.class,
116+
() -> {
117+
try (N5Reader test = createN5Reader(location)) {
118+
test.list("/");
119+
}
120+
});
121+
122+
try (N5Writer writer = createTempN5Writer(location)) {
123+
try (N5Reader n5r = createN5Reader(location)) {
124+
assertNotNull(n5r);
125+
}
126+
127+
// existing directory without attributes is okay;
128+
// Remove and create to remove attributes store
129+
writer.removeAttribute("/", "/");
130+
try (N5Reader na = createN5Reader(location)) {
131+
assertNotNull(na);
132+
}
133+
134+
// existing location with attributes, but no version
135+
writer.removeAttribute("/", "/");
136+
writer.setAttribute("/", "mystring", "ms");
137+
try (N5Reader wa = createN5Reader(location)) {
138+
assertNotNull(wa);
139+
}
140+
141+
// existing directory with incompatible version should fail
142+
writer.removeAttribute("/", "/");
143+
final String invalidVersion = new N5Reader.Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString();
144+
writer.setAttribute("/", N5Reader.VERSION_KEY, invalidVersion);
145+
assertThrows("Incompatible version throws error", N5Exception.class, () -> {
146+
try (final N5Reader ignored = createN5Reader(location)) {
147+
/*Only try with resource to ensure `close()` is called.*/
148+
}
149+
});
150+
}
151+
}
152+
105153
@Ignore("N5Writer not supported for HTTP")
106154
@Override
107155
public void testVersion() throws NumberFormatException {
@@ -137,4 +185,6 @@ public void testVersion() throws NumberFormatException {
137185
@Override public void testWriterSeparation() {
138186

139187
}
188+
189+
140190
}

‎src/test/java/org/janelia/saalfeldlab/n5/universe/storage/zarr/ZarrGoogleCloudFactoryTest.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
package org.janelia.saalfeldlab.n5.universe.storage.zarr;
22

33
import com.google.cloud.storage.Storage;
4+
import org.janelia.saalfeldlab.n5.DataType;
5+
import org.janelia.saalfeldlab.n5.DatasetAttributes;
6+
import org.janelia.saalfeldlab.n5.N5Exception;
7+
import org.janelia.saalfeldlab.n5.N5Reader;
48
import org.janelia.saalfeldlab.n5.N5Writer;
9+
import org.janelia.saalfeldlab.n5.RawCompression;
510
import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess;
611
import org.janelia.saalfeldlab.n5.googlecloud.N5GoogleCloudStorageTests;
712
import org.janelia.saalfeldlab.n5.googlecloud.backend.BackendGoogleCloudStorageFactory;
@@ -10,10 +15,14 @@
1015
import org.junit.AfterClass;
1116
import org.junit.BeforeClass;
1217

18+
import java.io.IOException;
1319
import java.net.URI;
1420
import java.net.URISyntaxException;
1521

1622
import static org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.tempContainerPath;
23+
import static org.junit.Assert.assertArrayEquals;
24+
import static org.junit.Assert.assertEquals;
25+
import static org.junit.Assert.assertThrows;
1726
import static org.junit.Assert.assertTrue;
1827

1928
public abstract class ZarrGoogleCloudFactoryTest extends ZarrStorageTests.ZarrFactoryTest {
@@ -62,6 +71,41 @@ public ZarrGoogleCloudMockTest() {
6271

6372
ZarrGoogleCloudFactoryTest.storage = MockGoogleCloudStorageFactory.getOrCreateStorage();
6473
}
74+
75+
@Override public void testPathsWithIllegalUriCharacters() {
76+
77+
// NOTE: This is currently identical to the AbstractN5Test version, except we remove "%" from the array of illegalChars to test.
78+
// Specifically we ONLY remove it in this Mock version of the test. The actual backend has no problem, and the test passes
79+
// just fine, but the mock version seems to have some incorrect handling of % in keys. Likely because % is a valid UTF-8 character
80+
// but requires special encoding for URL over HTTP, and it seems the mock channels don't handle that case correctly
81+
try (N5Writer writer = createTempN5Writer()) {
82+
try (N5Reader reader = createN5Reader(writer.getURI().toString())) {
83+
84+
final String[] illegalChars = {" ", "#"};
85+
for (final String illegalChar : illegalChars) {
86+
final String groupWithIllegalChar = "test" + illegalChar + "group";
87+
assertThrows("list over group should throw prior to create", N5Exception.N5IOException.class, () -> writer.list(groupWithIllegalChar));
88+
writer.createGroup(groupWithIllegalChar);
89+
assertTrue("Newly created group should exist", writer.exists(groupWithIllegalChar));
90+
assertArrayEquals("list over empty group should be empty list", new String[0], writer.list(groupWithIllegalChar));
91+
writer.setAttribute(groupWithIllegalChar, "/a/b/key1", "value1");
92+
final String attrFromWriter = writer.getAttribute(groupWithIllegalChar, "/a/b/key1", String.class);
93+
final String attrFromReader = reader.getAttribute(groupWithIllegalChar, "/a/b/key1", String.class);
94+
assertEquals("value1", attrFromWriter);
95+
assertEquals("value1", attrFromReader);
96+
97+
98+
final String datasetWithIllegalChar = "test" + illegalChar + "dataset";
99+
final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64, new RawCompression());
100+
writer.createDataset(datasetWithIllegalChar, datasetAttributes);
101+
final DatasetAttributes datasetFromWriter = writer.getDatasetAttributes(datasetWithIllegalChar);
102+
final DatasetAttributes datasetFromReader = reader.getDatasetAttributes(datasetWithIllegalChar);
103+
assertDatasetAttributesEquals(datasetAttributes, datasetFromWriter);
104+
assertDatasetAttributesEquals(datasetAttributes, datasetFromReader);
105+
}
106+
}
107+
}
108+
}
65109
}
66110

67111
public static class ZarrGoogleCloudBackendTest extends ZarrGoogleCloudFactoryTest {

‎src/test/java/org/janelia/saalfeldlab/n5/universe/storage/zarr/ZarrHttpFactoryTest.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader;
55
import org.janelia.saalfeldlab.n5.GsonKeyValueN5Writer;
66
import org.janelia.saalfeldlab.n5.HttpKeyValueAccess;
7+
import org.janelia.saalfeldlab.n5.N5Exception;
78
import org.janelia.saalfeldlab.n5.N5KeyValueWriter;
89
import org.janelia.saalfeldlab.n5.N5Reader;
910
import org.janelia.saalfeldlab.n5.N5Writer;
@@ -14,6 +15,7 @@
1415
import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter;
1516
import org.junit.After;
1617
import org.junit.AfterClass;
18+
import org.junit.Assert;
1719
import org.junit.Ignore;
1820
import org.junit.Test;
1921
import org.junit.runner.RunWith;
@@ -27,6 +29,8 @@
2729
import java.util.ArrayList;
2830

2931
import static org.junit.Assert.assertEquals;
32+
import static org.junit.Assert.assertNotNull;
33+
import static org.junit.Assert.assertThrows;
3034
import static org.junit.Assert.assertTrue;
3135

3236
@RunWith(RunnerWithHttpServer.class)
@@ -142,6 +146,36 @@ public void testCreateNestedDataset() throws IOException {
142146
// TODO test that parents of nested dataset are groups
143147
}
144148

149+
150+
151+
@Test
152+
public void testReaderCreation() {
153+
154+
// non-existent location should fail
155+
final String location = tempN5Location();
156+
assertThrows("Non-existent location throws error", N5Exception.N5IOException.class,
157+
() -> {
158+
try (N5Reader test = createN5Reader(location)) {
159+
test.list("/");
160+
}
161+
});
162+
163+
try (N5Writer writer = createTempN5Writer(location)) {
164+
165+
assertNotNull(createN5Reader(location));
166+
167+
// existing directory without attributes is okay;
168+
// Remove and create to remove attributes store
169+
writer.removeAttribute("/", "/");
170+
assertNotNull(createN5Reader(location));
171+
172+
// existing location with attributes, but no version
173+
writer.removeAttribute("/", "/");
174+
writer.setAttribute("/", "mystring", "ms");
175+
assertNotNull(createN5Reader(location));
176+
}
177+
}
178+
145179
@Ignore("N5Writer not supported for HTTP")
146180
@Override
147181
public void testVersion() throws NumberFormatException {

0 commit comments

Comments
 (0)