Skip to content

Commit 668bba3

Browse files
cmhulbertcmhulbert
authored
Merge pull request #54 from saalfeldlab/ome-zarr-big-changes
OME-Zarr metadata parsing fixes and refactor
2 parents 5d3f25c + cc5ab4e commit 668bba3

45 files changed

Lines changed: 865 additions & 1441 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/main/java/org/janelia/saalfeldlab/n5/universe/N5DatasetDiscoverer.java‎

Lines changed: 69 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.text.Collator;
3030
import java.util.ArrayList;
3131
import java.util.Arrays;
32+
import java.util.Collections;
3233
import java.util.Comparator;
3334
import java.util.Iterator;
3435
import java.util.List;
@@ -51,8 +52,7 @@
5152
import org.janelia.saalfeldlab.n5.universe.metadata.N5SingleScaleMetadataParser;
5253
import org.janelia.saalfeldlab.n5.universe.metadata.N5ViewerMultiscaleMetadataParser;
5354
import org.janelia.saalfeldlab.n5.universe.metadata.canonical.CanonicalMetadataParser;
54-
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadataParser;
55-
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v05.OmeNgffV05MetadataParser;
55+
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.OmeNgffMetadataParser;
5656
import org.slf4j.Logger;
5757
import org.slf4j.LoggerFactory;
5858

@@ -88,21 +88,9 @@ public class N5DatasetDiscoverer {
8888
new N5GenericSingleScaleMetadataParser(),
8989
};
9090

91-
public static final N5MetadataParser<?>[] DEFAULT_GROUP_PARSERS = new N5MetadataParser<?>[] {
92-
new OmeNgffV05MetadataParser(),
93-
new OmeNgffMetadataParser(),
94-
new N5CosemMultiScaleMetadata.CosemMultiScaleParser(),
95-
new N5ViewerMultiscaleMetadataParser(),
96-
new CanonicalMetadataParser(),
97-
};
98-
99-
public static final N5MetadataParser<?>[] DEFAULT_SHALLOW_GROUP_PARSERS = new N5MetadataParser<?>[] {
100-
new OmeNgffV05MetadataParser(true),
101-
new OmeNgffMetadataParser(true)
102-
};
103-
10491
private final List<N5MetadataParser<?>> metadataParsers;
10592
private final List<N5MetadataParser<?>> groupParsers;
93+
private final List<N5MetadataParser<?>> shallowGroupParsers;
10694

10795
private final Comparator<? super String> comparator;
10896

@@ -116,22 +104,12 @@ public class N5DatasetDiscoverer {
116104

117105
private N5Reader n5;
118106

119-
/**
120-
* Creates an N5 discoverer with alphanumeric sorting order of
121-
* groups/datasets (such as, s9 goes before s10).
122-
*
123-
* @param executor
124-
* the executor
125-
* @param metadataParsers
126-
* metadata parsers
127-
* @param groupParsers
128-
* group parsers
129-
*/
130-
public N5DatasetDiscoverer(final ExecutorService executor, final List<N5MetadataParser<?>> metadataParsers,
131-
final List<N5MetadataParser<?>> groupParsers) {
107+
108+
public N5DatasetDiscoverer(final N5Reader n5, final ExecutorService executor) {
132109

133-
this(executor, Optional.of(new AlphanumericComparator(Collator.getInstance())), null, metadataParsers,
134-
groupParsers);
110+
this(n5, executor, Optional.of(new AlphanumericComparator(Collator.getInstance())), null,
111+
Arrays.asList(DEFAULT_PARSERS),
112+
defaultGroupParsers(n5));
135113
}
136114

137115
public N5DatasetDiscoverer(final N5Reader n5, final ExecutorService executor,
@@ -141,21 +119,6 @@ public N5DatasetDiscoverer(final N5Reader n5, final ExecutorService executor,
141119
groupParsers);
142120
}
143121

144-
/**
145-
* Creates an N5 discoverer.
146-
*
147-
* @param metadataParsers
148-
* metadata parsers
149-
* @param groupParsers
150-
* group parsers
151-
*/
152-
public N5DatasetDiscoverer(final List<N5MetadataParser<?>> metadataParsers,
153-
final List<N5MetadataParser<?>> groupParsers) {
154-
155-
this(Executors.newSingleThreadExecutor(), Optional.of(new AlphanumericComparator(Collator.getInstance())), null,
156-
metadataParsers, groupParsers);
157-
}
158-
159122
/**
160123
* Creates an N5 discoverer.
161124
*
@@ -166,46 +129,37 @@ public N5DatasetDiscoverer(final List<N5MetadataParser<?>> metadataParsers,
166129
* @param groupParsers
167130
* group parsers
168131
*/
169-
public N5DatasetDiscoverer(final N5Reader n5, final List<N5MetadataParser<?>> metadataParsers,
132+
public N5DatasetDiscoverer(final N5Reader n5,
133+
final List<N5MetadataParser<?>> metadataParsers,
170134
final List<N5MetadataParser<?>> groupParsers) {
171135

172136
this(n5, Executors.newSingleThreadExecutor(), Optional.of(new AlphanumericComparator(Collator.getInstance())),
173137
null, metadataParsers, groupParsers);
174138
}
175139

176-
public N5DatasetDiscoverer(final ExecutorService executor, final Predicate<N5TreeNode> filter,
177-
final List<N5MetadataParser<?>> metadataParsers, final List<N5MetadataParser<?>> groupParsers) {
178-
179-
this(executor, Optional.of(new AlphanumericComparator(Collator.getInstance())), filter, metadataParsers,
180-
groupParsers);
181-
}
182-
183140
public N5DatasetDiscoverer(final N5Reader n5, final ExecutorService executor, final Predicate<N5TreeNode> filter,
184141
final List<N5MetadataParser<?>> metadataParsers, final List<N5MetadataParser<?>> groupParsers) {
185142

186143
this(n5, executor, Optional.of(new AlphanumericComparator(Collator.getInstance())), filter, metadataParsers,
187144
groupParsers);
188145
}
189146

190-
public N5DatasetDiscoverer(final ExecutorService executor, final Optional<Comparator<? super String>> comparator,
191-
final List<N5MetadataParser<?>> metadataParsers, final List<N5MetadataParser<?>> groupParsers) {
192-
193-
this(executor, comparator, null, metadataParsers, groupParsers);
194-
}
195-
196147
public N5DatasetDiscoverer(final N5Reader n5, final ExecutorService executor,
197148
final Optional<Comparator<? super String>> comparator, final List<N5MetadataParser<?>> metadataParsers,
198149
final List<N5MetadataParser<?>> groupParsers) {
199150

200151
this(n5, executor, comparator, null, metadataParsers, groupParsers);
201152
}
202153

154+
203155
/**
204156
* Creates an N5 discoverer.
205157
* <p>
206158
* If the optional parameter {@code comparator} is specified, the groups and
207159
* datasets will be listed in the order determined by this comparator.
208160
*
161+
* @param n5
162+
* the n5 reader
209163
* @param executor
210164
* the executor
211165
* @param comparator
@@ -217,17 +171,15 @@ public N5DatasetDiscoverer(final N5Reader n5, final ExecutorService executor,
217171
* @param groupParsers
218172
* group parsers
219173
*/
220-
public N5DatasetDiscoverer(final ExecutorService executor, final Optional<Comparator<? super String>> comparator,
221-
final Predicate<N5TreeNode> filter, final List<N5MetadataParser<?>> metadataParsers,
222-
final List<N5MetadataParser<?>> groupParsers) {
174+
public N5DatasetDiscoverer(final N5Reader n5, final ExecutorService executor,
175+
final Optional<Comparator<? super String>> comparator, final Predicate<N5TreeNode> filter,
176+
final List<N5MetadataParser<?>> metadataParsers, final List<N5MetadataParser<?>> groupParsers) {
223177

224-
this.executor = executor;
225-
this.comparator = comparator.orElseGet(null);
226-
this.filter = filter;
227-
this.metadataParsers = metadataParsers;
228-
this.groupParsers = groupParsers;
178+
this(n5, executor, comparator, filter,
179+
metadataParsers, groupParsers,
180+
defaultShallowGroupParsers(n5));
229181
}
230-
182+
231183
/**
232184
* Creates an N5 discoverer.
233185
* <p>
@@ -246,17 +198,21 @@ public N5DatasetDiscoverer(final ExecutorService executor, final Optional<Compar
246198
* metadata parsers
247199
* @param groupParsers
248200
* group parsers
201+
* @param shallowGroupParsers
202+
* shallow group parsers
249203
*/
250204
public N5DatasetDiscoverer(final N5Reader n5, final ExecutorService executor,
251205
final Optional<Comparator<? super String>> comparator, final Predicate<N5TreeNode> filter,
252-
final List<N5MetadataParser<?>> metadataParsers, final List<N5MetadataParser<?>> groupParsers) {
206+
final List<N5MetadataParser<?>> metadataParsers, final List<N5MetadataParser<?>> groupParsers,
207+
final List<N5MetadataParser<?>> shallowGroupParsers ) {
253208

254209
this.n5 = n5;
255210
this.executor = executor;
256211
this.comparator = comparator.orElseGet(null);
257212
this.filter = filter;
258213
this.metadataParsers = metadataParsers;
259214
this.groupParsers = groupParsers;
215+
this.shallowGroupParsers = shallowGroupParsers;
260216
}
261217

262218
public static void parseMetadata(final N5Reader n5, final N5TreeNode node,
@@ -312,11 +268,13 @@ public static void parseMetadata(final N5Reader n5, final N5TreeNode node,
312268
/**
313269
* Parses metadata for a node using the given parsers, stopping after the first
314270
* success.
271+
* <p>
272+
* The group parsers given are assumed to be shallow.
315273
*
316274
* @param n5 the N5Reader
317275
* @param node the tree node
318276
* @param metadataParsers list of metadata parsers
319-
* @param groupParsers list of group parsers
277+
* @param groupParsers list of shallow group parsers
320278
*/
321279
public static void parseMetadataShallow(final N5Reader n5, final N5TreeNode node,
322280
final List<N5MetadataParser<?>> metadataParsers, final List<N5MetadataParser<?>> groupParsers) {
@@ -335,11 +293,13 @@ public static void parseMetadataShallow(final N5Reader n5, final N5TreeNode node
335293
}
336294

337295
// this may be a group (e.g. multiscale pyramid) try to parse groups
338-
for (final N5MetadataParser<?> gp : DEFAULT_SHALLOW_GROUP_PARSERS) {
339-
final Optional<? extends N5Metadata> groupMeta = gp.apply(n5, node);
340-
groupMeta.ifPresent(node::setMetadata);
341-
if (groupMeta.isPresent())
342-
break;
296+
for (final N5MetadataParser<?> gp : groupParsers) {
297+
try {
298+
final Optional<? extends N5Metadata> groupMeta = gp.apply(n5, node);
299+
groupMeta.ifPresent(node::setMetadata);
300+
if (groupMeta.isPresent())
301+
break;
302+
} catch (Throwable ignore) {}
343303
}
344304
}
345305

@@ -426,7 +386,7 @@ public N5TreeNode discoverShallow(final N5TreeNode base) {
426386

427387
public N5TreeNode discoverShallow(final N5TreeNode base, final Consumer<N5TreeNode> callback) {
428388

429-
parseMetadataShallow(n5, base, metadataParsers, groupParsers);
389+
parseMetadataShallow(n5, base, metadataParsers, shallowGroupParsers);
430390
return base;
431391
}
432392

@@ -690,12 +650,12 @@ public static N5TreeNode discover(final N5Reader n5, final List<N5MetadataParser
690650

691651
public static N5TreeNode discover(final N5Reader n5, final String basePath) {
692652

693-
return discover(n5, basePath, Arrays.asList(DEFAULT_PARSERS), Arrays.asList(DEFAULT_GROUP_PARSERS));
653+
return discover(n5, basePath, Arrays.asList(DEFAULT_PARSERS), defaultGroupParsers(n5));
694654
}
695655

696656
public static N5TreeNode discover(final N5Reader n5) {
697657

698-
return discover(n5, Arrays.asList(DEFAULT_PARSERS), Arrays.asList(DEFAULT_GROUP_PARSERS));
658+
return discover(n5, Arrays.asList(DEFAULT_PARSERS), defaultGroupParsers(n5));
699659
}
700660

701661
/**
@@ -709,7 +669,7 @@ public static N5TreeNode discoverShallow(final N5Reader n5, final String dataset
709669

710670
final N5TreeNode node = new N5TreeNode(dataset);
711671
parseMetadataShallow(n5, node, Arrays.asList(DEFAULT_PARSERS),
712-
Arrays.asList(DEFAULT_SHALLOW_GROUP_PARSERS));
672+
defaultShallowGroupParsers(n5));
713673

714674
return node;
715675
}
@@ -724,6 +684,36 @@ public static N5TreeNode discoverShallow(final N5Reader n5) {
724684

725685
return discoverShallow(n5, "/");
726686
}
687+
688+
/**
689+
* Parser configuration depends on the n5 reader. OME-Zarr parsers reverse
690+
* axis parmeters for zarr, but not for n5.
691+
*
692+
* @param n5
693+
* the n5 reader
694+
* @return the group parsers
695+
*/
696+
public static List<N5MetadataParser<?>> defaultGroupParsers(N5Reader n5) {
697+
698+
final ArrayList<N5MetadataParser<?>> out = new ArrayList<>();
699+
out.add(new OmeNgffMetadataParser(n5));
700+
out.add(new N5CosemMultiScaleMetadata.CosemMultiScaleParser());
701+
out.add(new N5ViewerMultiscaleMetadataParser());
702+
out.add(new CanonicalMetadataParser());
703+
return out;
704+
}
705+
706+
/**
707+
* Parser configuration depends on the n5 reader.
708+
* OME-Zarr parsers reverse axis parmeters for zarr, but not for n5.
709+
*
710+
* @param n5
711+
* the n5 reader
712+
* @return the shallow group parsers
713+
*/
714+
public static List<N5MetadataParser<?>> defaultShallowGroupParsers(N5Reader n5) {
715+
return Collections.singletonList(new OmeNgffMetadataParser(n5));
716+
}
727717

728718
}
729719

‎src/main/java/org/janelia/saalfeldlab/n5/universe/N5MetadataUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
import org.janelia.saalfeldlab.n5.universe.metadata.N5SingleScaleMetadataParser;
1616
import org.janelia.saalfeldlab.n5.universe.metadata.N5ViewerMultiscaleMetadataParser;
1717
import org.janelia.saalfeldlab.n5.universe.metadata.canonical.CanonicalMetadataParser;
18-
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadataParser;
18+
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.OmeNgffMetadataParser;
1919

2020
public class N5MetadataUtils {
2121

‎src/main/java/org/janelia/saalfeldlab/n5/universe/metadata/MetadataUtils.java‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,12 @@
1212
import org.janelia.saalfeldlab.n5.universe.metadata.N5CosemMetadata.CosemTransform;
1313
import org.janelia.saalfeldlab.n5.universe.metadata.axes.Axis;
1414
import org.janelia.saalfeldlab.n5.universe.metadata.axes.AxisUtils;
15-
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.NgffSingleScaleAxesMetadata;
16-
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.CoordinateTransformation;
17-
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.ScaleCoordinateTransformation;
18-
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.coordinateTransformations.TranslationCoordinateTransformation;
15+
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.NgffSingleScaleAxesMetadata;
16+
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.coordinateTransformations.CoordinateTransformation;
17+
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.coordinateTransformations.ScaleCoordinateTransformation;
18+
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.coordinateTransformations.TranslationCoordinateTransformation;
1919

20+
import com.google.gson.JsonArray;
2021
import com.google.gson.JsonElement;
2122
import com.google.gson.JsonNull;
2223

@@ -535,4 +536,12 @@ public static < T > T[] reversedCopy( T[] array )
535536
return result;
536537
}
537538

539+
public static JsonArray reversedCopy( JsonArray array )
540+
{
541+
final JsonArray result = new JsonArray(array.size());
542+
for (int i = array.size() - 1; i >= 0; i--)
543+
result.add(array.get(i));
544+
return result;
545+
}
546+
538547
}

0 commit comments

Comments
 (0)