Skip to content

Commit 2940dfb

Browse files
Merge pull request #84 from BioImageTools/add-s3-support
Add support for opening OME-Zarr datasets in public S3 stores
2 parents aa371be + 9bff2cd commit 2940dfb

15 files changed

Lines changed: 302 additions & 136 deletions

File tree

README.md

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ If the dropped / pasted target is not recognized as a **OME-Zarr v0.3 - v0.5** r
1212

1313
# Features
1414

15-
## Drag & Drop of local OME-Zarr folders and Copy & Paste of OME-Zarr URIs (local folder, http, https)
15+
## Drag & Drop of local OME-Zarr folders and Copy & Paste of OME-Zarr URIs (local folder, http, https, s3)
1616

1717
There are several options for what Fiji can do after drag & drop / copy & paste:
1818

@@ -37,7 +37,12 @@ and easily handles even the huge ones.
3737

3838
### Copy & Paste:
3939

40-
* Supports local paths and http(s) URLs (s3 is planned)
40+
* Supports local paths, http(s) URLs, and `s3://` URIs
41+
* Public (anonymous) S3 buckets work out of the box, e.g.
42+
`s3://janelia-cosem-datasets/jrc_mus-choroid-plexus-3/jrc_mus-choroid-plexus-3.zarr/recon-1/em/fibsem-uint8`.
43+
* Private buckets use your ambient AWS credentials (environment variables, `~/.aws/credentials`,
44+
instance profile, etc.); if those are absent, access falls back to anonymous.
45+
* The AWS region defaults to `us-east-1`.
4146
* Three entry points:
4247
* Paste with `CTRL` / `CMD` / `SHIFT` + `V` (requires FIJI latest)
4348
* Paste via menu: Plugins -> OME-Zarr -> Paste OME-Zarr URI

ome-zarr-fiji-ui/src/main/java/ome/zarr/fijiui/open/PasteToOpenAction.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,11 @@ public static boolean pasteFromClipboard( final Context context, final Consumer<
6666
final URI uri = ClipboardUtils.readClipboardAsUri( errorHandler );
6767
if ( uri == null )
6868
return false;
69-
if ( !ZarrUtils.isZarr( uri ) )
69+
// For s3:// URIs the probe would require its own short-lived S3Client purely
70+
// for detection.
71+
// It is skipped because the actual open method creates the client it needs anyway
72+
// and reports and error if the location turns out not to be OME-Zarr.
73+
if ( !"s3".equalsIgnoreCase( uri.getScheme() ) && !ZarrUtils.isZarr( uri ) )
7074
{
7175
if ( errorHandler != null )
7276
errorHandler.accept( "The pasted location does not appear to be an OME-Zarr dataset:\n" + uri + "." );

ome-zarr-fiji-ui/src/main/java/ome/zarr/fijiui/open/ZarrOpenActions.java

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,12 @@
4141
import java.nio.file.Paths;
4242
import java.util.Collections;
4343
import java.util.function.Consumer;
44-
import java.util.function.Function;
4544

46-
import net.imglib2.img.Img;
4745

4846
import ij.IJ;
4947
import ome.zarr.fijiui.open.options.ZarrOpenBehavior;
5048
import ome.zarr.fijiui.open.options.ZarrOpeningSettings;
5149
import ome.zarr.fijiui.open.options.ZarrReaderBackend;
52-
import ome.zarr.fiji.PyramidalDataset;
5350
import ome.zarr.fiji.open.ZarrOpener;
5451
import ome.zarr.fijiui.dialog.DnDActionChooser;
5552
import ome.zarr.fijiui.util.ScriptUtils;
@@ -238,11 +235,6 @@ public Object openBDVWithImage()
238235
return opener.openBDVWithImage();
239236
}
240237

241-
Object openImage( final Function< PyramidalDataset, Object > multiScaleImageOpener,
242-
final Function< Img< ? >, Object > singleScaleImageOpener )
243-
{
244-
return opener.openImage( multiScaleImageOpener, singleScaleImageOpener );
245-
}
246238

247239
/**
248240
* Opens the Fiji script editor pre-filled with a scriptlet that opens the

ome-zarr-fiji-ui/src/main/java/ome/zarr/fijiui/util/ClipboardUtils.java

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,13 @@
66
* %%
77
* Redistribution and use in source and binary forms, with or without
88
* modification, are permitted provided that the following conditions are met:
9-
*
9+
*
1010
* 1. Redistributions of source code must retain the above copyright notice,
1111
* this list of conditions and the following disclaimer.
1212
* 2. Redistributions in binary form must reproduce the above copyright notice,
1313
* this list of conditions and the following disclaimer in the documentation
1414
* and/or other materials provided with the distribution.
15-
*
15+
*
1616
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
1717
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1818
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -36,6 +36,7 @@
3636
import java.io.IOException;
3737
import java.lang.invoke.MethodHandles;
3838
import java.net.URI;
39+
import java.net.URISyntaxException;
3940
import java.nio.file.InvalidPathException;
4041
import java.nio.file.Paths;
4142
import java.util.function.Consumer;
@@ -81,10 +82,14 @@ public static String readClipboard()
8182

8283
/**
8384
* Resolve text read from clipboard to a URI suitable for opening an OME-Zarr dataset.
84-
* Handles three input forms:
85+
* Handles four input forms:
8586
* <ul>
8687
* <li>{@code http://} or {@code https://} URLs &ndash; used as-is</li>
87-
* <li>{@code file:} URIs &ndash; used as-is</li>
88+
* <li>{@code s3://} URIs &ndash; used as-is</li>
89+
* <li>{@code file:} URIs &ndash; used as-is; both {@code file:/path/to/data.zarr} and
90+
* {@code file:///path/to/data.zarr} are usable. A URI naming a host
91+
* ({@code file://some-server/path/to/data.zarr}) is rejected by the backends,
92+
* because {@link Paths#get(URI)} accepts no authority component.</li>
8893
* <li>plain filesystem paths &ndash; converted with
8994
* {@link Paths#get(String, String...)}{@code .toUri()}</li>
9095
* </ul>
@@ -99,10 +104,14 @@ public static URI readClipboardAsUri( final Consumer< String > errorHandler )
99104

100105
/**
101106
* Converts a string to a {@link URI} suitable for opening an OME-Zarr dataset.
102-
* Handles three input forms:
107+
* Handles four input forms:
103108
* <ul>
104109
* <li>{@code http://} or {@code https://} URLs &ndash; used as-is</li>
105-
* <li>{@code file:} URIs &ndash; used as-is</li>
110+
* <li>{@code s3://} URIs &ndash; used as-is</li>
111+
* <li>{@code file:} URIs &ndash; used as-is; both {@code file:/path/to/data.zarr} and
112+
* {@code file:///path/to/data.zarr} are usable. A URI naming a host
113+
* ({@code file://some-server/path/to/data.zarr}) is rejected by the backends,
114+
* because {@link Paths#get(URI)} accepts no authority component.</li>
106115
* <li>plain filesystem paths &ndash; converted with
107116
* {@link Paths#get(String, String...)}{@code .toUri()}</li>
108117
* </ul>
@@ -128,7 +137,7 @@ public static URI stringToUri( final String possibleUri, final Consumer< String
128137
{
129138
parsed = new URI( text );
130139
}
131-
catch ( Exception e )
140+
catch ( URISyntaxException e )
132141
{
133142
logger.debug( "Text is not valid URI syntax, will try as a local path: {}", e.getMessage() );
134143
}
@@ -137,12 +146,19 @@ public static URI stringToUri( final String possibleUri, final Consumer< String
137146
if ( parsed != null )
138147
{
139148
final String scheme = parsed.getScheme();
140-
if ( "http".equalsIgnoreCase( scheme ) || "https".equalsIgnoreCase( scheme ) || "file".equalsIgnoreCase( scheme ) )
149+
if ( "file".equalsIgnoreCase( scheme ) && parsed.getAuthority() != null )
150+
{
151+
errorHandler.accept( "A 'file:' URL cannot name a host:\n" + text + "\n\n"
152+
+ "Use file:///path/to/data.zarr (three slashes) for a local path." );
153+
return null;
154+
}
155+
if ( "http".equalsIgnoreCase( scheme ) || "https".equalsIgnoreCase( scheme )
156+
|| "file".equalsIgnoreCase( scheme ) || "s3".equalsIgnoreCase( scheme ) )
141157
return parsed;
142158
if ( scheme != null )
143159
{
144160
errorHandler.accept( "Unsupported URL scheme '" + scheme + "':\n" + text + "\n\n"
145-
+ "Supported schemes are http, https, and file." );
161+
+ "Supported schemes are http, https, file, and s3." );
146162
return null;
147163
}
148164
}

ome-zarr-fiji-ui/src/test/java/ome/zarr/fijiui/open/ZarrOpenActionsTest.java

Lines changed: 82 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434
import static org.junit.jupiter.api.Assertions.assertFalse;
3535
import static org.junit.jupiter.api.Assertions.assertNotNull;
3636
import static org.junit.jupiter.api.Assertions.assertNotSame;
37-
import static org.junit.jupiter.api.Assertions.assertNull;
3837
import static org.junit.jupiter.api.Assertions.assertSame;
3938
import static org.junit.jupiter.api.Assertions.assertTrue;
4039
import static org.mockito.Mockito.any;
@@ -77,10 +76,8 @@
7776
import java.util.Arrays;
7877
import java.util.List;
7978
import java.util.concurrent.atomic.AtomicBoolean;
80-
import java.util.concurrent.atomic.AtomicInteger;
8179
import java.util.concurrent.atomic.AtomicReference;
8280
import java.util.function.Consumer;
83-
import java.util.function.Function;
8481
import java.util.stream.Stream;
8582

8683
import bdv.tools.brightness.ConverterSetup;
@@ -97,7 +94,9 @@
9794
import ij.ImagePlus;
9895
import ome.zarr.fijiui.settings.UserScriptSettings;
9996
import ome.zarr.fiji.Pyramidal;
97+
import ome.zarr.imglib2.PyramidBackend;
10098
import ome.zarr.imglib2.PyramidContents;
99+
import ome.zarr.fiji.open.ZarrOpener;
101100
import ome.zarr.fiji.PyramidalBdv;
102101
import ome.zarr.fiji.PyramidalDataset;
103102
import ome.zarr.fijiui.open.options.ZarrOpeningSettings;
@@ -110,11 +109,28 @@
110109

111110
class ZarrOpenActionsTest
112111
{
112+
113113
static Stream< ZarrReaderBackend > readerBackends()
114114
{
115115
return Stream.of( ZarrReaderBackend.N5, ZarrReaderBackend.ZARR_JAVA );
116116
}
117117

118+
/**
119+
* Loads the dataset headlessly through {@link ZarrOpener#getContents()} with
120+
* the given backend, without instantiating any UI. Returns the loaded
121+
* {@link PyramidContents}; throws the relevant domain exception (e.g.
122+
* {@link ome.zarr.imglib2.exceptions.NotAMultiscaleImageException} or
123+
* {@link ome.zarr.imglib2.exceptions.MultiImageDatasetException}).
124+
* Lets tests assert that a dataset opens as a multiscale image without showing a window.
125+
*/
126+
private static PyramidContents< ? > loadMultiscaleHeadless( final URI uri, final Context context,
127+
final ZarrReaderBackend backend )
128+
{
129+
final PyramidBackend pyramidBackend = backend.createBackend();
130+
final ZarrOpener opener = new ZarrOpener( uri, context, pyramidBackend, null, error -> {} );
131+
return opener.getContents();
132+
}
133+
118134
static Stream< String > omeZarrExamples()
119135
{
120136
return Stream.of(
@@ -287,18 +303,13 @@ void testOpenValidMultiScaleImagePath( String resource ) throws URISyntaxExcepti
287303
Path path = ZarrTestUtils.resourcePath( resource );
288304
try (Context context = new Context())
289305
{
290-
ZarrOpenActions actions = new ZarrOpenActions( path.toUri(), context );
291-
AtomicInteger multiScaleCounter = new AtomicInteger( 0 );
292-
AtomicInteger singleScaleCounter = new AtomicInteger( 0 );
293-
Function< PyramidalDataset, Object > multiScaleOpeningCounter = dataset -> multiScaleCounter.incrementAndGet();
294-
Function< Img< ? >, Object > singleScaleOpeningCounter = img -> singleScaleCounter.incrementAndGet();
295-
actions.openImage( multiScaleOpeningCounter, singleScaleOpeningCounter );
296-
assertEquals( 1, multiScaleCounter.get() );
297-
assertEquals( 0, singleScaleCounter.get() );
306+
assertNotNull( loadMultiscaleHeadless( path.toUri(), context, ZarrOpeningSettings.DEFAULT_READER_BACKEND ),
307+
"Expected " + resource + " to open as a multiscale image" );
298308
}
299309
}
300310

301311
@Test
312+
@SuppressWarnings( "java:S1612" )
302313
void testOpenValidSingleScaleImagePath() throws URISyntaxException
303314
{
304315
String[] validPaths = {
@@ -310,19 +321,19 @@ void testOpenValidSingleScaleImagePath() throws URISyntaxException
310321
for ( String invalidPath : validPaths )
311322
{
312323
Path path = ZarrTestUtils.resourcePath( invalidPath );
313-
ZarrOpenActions actions = new ZarrOpenActions( path.toUri(), context, null, System.out::println );
314-
AtomicInteger multiScaleCounter = new AtomicInteger( 0 );
315-
AtomicInteger singleScaleCounter = new AtomicInteger( 0 );
316-
Function< PyramidalDataset, Object > multiScaleOpeningCounter = dataset -> multiScaleCounter.incrementAndGet();
317-
Function< Img< ? >, Object > singleScaleOpeningCounter = img -> singleScaleCounter.incrementAndGet();
318-
actions.openImage( multiScaleOpeningCounter, singleScaleOpeningCounter );
319-
assertEquals( 0, multiScaleCounter.get() );
320-
assertEquals( 0, singleScaleCounter.get() ); // currently not supported
324+
AtomicReference< String > capturedError = new AtomicReference<>();
325+
ZarrOpenActions actions = new ZarrOpenActions( path.toUri(), context, null, capturedError::set );
326+
assertDoesNotThrow( () -> {
327+
actions.openIJWithImage();
328+
} );
329+
assertNotNull( capturedError.get(),
330+
"Single-scale path " + invalidPath + " should be reported as not (yet) supported" );
321331
}
322332
}
323333
}
324334

325335
@Test
336+
@SuppressWarnings( "java:S1612" )
326337
void testOpenInvalidImagePaths() throws URISyntaxException
327338
{
328339
String[] invalidPaths = {
@@ -335,15 +346,16 @@ void testOpenInvalidImagePaths() throws URISyntaxException
335346
{
336347
Path path = ZarrTestUtils.resourcePath( invalidPath );
337348
ZarrOpenActions actions = new ZarrOpenActions( path.toUri(), context, null, System.out::println );
338-
Function< PyramidalDataset, Object > multiScaleNoOp = pyramidalDataset -> null;
339-
Function< Img< ? >, Object > singleScaleNoOp = img -> null;
340-
assertDoesNotThrow( () -> actions.openImage( multiScaleNoOp, singleScaleNoOp ) );
349+
assertDoesNotThrow( () -> {
350+
actions.openIJWithImage();
351+
} );
341352
}
342353
}
343354
}
344355

345356
@ParameterizedTest
346357
@MethodSource( "readerBackends" )
358+
@SuppressWarnings( "java:S1612" )
347359
void testOpenBioformats2rawCollectionRootReportsMultiImage( ZarrReaderBackend backend ) throws URISyntaxException
348360
{
349361
Path path = ZarrTestUtils.resourcePath( "ome/zarr/testdata/bioformats2raw_testing/bf2raw_dataset_v5.ome.zarr" );
@@ -354,13 +366,9 @@ void testOpenBioformats2rawCollectionRootReportsMultiImage( ZarrReaderBackend ba
354366
ZarrOpeningSettings settings = new ZarrOpeningSettings();
355367
settings.setReaderBackend( backend );
356368
ZarrOpenActions actions = new ZarrOpenActions( path.toUri(), context, settings, errorHandler );
357-
AtomicInteger multiScaleCounter = new AtomicInteger( 0 );
358-
AtomicInteger singleScaleCounter = new AtomicInteger( 0 );
359-
Function< PyramidalDataset, Object > multiScaleOpener = dataset -> multiScaleCounter.incrementAndGet();
360-
Function< Img< ? >, Object > singleScaleOpener = img -> singleScaleCounter.incrementAndGet();
361-
assertDoesNotThrow( () -> actions.openImage( multiScaleOpener, singleScaleOpener ) );
362-
assertEquals( 0, multiScaleCounter.get(), "Multi-image collection must not be opened as a single multiscale image" );
363-
assertEquals( 0, singleScaleCounter.get() );
369+
assertDoesNotThrow( () -> {
370+
actions.openIJWithImage();
371+
} );
364372
assertNotNull( capturedError.get(), "Error handler should have been called for backend " + backend );
365373
assertTrue( capturedError.get().contains( "multiple images" ),
366374
"Expected multi-image message from backend, got: " + capturedError.get() );
@@ -380,37 +388,24 @@ void testOpenBioformats2rawCollectionChildOpens( ZarrReaderBackend backend ) thr
380388
for ( String childPath : childPaths )
381389
{
382390
Path path = ZarrTestUtils.resourcePath( childPath );
383-
AtomicReference< String > capturedError = new AtomicReference<>();
384-
Consumer< String > errorHandler = capturedError::set;
385-
ZarrOpeningSettings settings = new ZarrOpeningSettings();
386-
settings.setReaderBackend( backend );
387-
ZarrOpenActions actions = new ZarrOpenActions( path.toUri(), context, settings, errorHandler );
388-
AtomicInteger multiScaleCounter = new AtomicInteger( 0 );
389-
AtomicInteger singleScaleCounter = new AtomicInteger( 0 );
390-
Function< PyramidalDataset, Object > multiScaleOpener = dataset -> multiScaleCounter.incrementAndGet();
391-
Function< Img< ? >, Object > singleScaleOpener = img -> singleScaleCounter.incrementAndGet();
392-
assertDoesNotThrow( () -> actions.openImage( multiScaleOpener, singleScaleOpener ),
393-
"Opening child image " + childPath + " should not throw" );
394-
assertEquals( 1, multiScaleCounter.get(),
395-
"Child image " + childPath + " should be opened as a multiscale image" );
396-
assertEquals( 0, singleScaleCounter.get() );
397-
assertNull( capturedError.get(),
398-
"Error handler should not have been called for child " + childPath + ", got: " + capturedError.get() );
391+
assertNotNull( loadMultiscaleHeadless( path.toUri(), context, backend ),
392+
"Child image " + childPath + " should open as a multiscale image" );
399393
}
400394
}
401395
}
402396

403397
@Test
398+
@SuppressWarnings( "java:S1612" )
404399
void testOpenNonMatchingResolution() throws URISyntaxException
405400
{
406401
try (Context context = new Context())
407402
{
408403
Path path = ZarrTestUtils.resourcePath( "ome/zarr/testdata/5d_testing/5d_dataset_v4.ome.zarr" );
409404
ZarrOpeningSettings settings = new ZarrOpeningSettings( ZarrOpenBehavior.IMAGEJ_CUSTOM_RESOLUTION, 10 );
410405
ZarrOpenActions actions = new ZarrOpenActions( path.toUri(), context, settings, System.out::println );
411-
Function< PyramidalDataset, Object > multiScaleNoOp = pyramidalDataset -> null;
412-
Function< Img< ? >, Object > singleScaleNoOp = img -> null;
413-
assertDoesNotThrow( () -> actions.openImage( multiScaleNoOp, singleScaleNoOp ) );
406+
assertDoesNotThrow( () -> {
407+
actions.openIJWithImage();
408+
} );
414409
}
415410
}
416411

@@ -659,6 +654,45 @@ void testOpenSingleScaleImageInBDV( String resource ) throws URISyntaxException
659654
}
660655
}
661656

657+
// --- S3 integration tests (require network access) ---
658+
659+
static final URI S3_JANELIA_CHOROID_PLEXUS =
660+
URI.create( "s3://janelia-cosem-datasets/jrc_mus-choroid-plexus-3/jrc_mus-choroid-plexus-3.zarr/recon-1/em/fibsem-uint8" );
661+
662+
@ParameterizedTest
663+
@MethodSource( "readerBackends" )
664+
void openImageFromS3( final ZarrReaderBackend backend )
665+
{
666+
try (Context context = new Context())
667+
{
668+
final PyramidContents< ? > contents = loadMultiscaleHeadless( S3_JANELIA_CHOROID_PLEXUS, context, backend );
669+
assertNotNull( contents, "Expected the S3 dataset to open as a multiscale image for backend " + backend );
670+
}
671+
}
672+
673+
@ParameterizedTest
674+
@MethodSource( "readerBackends" )
675+
@SuppressWarnings( "java:S1612" )
676+
void storeAccessErrorIsReportedToErrorHandler( final ZarrReaderBackend backend )
677+
{
678+
try (Context context = new Context())
679+
{
680+
final URI uri = URI.create( "s3://nonexistent-bucket/some/path" );
681+
final AtomicReference< String > capturedError = new AtomicReference<>();
682+
final ZarrOpeningSettings settings = new ZarrOpeningSettings();
683+
settings.setReaderBackend( backend );
684+
685+
final ZarrOpenActions actions = new ZarrOpenActions( uri, context, settings, capturedError::set );
686+
assertDoesNotThrow( () -> {
687+
actions.openIJWithImage();
688+
}, "Store access failures must not escape openIJWithImage() for backend " + backend );
689+
690+
assertNotNull( capturedError.get(), "Error handler should have been called for backend " + backend );
691+
assertTrue( capturedError.get().contains( uri.toString() ),
692+
"Error message should contain the URI for backend " + backend + ", got: " + capturedError.get() );
693+
}
694+
}
695+
662696
@Test
663697
void testRunScriptWithNoScriptSpecified() throws URISyntaxException, InterruptedException, InvocationTargetException
664698
{

0 commit comments

Comments
 (0)