diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index a320b093b..0bf0fbe06 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -28,25 +28,16 @@ */ package org.janelia.saalfeldlab.n5; -import java.nio.file.StandardOpenOption; +import java.io.*; +import java.nio.file.*; + import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.net.URI; import java.net.URISyntaxException; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.FileSystem; -import java.nio.file.FileSystemException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.attribute.FileAttribute; import java.util.Arrays; import java.util.Comparator; @@ -68,49 +59,62 @@ */ public class FileSystemKeyValueAccess implements KeyValueAccess { - protected final FileSystem fileSystem; - - /** - * Opens a {@link FileSystemKeyValueAccess} with a {@link FileSystem}. - * - * @param fileSystem the file system - */ - public FileSystemKeyValueAccess(final FileSystem fileSystem) { - - this.fileSystem = fileSystem; + private static final IoPolicy ioPolicy = getIoPolicy(); + + private static IoPolicy getIoPolicy() { + String property = System.getProperty("n5.ioPolicy"); + if (property == null) + return FsIoPolicy.atomicWithFallback; + + switch (property) { + case "atomic": + return new FsIoPolicy.Atomic(); + case "unsafe": + return new FsIoPolicy.Unsafe(); + case "atomicFallbackUnsafe": + default: + return FsIoPolicy.atomicWithFallback; + } } @Override - public VolatileReadData createReadData(final String normalPath) { + public VolatileReadData createReadData(final String normalPath) throws N5IOException { + try { - return VolatileReadData.from(new FileLazyRead(fileSystem.getPath(normalPath))); - } catch (N5NoSuchKeyException e) { - return null; + return ioPolicy.read(normalPath); + } catch (final NoSuchFileException e) { + throw new N5NoSuchKeyException("No such file", e); + } catch (IOException | UncheckedIOException e) { + throw new N5IOException("Failed to lock file for reading: " + normalPath, e); } } @Override public void write(final String normalPath, final ReadData data) throws N5IOException { - final Path path = fileSystem.getPath(normalPath); - try (final LockedFileChannel channel = lockForWriting(path)) { - data.writeTo(channel.newOutputStream()); - } catch (IOException e) { - throw new N5IOException(e); + try { + ioPolicy.write(normalPath, data); + } catch (final NoSuchFileException e) { + throw new N5NoSuchKeyException("No such file", e); + } catch (IOException | UncheckedIOException e) { + throw new N5IOException("Failed to lock file for writing: " + normalPath, e); } } @Override + @Deprecated public LockedFileChannel lockForReading(final String normalPath) throws N5IOException { - return lockForReading(fileSystem.getPath(normalPath)); + return lockForReading(Paths.get(normalPath)); } @Override + @Deprecated public LockedChannel lockForWriting(final String normalPath) throws N5IOException { - return lockForWriting(fileSystem.getPath(normalPath)); + return lockForWriting(Paths.get(normalPath)); } + @Deprecated protected static LockedFileChannel lockForReading(final Path path) throws N5IOException { try { @@ -122,6 +126,7 @@ protected static LockedFileChannel lockForReading(final Path path) throws N5IOEx } } + @Deprecated static LockedFileChannel lockForWriting(final Path path) throws N5IOException { try { @@ -136,28 +141,28 @@ static LockedFileChannel lockForWriting(final Path path) throws N5IOException { @Override public boolean isDirectory(final String normalPath) { - final Path path = fileSystem.getPath(normalPath); + final Path path = Paths.get(normalPath); return Files.isDirectory(path); } @Override public boolean isFile(final String normalPath) { - final Path path = fileSystem.getPath(normalPath); + final Path path = Paths.get(normalPath); return Files.isRegularFile(path); } @Override public boolean exists(final String normalPath) { - final Path path = fileSystem.getPath(normalPath); + final Path path = Paths.get(normalPath); return Files.exists(path); } @Override public long size(final String normalPath) { - return size(fileSystem.getPath(normalPath)); + return size(Paths.get(normalPath)); } protected static long size(final Path path) { @@ -165,16 +170,16 @@ protected static long size(final Path path) { try { return Files.size(path); } catch (NoSuchFileException e) { - throw new N5Exception.N5NoSuchKeyException("No such file", e); + throw new N5NoSuchKeyException("No such file", e); } catch (IOException | UncheckedIOException e) { - throw new N5Exception.N5IOException(e); + throw new N5IOException(e); } } @Override public String[] listDirectories(final String normalPath) throws N5IOException { - final Path path = fileSystem.getPath(normalPath); + final Path path = Paths.get(normalPath); try (final Stream pathStream = Files.list(path)) { return pathStream .filter(Files::isDirectory) @@ -190,7 +195,7 @@ public String[] listDirectories(final String normalPath) throws N5IOException { @Override public String[] list(final String normalPath) throws N5IOException { - final Path path = fileSystem.getPath(normalPath); + final Path path = Paths.get(normalPath); try (final Stream pathStream = Files.list(path)) { return pathStream .map(a -> path.relativize(a).toString()) @@ -205,8 +210,9 @@ public String[] list(final String normalPath) throws N5IOException { @Override public String[] components(final String path) { - final Path fsPath = fileSystem.getPath(path); + final Path fsPath = Paths.get(path); final Path root = fsPath.getRoot(); + final String separator = fsPath.getFileSystem().getSeparator(); final String[] components; int o; if (root == null) { @@ -222,7 +228,6 @@ public String[] components(final String path) { String name = fsPath.getName(i - o).toString(); /* Preserve trailing slash on final component if present*/ if (i == components.length - 1) { - final String separator = fileSystem.getSeparator(); final String trailingSeparator = path.endsWith(separator) ? separator : path.endsWith("/") ? "/" : ""; name += trailingSeparator; } @@ -234,7 +239,7 @@ public String[] components(final String path) { @Override public String parent(final String path) { - final Path parent = fileSystem.getPath(path).getParent(); + final Path parent = Paths.get(path).getParent(); if (parent == null) return null; else @@ -244,8 +249,8 @@ public String parent(final String path) { @Override public String relativize(final String path, final String base) { - final Path basePath = fileSystem.getPath(base); - return basePath.relativize(fileSystem.getPath(path)).toString(); + final Path basePath = Paths.get(base); + return basePath.relativize(Paths.get(path)).toString(); } /** @@ -260,7 +265,7 @@ public String relativize(final String path, final String base) { @Override public String normalize(final String path) { - return fileSystem.getPath(path).normalize().toString(); + return Paths.get(path).normalize().toString(); } @Override @@ -283,8 +288,8 @@ public String compose(final String... components) { if (components == null || components.length == 0) return null; if (components.length == 1) - return fileSystem.getPath(components[0]).toString(); - return fileSystem.getPath(components[0], Arrays.copyOfRange(components, 1, components.length)).normalize().toString(); + return Paths.get(components[0]).toString(); + return Paths.get(components[0], Arrays.copyOfRange(components, 1, components.length)).normalize().toString(); } @Override public String compose(URI uri, String... components) { @@ -307,7 +312,7 @@ public String compose(final String... components) { public void createDirectories(final String normalPath) throws N5IOException { try { - createDirectories(fileSystem.getPath(normalPath)); + createDirectories(Paths.get(normalPath)); } catch (NoSuchFileException e) { throw new N5NoSuchKeyException("No such file", e); } catch (IOException | UncheckedIOException e) { @@ -319,25 +324,23 @@ public void createDirectories(final String normalPath) throws N5IOException { public void delete(final String normalPath) throws N5IOException { try { - final Path path = fileSystem.getPath(normalPath); + final Path path = Paths.get(normalPath); if (Files.isRegularFile(path)) - try (final LockedChannel channel = lockForWriting(path)) { - Files.delete(path); - } + ioPolicy.delete(normalPath); else { try (final Stream pathStream = Files.walk(path)) { - for (final Iterator i = pathStream.sorted(Comparator.reverseOrder()).iterator(); i.hasNext();) { + for (final Iterator i = pathStream.sorted(Comparator.reverseOrder()).iterator(); i.hasNext(); ) { final Path childPath = i.next(); if (Files.isRegularFile(childPath)) - try (final LockedChannel channel = lockForWriting(childPath)) { - Files.delete(childPath); - } + ioPolicy.delete(childPath.toString()); else tryDelete(childPath); } } } + } catch (NoSuchFileException ignore) { + /* It doesn't exist; that's sufficient for us to not complain on a `delete` call */ } catch (IOException | UncheckedIOException e) { throw new N5IOException("Failed to delete file at " + normalPath, e); } @@ -346,7 +349,7 @@ public void delete(final String normalPath) throws N5IOException { protected static void tryDelete(final Path path) throws IOException { try { - Files.delete(path); + ioPolicy.delete(path.toString()); } catch (final DirectoryNotEmptyException e) { /* * Even though path is expected to be an empty directory, sometimes @@ -356,7 +359,7 @@ protected static void tryDelete(final Path path) throws IOException { try { /* wait and reattempt */ Thread.sleep(100); - Files.delete(path); + ioPolicy.delete(path.toString()); } catch (final InterruptedException ex) { e.printStackTrace(); Thread.currentThread().interrupt(); @@ -499,14 +502,23 @@ protected static void createAndCheckIsDirectory( } } - private static class FileLazyRead implements LazyRead { + static class FileLazyRead implements LazyRead { + + private static final Closeable NO_OP = () -> { }; private final Path path; - private LockedFileChannel lock; + private Closeable lock; + + FileLazyRead(final Path path) throws IOException { + this(path, true); + } - FileLazyRead(final Path path) { + FileLazyRead(final Path path, final boolean requireLock ) throws IOException { this.path = path; - lock = lockForReading(path); + if (requireLock) + lock = FILE_LOCK_MANAGER.lockForReading(path); + else + lock = NO_OP; } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java new file mode 100644 index 000000000..457896acd --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java @@ -0,0 +1,61 @@ +package org.janelia.saalfeldlab.n5; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.*; + +import static org.janelia.saalfeldlab.n5.FileKeyLockManager.FILE_LOCK_MANAGER; + +public class FsIoPolicy { + + static final IoPolicy atomicWithFallback = IoPolicy.withFallback(new Atomic(), new Unsafe()); + + public static class Unsafe implements IoPolicy { + @Override + public void write(String key, ReadData readData) throws IOException { + final Path path = Paths.get(key); + Files.copy(readData.inputStream(), path, StandardCopyOption.REPLACE_EXISTING); + } + + @Override + public VolatileReadData read(final String key) throws IOException { + final Path path = Paths.get(key); + FileSystemKeyValueAccess.FileLazyRead fileLazyRead = new FileSystemKeyValueAccess.FileLazyRead(path, false); + return VolatileReadData.from(fileLazyRead); + } + + @Override + public void delete(final String key) throws IOException { + final Path path = Paths.get(key); + Files.deleteIfExists(path); + } + } + + public static class Atomic implements IoPolicy { + @Override + public void write(String key, ReadData readData) throws IOException { + final Path path = Paths.get(key); + try (LockedFileChannel channel = FILE_LOCK_MANAGER.lockForWriting(path)) { + readData.writeTo(channel.newOutputStream()); + } + } + + @Override + public VolatileReadData read(String key) throws IOException { + final Path path = Paths.get(key); + FileSystemKeyValueAccess.FileLazyRead fileLazyRead = new FileSystemKeyValueAccess.FileLazyRead(path, true); + return VolatileReadData.from(fileLazyRead); + } + + @Override + public void delete(final String key) throws IOException { + final Path path = Paths.get(key); + try (LockedFileChannel ignore = FILE_LOCK_MANAGER.lockForWriting(path)) { + Files.delete(path); + } + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java b/src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java new file mode 100644 index 000000000..b1dcb3044 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/IoPolicy.java @@ -0,0 +1,47 @@ +package org.janelia.saalfeldlab.n5; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; + +import java.io.IOException; + +public interface IoPolicy { + + void write(String key, ReadData readData) throws IOException; + + VolatileReadData read(String key) throws IOException; + + void delete(String key) throws IOException; + + static IoPolicy withFallback(IoPolicy primary, IoPolicy fallback) { + return new IoPolicy() { + + @Override + public void write(String key, ReadData readData) throws IOException { + try { + primary.write(key, readData); + } catch (IOException e) { + fallback.write(key, readData); + } + } + + @Override + public VolatileReadData read(String key) throws IOException { + try { + return primary.read(key); + } catch (IOException e) { + return fallback.read(key); + } + } + + @Override + public void delete(String key) throws IOException { + try { + primary.delete(key); + } catch (IOException e) { + fallback.delete(key); + } + } + }; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index ac94567f5..e6465c5b8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -249,14 +249,15 @@ default URI uri(final String uriString) throws URISyntaxException { * If supported by this KeyValueAccess implementation, partial reads are * possible by {@link ReadData#slice slicing} the returned {@code ReadData}. *

- * If the requested key does not exist, either {@code null} is returned or a - * lazy {@code VolatileReadData} that will throw {@code N5NoSuchKeyException} - * when trying to materialize. + * The resulting {@code VolatileReadData} is potentially lazy. If the requested + * key does not exist, it will throw {@code N5NoSuchKeyException}. Whether + * the exception is thrown when {@link KeyValueAccess#createReadData(String)}] is called, + * or when trying to materialize the {@code VolatileReadData} is implementation dependent. * * @param normalPath * is expected to be in normalized form, no further efforts are made to normalize it * - * @return a ReadData or null + * @return a ReadData * * @throws N5IOException * if an error occurs @@ -293,7 +294,9 @@ default URI uri(final String uriString) throws URISyntaxException { * @return the locked channel * @throws N5IOException * if a locked channel could not be created + * @deprecated migrate to {@link KeyValueAccess#createReadData(String)} */ + @Deprecated LockedChannel lockForReading( final String normalPath ) throws N5IOException; /** @@ -312,7 +315,9 @@ default URI uri(final String uriString) throws URISyntaxException { * @return the locked channel * @throws N5IOException * if a locked channel could not be created + * @deprecated migrate to {@link KeyValueAccess#write(String, ReadData)} */ + @Deprecated LockedChannel lockForWriting( final String normalPath ) throws N5IOException; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 4397b4544..70904544a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -69,7 +69,7 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final bo throws N5Exception { super( - new FileSystemKeyValueAccess(FileSystems.getDefault()), + new FileSystemKeyValueAccess(), basePath, gsonBuilder, cacheMeta); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index d856ebb22..3a57881dd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -70,7 +70,7 @@ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final bo throws N5Exception { super( - new FileSystemKeyValueAccess(FileSystems.getDefault()), + new FileSystemKeyValueAccess(), basePath, gsonBuilder, cacheAttributes); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java index 36a1645fb..80035b958 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/shard/PositionValueAccess.java @@ -119,7 +119,11 @@ protected String absolutePath(final long... gridPosition) { @Override public VolatileReadData get(final long[] key) throws N5IOException { - return kva.createReadData(absolutePath(key)); + try { + return kva.createReadData(absolutePath(key)); + } catch (N5Exception.N5NoSuchKeyException e) { + return null; + } } @Override diff --git a/src/test/java/org/janelia/saalfeldlab/n5/FsLockTest.java b/src/test/java/org/janelia/saalfeldlab/n5/FsLockTest.java new file mode 100644 index 000000000..89836200a --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/FsLockTest.java @@ -0,0 +1,207 @@ +package org.janelia.saalfeldlab.n5; + +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class FsLockTest { + + private static final FileKeyLockManager LOCK_MANAGER = FileKeyLockManager.FILE_LOCK_MANAGER; + + private static String tempPathName() { + + try { + final File tmpFile = Files.createTempDirectory("fs-key-lock-test-").toFile(); + tmpFile.delete(); + tmpFile.mkdir(); + tmpFile.deleteOnExit(); + return tmpFile.getCanonicalPath(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void testReadLock() throws IOException { + + final Path path = Paths.get(tempPathName(), "lock"); + path.toFile().createNewFile(); + assertTrue("File Created", path.toFile().exists()); + LockedChannel lock = LOCK_MANAGER.lockForReading(path); + lock.close(); + lock = LOCK_MANAGER.lockForReading(path); + + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future future = exec.submit(() -> { + LOCK_MANAGER.lockForWriting(path).close(); + return null; + }); + + try { + System.out.println("Trying to acquire locked readable channel..."); + System.out.println(future.get(3, TimeUnit.SECONDS)); + fail("Lock broken!"); + } catch (final TimeoutException e) { + System.out.println("Lock held!"); + future.cancel(true); + } catch (final InterruptedException | ExecutionException e) { + future.cancel(true); + System.out.println("Test was interrupted!"); + } finally { + lock.close(); + Files.delete(path); + } + + exec.shutdownNow(); + } + + @Test + public void testWriteLock() throws IOException { + + final Path path = Paths.get(tempPathName(), "lock"); + final LockedChannel lock = LOCK_MANAGER.lockForWriting(path); + System.out.println("locked"); + + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future future = exec.submit(() -> { + LOCK_MANAGER.lockForReading(path).close(); + return null; + }); + + try { + System.out.println("Trying to acquire locked writable channel..."); + System.out.println(future.get(3, TimeUnit.SECONDS)); + fail("Lock broken!"); + } catch (final TimeoutException e) { + System.out.println("Lock held!"); + future.cancel(true); + } catch (final InterruptedException | ExecutionException e) { + future.cancel(true); + System.out.println("Test was interrupted!"); + } finally { + lock.close(); + Files.delete(path); + } + + exec.shutdownNow(); + } + + @Test + public void testFSLockRelease() throws IOException, ExecutionException, InterruptedException, TimeoutException { + + + final Path path = Paths.get(tempPathName(), "lock"); + final ExecutorService exec = Executors.newFixedThreadPool(2); + + // first thread acquires the lock, waits for 200ms then should release it + exec.submit(() -> { + try { + try(final LockedChannel lock = LOCK_MANAGER.lockForWriting(path)) { + lock.newReader(); + Thread.sleep(200); + } + } catch (IOException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }); + + // second thread waits for the lock. + // it should get it within a few seconds. + final Future future = exec.submit(() -> { + LOCK_MANAGER.lockForWriting(path).close(); + return null; + }); + + future.get(3, TimeUnit.SECONDS); + Files.delete(path); + exec.shutdownNow(); + } + + @Test + public void testReadLockBehavior() throws IOException, InterruptedException, ExecutionException, TimeoutException { + + final Path path = Paths.get(tempPathName(), "read-lock"); + path.toFile().createNewFile(); + + final ExecutorService exec = Executors.newFixedThreadPool(3); + + final AtomicBoolean v = new AtomicBoolean(false); + + // first thread acquires a read lock, waits for 200ms + Future f = exec.submit(() -> { + try { + try(final LockedChannel lock = LOCK_MANAGER.lockForReading(path)) { + lock.newReader(); + Thread.sleep(200); + + // ensure that the other thread updated the value + assertTrue(v.get()); + + } + } catch (IOException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }); + + // second thread gets a read lock + // and should not be blocked + // this thread updates the boolean + exec.submit(() -> { + try( final LockedChannel lock = LOCK_MANAGER.lockForReading(path)) { + lock.newReader(); + v.set(true); + } + return null; + }); + + f.get(3, TimeUnit.SECONDS); + exec.shutdownNow(); + Files.delete(path); + } + + @Test + public void testWriteLockBehavior() throws IOException, ExecutionException, InterruptedException, TimeoutException { + + + final Path path = Paths.get(tempPathName(), "lock"); + final ExecutorService exec = Executors.newFixedThreadPool(2); + + // first thread acquires the lock, waits for 200ms then should release it + exec.submit(() -> { + try { + try(final LockedChannel lock = LOCK_MANAGER.lockForWriting(path)) { + lock.newReader(); + Thread.sleep(200); + } + } catch (IOException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + }); + + // second thread waits for the lock. + // it should get it within a few seconds. + final Future future = exec.submit(() -> { + LOCK_MANAGER.lockForWriting(path).close(); + return null; + }); + + future.get(3, TimeUnit.SECONDS); + Files.delete(path); + exec.shutdownNow(); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 4a7a94b8b..cb9467b74 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -171,7 +171,7 @@ public void cacheBehaviorTest() throws IOException, URISyntaxException { final String loc = tempN5Location(); // make an uncached n5 writer - try (final N5TrackingStorage n5 = new N5TrackingStorage(new FileSystemKeyValueAccess(FileSystems.getDefault()), loc, + try (final N5TrackingStorage n5 = new N5TrackingStorage(new FileSystemKeyValueAccess(), loc, new GsonBuilder(), true)) { cacheBehaviorHelper(n5); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 64bd88a3c..baca25ea9 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -41,7 +41,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.attribute.FileAttribute; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -49,9 +48,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import org.junit.Ignore; import org.junit.Test; import com.google.gson.GsonBuilder; @@ -64,8 +61,6 @@ */ public class N5FSTest extends AbstractN5Test { - private static final FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); - private static String tempN5PathName() { try { @@ -107,173 +102,4 @@ protected N5Reader createN5Reader( return new N5FSReader(location, gson); } - - @Test - public void testReadLock() throws IOException { - - final Path path = Paths.get(tempN5PathName(), "lock"); - LockedChannel lock = access.lockForWriting(path); - lock.close(); - lock = access.lockForReading(path); - System.out.println("locked"); - - final ExecutorService exec = Executors.newSingleThreadExecutor(); - final Future future = exec.submit(() -> { - access.lockForWriting(path).close(); - return null; - }); - - try { - System.out.println("Trying to acquire locked readable channel..."); - System.out.println(future.get(3, TimeUnit.SECONDS)); - fail("Lock broken!"); - } catch (final TimeoutException e) { - System.out.println("Lock held!"); - future.cancel(true); - } catch (final InterruptedException | ExecutionException e) { - future.cancel(true); - System.out.println("Test was interrupted!"); - } finally { - lock.close(); - Files.delete(path); - } - - exec.shutdownNow(); - } - - @Test - public void testWriteLock() throws IOException { - - final Path path = Paths.get(tempN5PathName(), "lock"); - final LockedChannel lock = access.lockForWriting(path); - System.out.println("locked"); - - final ExecutorService exec = Executors.newSingleThreadExecutor(); - final Future future = exec.submit(() -> { - access.lockForReading(path).close(); - return null; - }); - - try { - System.out.println("Trying to acquire locked writable channel..."); - System.out.println(future.get(3, TimeUnit.SECONDS)); - fail("Lock broken!"); - } catch (final TimeoutException e) { - System.out.println("Lock held!"); - future.cancel(true); - } catch (final InterruptedException | ExecutionException e) { - future.cancel(true); - System.out.println("Test was interrupted!"); - } finally { - lock.close(); - Files.delete(path); - } - - exec.shutdownNow(); - } - - @Test - public void testFSLockRelease() throws IOException, ExecutionException, InterruptedException, TimeoutException { - - - final Path path = Paths.get(tempN5PathName(), "lock"); - final ExecutorService exec = Executors.newFixedThreadPool(2); - - // first thread acquires the lock, waits for 200ms then should release it - exec.submit(() -> { - try( final LockedChannel lock = access.lockForWriting(path)) { - lock.newReader(); - Thread.sleep(200); - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - }); - - // second thread waits for the lock. - // it should get it within a few seconds. - final Future future = exec.submit(() -> { - access.lockForWriting(path).close(); - return null; - }); - - future.get(3, TimeUnit.SECONDS); - Files.delete(path); - exec.shutdownNow(); - } - - @Test - public void testReadLockBehavior() throws IOException, InterruptedException, ExecutionException, TimeoutException { - - final Path path = Paths.get(tempN5PathName(), "read-lock"); - path.toFile().createNewFile(); - - final ExecutorService exec = Executors.newFixedThreadPool(3); - - final AtomicBoolean v = new AtomicBoolean(false); - - // first thread acquires a read lock, waits for 200ms - Future f = exec.submit(() -> { - try( final LockedChannel lock = access.lockForReading(path)) { - lock.newReader(); - Thread.sleep(200); - - // ensure that the other thread updated the value - assertTrue(v.get()); - - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - }); - - // second thread gets a read lock - // and should not be blocked - // this thread updates the boolean - exec.submit(() -> { - try( final LockedChannel lock = access.lockForReading(path)) { - lock.newReader(); - v.set(true); - } - return null; - }); - - f.get(3, TimeUnit.SECONDS); - exec.shutdownNow(); - Files.delete(path); - } - - @Test - public void testWriteLockBehavior() throws IOException, ExecutionException, InterruptedException, TimeoutException { - - - final Path path = Paths.get(tempN5PathName(), "lock"); - final ExecutorService exec = Executors.newFixedThreadPool(2); - - // first thread acquires the lock, waits for 200ms then should release it - exec.submit(() -> { - try( final LockedChannel lock = access.lockForWriting(path)) { - lock.newReader(); - Thread.sleep(200); - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - }); - - // second thread waits for the lock. - // it should get it within a few seconds. - final Future future = exec.submit(() -> { - access.lockForWriting(path).close(); - return null; - }); - - future.get(3, TimeUnit.SECONDS); - Files.delete(path); - exec.shutdownNow(); - } - } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java index 01802d879..ffa35156f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/N5BlockWriteBenchmarks.java @@ -113,7 +113,7 @@ public void setup() { File tmpDir; try { tmpDir = Files.createTempDirectory("n5-blockWriteBenchmark-").toFile(); - FileSystemKeyValueAccess kva = new FileSystemKeyValueAccess(FileSystems.getDefault()); + FileSystemKeyValueAccess kva = new FileSystemKeyValueAccess(); n5 = new N5KeyValueWriter(kva, tmpDir.getAbsolutePath(), new GsonBuilder(), true); int[] blockSize = new int[numDimensions]; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java index 9dcc81193..19254700d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/benchmarks/ReadDataBenchmarks.java @@ -109,7 +109,7 @@ protected Path getPath() { public void setup() throws IOException { random = new Random(); - kva = new FileSystemKeyValueAccess(FileSystems.getDefault()); + kva = new FileSystemKeyValueAccess(); basePath = Files.createTempDirectory("ReadDataBenchmark-"); tmpPaths = new ArrayList<>(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java index ff1bf517f..9f017a133 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/kva/FileSystemKeyValueAccessTest.java @@ -57,7 +57,7 @@ public class FileSystemKeyValueAccessTest extends AbstractKeyValueAccessTest { private static String separator = FileSystems.getDefault().getSeparator(); - private static final FileSystemKeyValueAccess fileSystemKva = new FileSystemKeyValueAccess(FileSystems.getDefault()); + private static final FileSystemKeyValueAccess fileSystemKva = new FileSystemKeyValueAccess(); @Override protected KeyValueAccess newKeyValueAccess(URI root) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java index 220a5a497..c4f3fa51b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/readdata/ReadDataTests.java @@ -113,7 +113,7 @@ public void testFileKvaReadData() throws IOException { os.write(data); } - try( final VolatileReadData readData = new FileSystemKeyValueAccess(FileSystems.getDefault()) + try( final VolatileReadData readData = new FileSystemKeyValueAccess() .createReadData(tmpF.getAbsolutePath())) { assertEquals("file read data length", -1, readData.length()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerializationTest.java similarity index 99% rename from src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java rename to src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerializationTest.java index 31861b250..b06cae606 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerialization.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/serialization/CodecSerializationTest.java @@ -47,7 +47,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; -public class CodecSerialization { +public class CodecSerializationTest { private Gson gson; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java index a4118d753..97d45cc6c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/shard/ShardTest.java @@ -37,12 +37,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; +import java.nio.file.*; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -691,7 +686,7 @@ public static class TrackingN5Writer extends N5KeyValueWriter { final TrackingFileSystemKeyValueAccess tkva; public TrackingN5Writer(String basePath) { - super( new TrackingFileSystemKeyValueAccess(FileSystems.getDefault()), basePath, new GsonBuilder(), false); + super( new TrackingFileSystemKeyValueAccess(), basePath, new GsonBuilder(), false); tkva = (TrackingFileSystemKeyValueAccess)getKeyValueAccess(); } @@ -732,8 +727,8 @@ private static class TrackingFileSystemKeyValueAccess extends FileSystemKeyValue private int numIsFileCalls = 0; private long totalBytesRead = 0; - protected TrackingFileSystemKeyValueAccess(FileSystem fileSystem) { - super(fileSystem); + protected TrackingFileSystemKeyValueAccess() { + super(); } @Override @@ -745,7 +740,7 @@ public boolean isFile(String normalPath) { @Override public VolatileReadData createReadData(final String normalPath) { try { - return VolatileReadData.from(new TrackingFileLazyRead(fileSystem.getPath(normalPath))); + return VolatileReadData.from(new TrackingFileLazyRead(Paths.get(normalPath))); } catch (N5NoSuchKeyException e) { // return VolatileReadData.from(new NoSuchKeyLazyRead()); return null;