From 40a8ac5b4a5993bbfb0ce5f5772105ff2fbfa518 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 25 Nov 2025 11:23:51 -0500 Subject: [PATCH 1/4] feat: add InMemoryKeyValueAccess --- .../n5/InMemoryKeyValueAccess.java | 335 ++++++++++++++++++ .../saalfeldlab/n5/N5InMemoryTest.java | 81 +++++ 2 files changed, 416 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/N5InMemoryTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java new file mode 100644 index 000000000..fe725ca76 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java @@ -0,0 +1,335 @@ +/*- + * #%L + * Not HDF5 + * %% + * Copyright (C) 2017 - 2025 Stephan Saalfeld + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.janelia.saalfeldlab.n5; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.UncheckedIOException; +import java.io.Writer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; + +import org.janelia.saalfeldlab.n5.readdata.ReadData; + +/** + * An in-memory {@link KeyValueAccess}. + */ +public class InMemoryKeyValueAccess implements KeyValueAccess { + + public static final String SCHEME = "memory"; + public static final String SEPARATOR = "/"; + + private static final String ROOT = "/"; + + private HashMap files; + private HashSet directories; + + /** + * Creates a {@link InMemoryKeyValueAccess}. + */ + public InMemoryKeyValueAccess() { + files = new HashMap<>(); + directories = new HashSet<>(); + directories.add(ROOT); + } + + @Override + public ReadData createReadData(final String normalPath) { + return new KeyValueAccessReadData(new MemoryLazyRead(normalPath)); + } + + @Override + public LockedChannel lockForReading(final String normalPath) throws N5IOException { + + if (!isFile(normalPath)) + throw new N5NoSuchKeyException("No such key: " + normalPath); + + try { + return new MemoryLockedChannel(normalPath); + } catch (UncheckedIOException e) { + throw new N5IOException("Failed to lock file for reading: " + normalPath, e); + } + } + + @Override + public synchronized LockedChannel lockForWriting(final String normalPath) throws N5IOException { + + try { + return new MemoryLockedChannel(normalPath); + } catch (UncheckedIOException e) { + throw new N5IOException("Failed to lock file for writing: " + normalPath, e); + } + } + + @Override + public boolean isDirectory(final String normalPath) { + + return directories.contains(normalPath) || + (normalPath.endsWith("/") && directories.contains(normalPath.substring(0, normalPath.length() - 1))); + + } + + @Override + public boolean isFile(final String normalPath) { + + return files.containsKey(normalPath); + + } + + @Override + public boolean exists(final String normalPath) { + + return isDirectory(normalPath) || isFile(normalPath); + } + + @Override + public long size(final String normalPath) { + + if (!isFile(normalPath)) + throw new N5Exception.N5NoSuchKeyException("No such file: " + normalPath); + + try { + return files.get(normalPath).requireLength(); + } catch (UncheckedIOException e) { + throw new N5Exception.N5IOException(e); + } + } + + @Override + public String[] listDirectories(final String normalPath) throws N5IOException { + + if (!isDirectory(normalPath)) + throw new N5IOException("Attempted to list directory that does not exist: " + normalPath); + + return listDirectoriesHelper(normalPath).toArray(new String[0]); + } + + private ArrayList listDirectoriesHelper(final String normalPath) { + + Path dir = Paths.get(normalPath); + final ArrayList children = new ArrayList<>(); + for (String p : directories) { + + Path path = Paths.get(p); + Path parent = path.getParent(); + if (dir.equals(parent)) + children.add(parent.relativize(path).toString()); + } + return children; + } + + @Override + public String[] list(final String normalPath) throws N5IOException { + + if (!isDirectory(normalPath)) + throw new N5IOException("Attempted to list directory that does not exist: " + normalPath); + + final Path dir = Paths.get(normalPath); + final ArrayList children = new ArrayList<>(); + for (String p : files.keySet()) { + + Path path = Paths.get(p); + Path parent = path.getParent(); + if (dir.equals(parent)) + children.add(parent.relativize(path).toString()); + } + + children.addAll(listDirectoriesHelper(normalPath)); + return children.toArray(new String[0]); + } + + /** + * Returns a normalized path. It ensures correctness on both Unix and + * Windows, otherwise {@code pathName} is treated as UNC path on Windows, + * and {@code Paths.get(pathName, ...)} fails with + * {@code InvalidPathException}. + * + * @param path + * the path + * @return the normalized path, without leading slash + */ + @Override + public String normalize(final String path) { + return N5URI.normalizeGroupPath(path); + } + + /** + * Get the parent of a path string. + * + * @param path + * the path + * @return the parent path or null if the path has no parent + */ + @Override + public String parent(final String path) { + final String removeTrailingSlash = path.replaceAll("/+$", ""); + return N5URI.getAsUri(removeTrailingSlash).resolve("").toString(); + } + + @Override + public synchronized void createDirectories(final String normalPath) throws N5IOException { + + try { + String p = normalPath; + p = p.replaceAll("/+$", ""); + while( !p.equals(ROOT)) { + p = p.replaceAll("/+$", ""); + directories.add(p); + p = parent(p); + } + } catch (UncheckedIOException e) { + throw new N5IOException("Failed to create directories", e); + } + } + + @Override + public synchronized void delete(final String normalPath) throws N5IOException { + + // disallow deletion of root + if (normalPath.equals("/") || normalPath.equals("")) + return; + + deleteHelper(normalPath, directories); + deleteHelper(normalPath, files.keySet()); + } + + private void deleteHelper(String normalPath, Iterable set) { + + for (final Iterator it = set.iterator(); it.hasNext();) { + String d = it.next(); + if (d.startsWith(normalPath)) + it.remove(); + } + } + + private class MemoryLazyRead implements LazyRead { + + private final String normalKey; + + MemoryLazyRead(String normalKey) { + this.normalKey = normalKey; + } + + @Override + public long size() { + return InMemoryKeyValueAccess.this.size(normalKey); + } + + @Override + public ReadData materialize(final long offset, final long length) { + + if (!isFile(normalKey)) + throw new N5NoSuchKeyException("No such key: " + normalKey); + + try { + final ReadData rd = files.get(normalKey); + if (length > Integer.MAX_VALUE) + throw new IOException("Attempt to materialize too large data"); + + final long channelSize = rd.requireLength(); + if (!validBounds(channelSize, offset, length)) + throw new IndexOutOfBoundsException(); + + return rd.slice(offset, length); + + } catch (IOException | UncheckedIOException e) { + throw new N5Exception.N5IOException(e); + } + } + + } + + private static boolean validBounds(long channelSize, long offset, long length) { + + if (offset < 0) + return false; + else if (channelSize > 0 && offset >= channelSize) // offset == 0 and arrayLength == 0 is okay + return false; + else if (length >= 0 && offset + length > channelSize) + return false; + + return true; + } + + private class MemoryLockedChannel implements LockedChannel { + + private final String key; + + private ByteArrayOutputStream os; + + MemoryLockedChannel(final String normalKey) { + this.key = normalKey; + } + + @Override + public void close() throws IOException { + + files.put(key, ReadData.from(os.toByteArray())); + } + + @Override + public Reader newReader() throws N5IOException { + + return new InputStreamReader(newInputStream()); + } + + @Override + public InputStream newInputStream() throws N5IOException { + + return InMemoryKeyValueAccess.this.files.get(key).inputStream(); + } + + @Override + public Writer newWriter() throws N5IOException { + + return new OutputStreamWriter(newOutputStream()); + } + + @Override + public OutputStream newOutputStream() throws N5IOException { + + os = new ByteArrayOutputStream(); + return os; + } + + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5InMemoryTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5InMemoryTest.java new file mode 100644 index 000000000..8e28af427 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5InMemoryTest.java @@ -0,0 +1,81 @@ +/*- + * #%L + * Not HDF5 + * %% + * Copyright (C) 2017 - 2025 Stephan Saalfeld + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.janelia.saalfeldlab.n5; + + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Random; + +import com.google.gson.GsonBuilder; + +/** + * Initiates testing of N5 using an in-memory key value access. + */ +public class N5InMemoryTest extends AbstractN5Test { + + private static final InMemoryKeyValueAccess memAccess = new InMemoryKeyValueAccess(); + + private static Random RANDOM = new Random(999); + + private static String tempN5PathName() { + return "/tmp/n5-test-" + RANDOM.nextLong(); + } + + @Override + protected String tempN5Location() throws URISyntaxException { + + final String basePath= new File(tempN5PathName()).toURI().normalize().getPath(); + return new URI(null, null, basePath, null).toString(); + } + + @Override protected N5Writer createN5Writer() throws IOException, URISyntaxException { + + return createN5Writer(tempN5Location(), new GsonBuilder()); + } + + @Override + protected N5Writer createN5Writer( + final String location, + final GsonBuilder gson) throws IOException, URISyntaxException { + + return new N5KeyValueWriter(memAccess, location, gson, false); + } + + @Override + protected N5Reader createN5Reader( + final String location, + final GsonBuilder gson) throws IOException, URISyntaxException { + + return new N5KeyValueReader(memAccess, location, gson, false); + } + +} From c613050d20923c3829cd0891c00c79ea4e869ee4 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 7 Apr 2026 17:41:21 -0400 Subject: [PATCH 2/4] feat: update in-memory kva * to implement latest KVA interface methods --- .../n5/InMemoryKeyValueAccess.java | 88 +++---------------- 1 file changed, 14 insertions(+), 74 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java index fe725ca76..22e84f5fe 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java @@ -28,18 +28,8 @@ */ package org.janelia.saalfeldlab.n5; -import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; -import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; - -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Reader; import java.io.UncheckedIOException; -import java.io.Writer; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -47,10 +37,16 @@ import java.util.HashSet; import java.util.Iterator; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; /** * An in-memory {@link KeyValueAccess}. + *

+ * This implementation does not implement an {@link IoPolicy}. */ public class InMemoryKeyValueAccess implements KeyValueAccess { @@ -72,31 +68,15 @@ public InMemoryKeyValueAccess() { } @Override - public ReadData createReadData(final String normalPath) { - return new KeyValueAccessReadData(new MemoryLazyRead(normalPath)); - } - - @Override - public LockedChannel lockForReading(final String normalPath) throws N5IOException { - - if (!isFile(normalPath)) - throw new N5NoSuchKeyException("No such key: " + normalPath); + public VolatileReadData createReadData(final String normalPath) { - try { - return new MemoryLockedChannel(normalPath); - } catch (UncheckedIOException e) { - throw new N5IOException("Failed to lock file for reading: " + normalPath, e); - } + return VolatileReadData.from(new MemoryLazyRead(normalPath)); } @Override - public synchronized LockedChannel lockForWriting(final String normalPath) throws N5IOException { + public void write(String normalPath, ReadData data) { - try { - return new MemoryLockedChannel(normalPath); - } catch (UncheckedIOException e) { - throw new N5IOException("Failed to lock file for writing: " + normalPath, e); - } + files.put(normalPath, data); } @Override @@ -111,7 +91,6 @@ public boolean isDirectory(final String normalPath) { public boolean isFile(final String normalPath) { return files.containsKey(normalPath); - } @Override @@ -275,6 +254,10 @@ public ReadData materialize(final long offset, final long length) { } } + @Override + public void close() { + // No-op + } } private static boolean validBounds(long channelSize, long offset, long length) { @@ -289,47 +272,4 @@ else if (length >= 0 && offset + length > channelSize) return true; } - private class MemoryLockedChannel implements LockedChannel { - - private final String key; - - private ByteArrayOutputStream os; - - MemoryLockedChannel(final String normalKey) { - this.key = normalKey; - } - - @Override - public void close() throws IOException { - - files.put(key, ReadData.from(os.toByteArray())); - } - - @Override - public Reader newReader() throws N5IOException { - - return new InputStreamReader(newInputStream()); - } - - @Override - public InputStream newInputStream() throws N5IOException { - - return InMemoryKeyValueAccess.this.files.get(key).inputStream(); - } - - @Override - public Writer newWriter() throws N5IOException { - - return new OutputStreamWriter(newOutputStream()); - } - - @Override - public OutputStream newOutputStream() throws N5IOException { - - os = new ByteArrayOutputStream(); - return os; - } - - } - } From cb071feb68bf6175f1af74e02396c73e0e4aa284 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 13 Apr 2026 08:58:33 -0400 Subject: [PATCH 3/4] fix: in memory kva use thread safe data structures --- .../saalfeldlab/n5/InMemoryKeyValueAccess.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java index 22e84f5fe..f400b7851 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java @@ -30,12 +30,13 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; @@ -55,15 +56,15 @@ public class InMemoryKeyValueAccess implements KeyValueAccess { private static final String ROOT = "/"; - private HashMap files; - private HashSet directories; + private Map files; + private Set directories; /** * Creates a {@link InMemoryKeyValueAccess}. */ public InMemoryKeyValueAccess() { - files = new HashMap<>(); - directories = new HashSet<>(); + files = new ConcurrentHashMap<>(); + directories = Collections.synchronizedSet(new HashSet<>()); directories.add(ROOT); } From cfaaf187a19c50cb63a51af650539e3042eef8fb Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 13 Apr 2026 09:00:33 -0400 Subject: [PATCH 4/4] fix: in memory kva path normalization * avoid using system specific Path implementation --- .../n5/InMemoryKeyValueAccess.java | 84 +++++++------------ 1 file changed, 31 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java index f400b7851..4d541ece9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java @@ -68,6 +68,18 @@ public InMemoryKeyValueAccess() { directories.add(ROOT); } + /** + * Returns a normalized path. + * + * @param path + * the path + * @return the normalized path + */ + @Override + public String normalize(final String path) { + return N5URI.normalizeGroupPath(path); + } + @Override public VolatileReadData createReadData(final String normalPath) { @@ -124,14 +136,12 @@ public String[] listDirectories(final String normalPath) throws N5IOException { private ArrayList listDirectoriesHelper(final String normalPath) { - Path dir = Paths.get(normalPath); final ArrayList children = new ArrayList<>(); - for (String p : directories) { - - Path path = Paths.get(p); - Path parent = path.getParent(); - if (dir.equals(parent)) - children.add(parent.relativize(path).toString()); + for (String d : directories) { + // the parent method removes leading slash, so add it back + String parent = "/" + parent(d); + if (normalPath.equals(parent)) + children.add(relativize(d, parent)); } return children; } @@ -142,69 +152,37 @@ public String[] list(final String normalPath) throws N5IOException { if (!isDirectory(normalPath)) throw new N5IOException("Attempted to list directory that does not exist: " + normalPath); - final Path dir = Paths.get(normalPath); final ArrayList children = new ArrayList<>(); + // all keys are normalized for (String p : files.keySet()) { - - Path path = Paths.get(p); - Path parent = path.getParent(); - if (dir.equals(parent)) - children.add(parent.relativize(path).toString()); + final String parent = parent(p); + if (normalPath.equals(parent)) + children.add(relativize(p, parent)); } children.addAll(listDirectoriesHelper(normalPath)); return children.toArray(new String[0]); } - /** - * Returns a normalized path. It ensures correctness on both Unix and - * Windows, otherwise {@code pathName} is treated as UNC path on Windows, - * and {@code Paths.get(pathName, ...)} fails with - * {@code InvalidPathException}. - * - * @param path - * the path - * @return the normalized path, without leading slash - */ @Override - public String normalize(final String path) { - return N5URI.normalizeGroupPath(path); - } + public void createDirectories(final String normalPath) throws N5IOException { - /** - * Get the parent of a path string. - * - * @param path - * the path - * @return the parent path or null if the path has no parent - */ - @Override - public String parent(final String path) { - final String removeTrailingSlash = path.replaceAll("/+$", ""); - return N5URI.getAsUri(removeTrailingSlash).resolve("").toString(); + String p = normalPath; + while( !isRoot(p)) { + directories.add(p); + p = "/" + parent(p); + } } - @Override - public synchronized void createDirectories(final String normalPath) throws N5IOException { - - try { - String p = normalPath; - p = p.replaceAll("/+$", ""); - while( !p.equals(ROOT)) { - p = p.replaceAll("/+$", ""); - directories.add(p); - p = parent(p); - } - } catch (UncheckedIOException e) { - throw new N5IOException("Failed to create directories", e); - } + private boolean isRoot(String path) { + return path.isEmpty() || path.equals(ROOT); } @Override - public synchronized void delete(final String normalPath) throws N5IOException { + public void delete(final String normalPath) throws N5IOException { // disallow deletion of root - if (normalPath.equals("/") || normalPath.equals("")) + if (isRoot(normalPath)) return; deleteHelper(normalPath, directories);