diff --git a/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java new file mode 100644 index 00000000..4d541ece --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/InMemoryKeyValueAccess.java @@ -0,0 +1,254 @@ +/*- + * #%L + * Not HDF5 + * %% + * Copyright (C) 2017 - 2025 Stephan Saalfeld + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; +import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException; +import org.janelia.saalfeldlab.n5.readdata.LazyRead; +import org.janelia.saalfeldlab.n5.readdata.ReadData; +import org.janelia.saalfeldlab.n5.readdata.VolatileReadData; + +/** + * An in-memory {@link KeyValueAccess}. + *

+ * This implementation does not implement an {@link IoPolicy}. + */ +public class InMemoryKeyValueAccess implements KeyValueAccess { + + public static final String SCHEME = "memory"; + public static final String SEPARATOR = "/"; + + private static final String ROOT = "/"; + + private Map files; + private Set directories; + + /** + * Creates a {@link InMemoryKeyValueAccess}. + */ + public InMemoryKeyValueAccess() { + files = new ConcurrentHashMap<>(); + directories = Collections.synchronizedSet(new HashSet<>()); + directories.add(ROOT); + } + + /** + * Returns a normalized path. + * + * @param path + * the path + * @return the normalized path + */ + @Override + public String normalize(final String path) { + return N5URI.normalizeGroupPath(path); + } + + @Override + public VolatileReadData createReadData(final String normalPath) { + + return VolatileReadData.from(new MemoryLazyRead(normalPath)); + } + + @Override + public void write(String normalPath, ReadData data) { + + files.put(normalPath, data); + } + + @Override + public boolean isDirectory(final String normalPath) { + + return directories.contains(normalPath) || + (normalPath.endsWith("/") && directories.contains(normalPath.substring(0, normalPath.length() - 1))); + + } + + @Override + public boolean isFile(final String normalPath) { + + return files.containsKey(normalPath); + } + + @Override + public boolean exists(final String normalPath) { + + return isDirectory(normalPath) || isFile(normalPath); + } + + @Override + public long size(final String normalPath) { + + if (!isFile(normalPath)) + throw new N5Exception.N5NoSuchKeyException("No such file: " + normalPath); + + try { + return files.get(normalPath).requireLength(); + } catch (UncheckedIOException e) { + throw new N5Exception.N5IOException(e); + } + } + + @Override + public String[] listDirectories(final String normalPath) throws N5IOException { + + if (!isDirectory(normalPath)) + throw new N5IOException("Attempted to list directory that does not exist: " + normalPath); + + return listDirectoriesHelper(normalPath).toArray(new String[0]); + } + + private ArrayList listDirectoriesHelper(final String normalPath) { + + final ArrayList children = new ArrayList<>(); + for (String d : directories) { + // the parent method removes leading slash, so add it back + String parent = "/" + parent(d); + if (normalPath.equals(parent)) + children.add(relativize(d, parent)); + } + return children; + } + + @Override + public String[] list(final String normalPath) throws N5IOException { + + if (!isDirectory(normalPath)) + throw new N5IOException("Attempted to list directory that does not exist: " + normalPath); + + final ArrayList children = new ArrayList<>(); + // all keys are normalized + for (String p : files.keySet()) { + final String parent = parent(p); + if (normalPath.equals(parent)) + children.add(relativize(p, parent)); + } + + children.addAll(listDirectoriesHelper(normalPath)); + return children.toArray(new String[0]); + } + + @Override + public void createDirectories(final String normalPath) throws N5IOException { + + String p = normalPath; + while( !isRoot(p)) { + directories.add(p); + p = "/" + parent(p); + } + } + + private boolean isRoot(String path) { + return path.isEmpty() || path.equals(ROOT); + } + + @Override + public void delete(final String normalPath) throws N5IOException { + + // disallow deletion of root + if (isRoot(normalPath)) + return; + + deleteHelper(normalPath, directories); + deleteHelper(normalPath, files.keySet()); + } + + private void deleteHelper(String normalPath, Iterable set) { + + for (final Iterator it = set.iterator(); it.hasNext();) { + String d = it.next(); + if (d.startsWith(normalPath)) + it.remove(); + } + } + + private class MemoryLazyRead implements LazyRead { + + private final String normalKey; + + MemoryLazyRead(String normalKey) { + this.normalKey = normalKey; + } + + @Override + public long size() { + return InMemoryKeyValueAccess.this.size(normalKey); + } + + @Override + public ReadData materialize(final long offset, final long length) { + + if (!isFile(normalKey)) + throw new N5NoSuchKeyException("No such key: " + normalKey); + + try { + final ReadData rd = files.get(normalKey); + if (length > Integer.MAX_VALUE) + throw new IOException("Attempt to materialize too large data"); + + final long channelSize = rd.requireLength(); + if (!validBounds(channelSize, offset, length)) + throw new IndexOutOfBoundsException(); + + return rd.slice(offset, length); + + } catch (IOException | UncheckedIOException e) { + throw new N5Exception.N5IOException(e); + } + } + + @Override + public void close() { + // No-op + } + } + + private static boolean validBounds(long channelSize, long offset, long length) { + + if (offset < 0) + return false; + else if (channelSize > 0 && offset >= channelSize) // offset == 0 and arrayLength == 0 is okay + return false; + else if (length >= 0 && offset + length > channelSize) + return false; + + return true; + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5InMemoryTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5InMemoryTest.java new file mode 100644 index 00000000..8e28af42 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5InMemoryTest.java @@ -0,0 +1,81 @@ +/*- + * #%L + * Not HDF5 + * %% + * Copyright (C) 2017 - 2025 Stephan Saalfeld + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.janelia.saalfeldlab.n5; + + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Random; + +import com.google.gson.GsonBuilder; + +/** + * Initiates testing of N5 using an in-memory key value access. + */ +public class N5InMemoryTest extends AbstractN5Test { + + private static final InMemoryKeyValueAccess memAccess = new InMemoryKeyValueAccess(); + + private static Random RANDOM = new Random(999); + + private static String tempN5PathName() { + return "/tmp/n5-test-" + RANDOM.nextLong(); + } + + @Override + protected String tempN5Location() throws URISyntaxException { + + final String basePath= new File(tempN5PathName()).toURI().normalize().getPath(); + return new URI(null, null, basePath, null).toString(); + } + + @Override protected N5Writer createN5Writer() throws IOException, URISyntaxException { + + return createN5Writer(tempN5Location(), new GsonBuilder()); + } + + @Override + protected N5Writer createN5Writer( + final String location, + final GsonBuilder gson) throws IOException, URISyntaxException { + + return new N5KeyValueWriter(memAccess, location, gson, false); + } + + @Override + protected N5Reader createN5Reader( + final String location, + final GsonBuilder gson) throws IOException, URISyntaxException { + + return new N5KeyValueReader(memAccess, location, gson, false); + } + +}