Add slice and limit methods to ReadData

pom.xml

@@ -202,10 +202,6 @@
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-compress</artifactId>
 		</dependency>
-		<dependency>
-			<groupId>commons-io</groupId>
-			<artifactId>commons-io</artifactId>
-		</dependency>
 
 		<!-- JMH -->
 		<dependency>

src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java

@@ -58,9 +58,11 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Reader;
+import java.io.UncheckedIOException;
 import java.io.Writer;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.FileChannel;
 import java.nio.channels.OverlappingFileLockException;
@@ -81,6 +83,8 @@
 import java.util.Iterator;
 import java.util.stream.Stream;
 
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+
 /**
  * Filesystem {@link KeyValueAccess}.
  *
@@ -141,6 +145,10 @@ protected LockedFileChannel(final Path path, final boolean readOnly) throws IOEx
 			}
 		}
 
+		protected FileChannel getFileChannel() {
+			return channel;
+		}
+
 		@Override
 		public Reader newReader() throws IOException {
 
@@ -186,6 +194,11 @@ public FileSystemKeyValueAccess(final FileSystem fileSystem) {
 		this.fileSystem = fileSystem;
 	}
 
+	@Override
+	public ReadData createReadData(final String normalPath) {
+		return new FileLazyReadData(this, normalPath, 0, -1);
+	}
+
 	@Override
 	public LockedFileChannel lockForReading(final String normalPath) throws IOException {
 
@@ -237,6 +250,18 @@ public boolean exists(final String normalPath) {
 		return Files.exists(path);
 	}
 
+	@Override
+	public long size(final String normalPath) {
+
+		try {
+			return Files.size(fileSystem.getPath(normalPath));
+		} catch (NoSuchFileException e) {
+			throw new N5Exception.N5NoSuchKeyException("No such file", e);
+		} catch (IOException | UncheckedIOException e) {
+			throw new N5Exception.N5IOException(e);
+		}
+	}
+
 	@Override
 	public String[] listDirectories(final String normalPath) throws IOException {
 
@@ -546,4 +571,52 @@ protected static void createAndCheckIsDirectory(
 				throw x;
 		}
 	}
+
+	private static class FileLazyReadData extends KeyValueAccessLazyReadData<FileSystemKeyValueAccess> {
+
+		public FileLazyReadData(FileSystemKeyValueAccess kva, String normalKey, long offset, long length) {
+			super(kva, normalKey, offset, length);
+		}
+
+		@Override
+		void read() throws IOException {
+
+			try (FileChannel channel = kva.lockForReading(normalKey).getFileChannel()) {
+				channel.position(offset);
+				if (length > Integer.MAX_VALUE)
+					throw new IOException("Attempt to materialize too large data");
+
+				final long channelSize = channel.size();
+				if( !validBounds(channelSize, offset, length))
+					throw new IndexOutOfBoundsException();
+
+				final int sz = (int)(length < 0 ? channelSize : length);
+				final byte[] data = new byte[sz];
+				final ByteBuffer buf = ByteBuffer.wrap(data);
+				channel.read(buf);
+				materialized = ReadData.from(data);
+
+			} catch (final NoSuchFileException e) {
+				throw new N5Exception.N5NoSuchKeyException(e);
+			}
+		}
+
+		@Override
+		KeyValueAccessLazyReadData<FileSystemKeyValueAccess> lazySlice(long offset, long length) {
+			return new FileLazyReadData(kva, normalKey, offset, length);
+		}
+	}
+
+	private static boolean validBounds(long channelSize, long offset, long length) {
+
+		if (offset < 0)
+			return false;
+		else if (channelSize > 0 && offset >= channelSize) // offset == 0 and arrayLength == 0 is okay
+			return false;
+		else if (length >= 0 && offset + length > channelSize)
+			return false;
+
+		return true;
+	}
+
 }

src/main/java/org/janelia/saalfeldlab/n5/HttpKeyValueAccess.java

@@ -49,8 +49,10 @@
 package org.janelia.saalfeldlab.n5;
 
 import org.apache.commons.io.IOUtils;
+
 import org.apache.commons.lang3.function.TriFunction;
 import org.janelia.saalfeldlab.n5.http.ListResponseParser;
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -76,6 +78,13 @@
  */
 public class HttpKeyValueAccess implements KeyValueAccess {
 
+	public static final String HEAD = "HEAD";
+	public static final String GET = "GET";
+
+	public static final String RANGE = "Range";
+	public static final String ACCEPT_RANGE = "Accept-Range";
+	public static final String BYTES = "bytes";
+
 	private int readTimeoutMilliseconds;
 	private int connectionTimeoutMilliseconds;
 
@@ -146,6 +155,12 @@ public boolean exists(final String normalPath) {
 		}
 	}
 
+	@Override public long size(String normalPath) {
+
+		final HttpURLConnection head = requireValidHttpResponse(normalPath, "HEAD", "Error checking existence: " + normalPath, true);
+		return head.getContentLengthLong();
+	}
+
 	/**
 	 * Test whether the path is a directory.
 	 * <p>
@@ -161,7 +176,7 @@ public boolean exists(final String normalPath) {
 	public boolean isDirectory(final String normalPath) {
 
 		try {
-			requireValidHttpResponse(getDirectoryPath(normalPath), "HEAD", (code,  msg,http) -> {
+			requireValidHttpResponse(getDirectoryPath(normalPath), HEAD, (code,  msg,http) -> {
 				final N5Exception cause = validExistsResponse(code, "Error checking directory: " + normalPath, msg, true);
 				if (code >= 300 && code < 400) {
 					final String redirectLocation = http.getHeaderField("Location");
@@ -203,7 +218,7 @@ public boolean isFile(final String normalPath) {
 
 		/* Files must not end in `/` And Don't accept a redirect to a location ending in `/` */
 		try {
-			requireValidHttpResponse(getFilePath(normalPath), "HEAD", (code, msg, http) -> {
+			requireValidHttpResponse(getFilePath(normalPath), HEAD, (code, msg, http) -> {
 				final N5Exception cause = validExistsResponse(code, "Error accessing file: " + normalPath, msg, true);
 				if (code >= 300 && code < 400) {
 					final String redirectLocation = http.getHeaderField("Location");
@@ -234,13 +249,18 @@ private HttpURLConnection httpRequest(String normalPath, String method) throws I
 		return connection;
 	}
 
+	@Override
+	public HttpLazyReadData createReadData(final String normalPath) {
+		return new HttpLazyReadData(this, normalPath, 0, -1);
+	}
+
 	@Override
 	public LockedChannel lockForReading(final String normalPath) throws IOException {
-		//TODO Caleb: Maybe check exists lazily when attempting to read
+
 		try {
 			if (!exists(normalPath))
 				throw new N5Exception.N5NoSuchKeyException("Key does not exist: " + normalPath);
-			return new HttpObjectChannel(uri(normalPath));
+			return new HttpObjectChannel(uri(normalPath), 0, -1);
 		} catch (URISyntaxException e) {
 			throw new N5Exception("Invalid URI Syntax", e);
 		}
@@ -291,7 +311,7 @@ public String[] list(final String normalPath) throws IOException {
 
 	private String[] queryListEntries(String normalPath, ListResponseParser parser, boolean allowRedirect) {
 
-		final HttpURLConnection http = requireValidHttpResponse(normalPath, "GET", "Error listing directory at " + normalPath, allowRedirect);
+		final HttpURLConnection http = requireValidHttpResponse(normalPath, GET, "Error listing directory at " + normalPath, allowRedirect);
 		try {
 			final String listResponse = responseToString(http.getInputStream());
 			return parser.parseListResponse(listResponse);
@@ -352,17 +372,41 @@ public void delete(final String normalPath) {
 	private class HttpObjectChannel implements LockedChannel {
 
 		protected final URI uri;
+		private final long startByte;
+		private final long size;
 		private final ArrayList<Closeable> resources = new ArrayList<>();
 
-		protected HttpObjectChannel(final URI uri) {
+		protected HttpObjectChannel(final URI uri, long startByte, long size) {
 
 			this.uri = uri;
+			this.startByte = startByte;
+			this.size = size;
+		}
+
+		private boolean isPartialRead() {
+			return startByte > 0 || (size >= 0 && size != Long.MAX_VALUE);
 		}
 
 		@Override
 		public InputStream newInputStream() throws IOException {
 
-			return uri.toURL().openStream();
+			HttpURLConnection conn = (HttpURLConnection)uri.toURL().openConnection();
+			if (isPartialRead()) {
+				conn.setRequestProperty(RANGE, rangeString());
+				final String acceptRanges = conn.getHeaderField(ACCEPT_RANGE);
+				if (acceptRanges == null || !acceptRanges.equals(BYTES)) {
+					conn.disconnect();
+					conn = (HttpURLConnection)uri.toURL().openConnection();
+					return ReadData.from(conn.getInputStream()).materialize().slice(startByte, size).inputStream();
+				}
+			}
+			return conn.getInputStream();
+		}
+
+		private String rangeString() {
+
+			final String lastByte = (size > 0) ? Long.toString(startByte + size - 1) : "";
+			return String.format("%s=%d-%s", BYTES, startByte, lastByte);
 		}
 
 		@Override
@@ -399,4 +443,24 @@ public void close() throws IOException {
 		}
 	}
 
+	private class HttpLazyReadData extends KeyValueAccessLazyReadData<HttpKeyValueAccess> {
+
+		public HttpLazyReadData(HttpKeyValueAccess kva, String normalKey, long offset, long length) {
+			super(kva, normalKey, offset, length);
+		}
+
+		@Override
+		void read() throws IOException {
+			// TODO does this throw out-of-bounds when it should
+			try( final HttpObjectChannel ch = new HttpObjectChannel(kva.uri(normalKey), offset, length) ) {
+				materialized = ReadData.from(ch.newInputStream()).materialize();
+			} catch (URISyntaxException e) {}
+		}
+
+		@Override
+		KeyValueAccessLazyReadData<HttpKeyValueAccess> lazySlice(long offset, long length) {
+			return new HttpLazyReadData(kva, normalKey, offset, length);
+		}
+	}
+
 }

src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java

@@ -60,6 +60,8 @@
 import java.util.Arrays;
 import java.util.stream.Collectors;
 
+import org.janelia.saalfeldlab.n5.readdata.ReadData;
+
 /**
  * Key value read primitives used by {@link N5KeyValueReader}
  * implementations. This interface implements a subset of access primitives
@@ -230,6 +232,17 @@ default URI uri(final String uriString) throws URISyntaxException {
 	 */
 	public boolean exists(final String normalPath);
 
+	/**
+	 * Returns the size in bytes of the object at the given normalPath if it exists.
+	 *
+	 * @param normalPath
+	 *            is expected to be in normalized form, no further
+	 *            efforts are made to normalize it.
+	 * @return the size of the object in bytes.
+	 * @throws N5Exception.N5NoSuchKeyException if the given key does not exist
+	 */
+	public long size(final String normalPath) throws N5Exception.N5NoSuchKeyException;
+
 	/**
 	 * Test whether the path is a directory.
 	 *
@@ -250,6 +263,22 @@ default URI uri(final String uriString) throws URISyntaxException {
 	 */
 	public boolean isFile(String normalPath); // TODO: Looks un-used. Remove?
 
+	/**
+	 * Create a {@link ReadData} through which data at the normal key can be read.
+	 * <p>
+	 * Implementations should read lazily if possible. Consumers may call {@link ReadData#materialize()} to force
+	 * a read operation if needed.
+	 * <p>
+	 * If supported by this KeyValueAccess implementation, partial reads are possible by calling slice on the output {@link ReadData}.
+	 *
+	 * @param normalPath is expected to be in normalized form, no further efforts are made to normalize it
+	 * @return a materialized Read data
+	 * @throws IOException if an error occurs
+	 */
+	default ReadData createReadData(final String normalPath) throws IOException {
+		return ReadData.from(this, normalPath);
+	}
+
 	/**
 	 * Create a lock on a path for reading. This isn't meant to be kept
 	 * around. Create, use, [auto]close, e.g.