Skip to content

Commit 8e99e07

Browse files
cmhulbertcmhulbert
authored
Merge pull request #206 from saalfeldlab/refactor/deprecated
Refactor/deprecated
2 parents 8ac4413 + 79c6010 commit 8e99e07

12 files changed

Lines changed: 424 additions & 369 deletions

File tree

‎src/main/java/org/janelia/saalfeldlab/n5/BufferedKvaLockedChannel.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package org.janelia.saalfeldlab.n5;
2+
3+
import org.janelia.saalfeldlab.n5.readdata.ReadData;
4+
5+
import java.io.*;
6+
7+
class BufferedKvaLockedChannel implements LockedChannel {
8+
9+
private final KeyValueAccess kva;
10+
private final String key;
11+
private ByteArrayOutputStream baos = null;
12+
13+
BufferedKvaLockedChannel(final KeyValueAccess kva, final String key) {
14+
this.kva = kva;
15+
this.key = key;
16+
}
17+
18+
@Override
19+
public Reader newReader() throws N5Exception.N5IOException {
20+
21+
return new InputStreamReader(newInputStream());
22+
}
23+
24+
@Override
25+
public InputStream newInputStream() throws N5Exception.N5IOException {
26+
return kva.createReadData(key).inputStream();
27+
}
28+
29+
@Override
30+
public Writer newWriter() throws N5Exception.N5IOException {
31+
32+
return new BufferedWriter(new OutputStreamWriter(newOutputStream()));
33+
}
34+
35+
@Override
36+
public OutputStream newOutputStream() throws N5Exception.N5IOException {
37+
if (baos == null)
38+
baos = new ByteArrayOutputStream();
39+
return baos;
40+
}
41+
42+
@Override
43+
public void close() throws IOException {
44+
if (baos != null && baos.size() > 0)
45+
kva.write(key, ReadData.from(baos.toByteArray()));
46+
}
47+
}

‎src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java‎

Lines changed: 13 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,6 @@
3434
import org.janelia.saalfeldlab.n5.N5Exception.N5IOException;
3535
import org.janelia.saalfeldlab.n5.N5Exception.N5NoSuchKeyException;
3636

37-
import java.nio.ByteBuffer;
38-
import java.nio.channels.FileChannel;
3937
import java.net.URI;
4038
import java.net.URISyntaxException;
4139
import java.nio.file.attribute.FileAttribute;
@@ -45,11 +43,8 @@
4543
import java.util.stream.Stream;
4644

4745
import org.janelia.saalfeldlab.n5.readdata.ReadData;
48-
import org.janelia.saalfeldlab.n5.readdata.LazyRead;
4946
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
5047

51-
import static org.janelia.saalfeldlab.n5.FileKeyLockManager.FILE_LOCK_MANAGER;
52-
5348
/**
5449
* Filesystem {@link KeyValueAccess}.
5550
*
@@ -67,11 +62,11 @@ private static IoPolicy getIoPolicy() {
6762
return FsIoPolicy.atomicWithFallback;
6863

6964
switch (property) {
70-
case "atomic":
65+
case "strict":
7166
return new FsIoPolicy.Atomic();
7267
case "unsafe":
7368
return new FsIoPolicy.Unsafe();
74-
case "atomicFallbackUnsafe":
69+
case "permissive":
7570
default:
7671
return FsIoPolicy.atomicWithFallback;
7772
}
@@ -100,44 +95,6 @@ public void write(final String normalPath, final ReadData data) throws N5IOExcep
10095
}
10196
}
10297

103-
@Override
104-
@Deprecated
105-
public LockedFileChannel lockForReading(final String normalPath) throws N5IOException {
106-
107-
return lockForReading(Paths.get(normalPath));
108-
}
109-
110-
@Override
111-
@Deprecated
112-
public LockedChannel lockForWriting(final String normalPath) throws N5IOException {
113-
114-
return lockForWriting(Paths.get(normalPath));
115-
}
116-
117-
@Deprecated
118-
protected static LockedFileChannel lockForReading(final Path path) throws N5IOException {
119-
120-
try {
121-
return FILE_LOCK_MANAGER.lockForReading(path);
122-
} catch (final NoSuchFileException e) {
123-
throw new N5NoSuchKeyException("No such file", e);
124-
} catch (IOException | UncheckedIOException e) {
125-
throw new N5IOException("Failed to lock file for reading: " + path, e);
126-
}
127-
}
128-
129-
@Deprecated
130-
static LockedFileChannel lockForWriting(final Path path) throws N5IOException {
131-
132-
try {
133-
return FILE_LOCK_MANAGER.lockForWriting(path);
134-
} catch (final NoSuchFileException e) {
135-
throw new N5NoSuchKeyException("No such file", e);
136-
} catch (IOException | UncheckedIOException e) {
137-
throw new N5IOException("Failed to lock file for writing: " + path, e);
138-
}
139-
}
140-
14198
@Override
14299
public boolean isDirectory(final String normalPath) {
143100

@@ -165,17 +122,6 @@ public long size(final String normalPath) {
165122
return size(Paths.get(normalPath));
166123
}
167124

168-
protected static long size(final Path path) {
169-
170-
try {
171-
return Files.size(path);
172-
} catch (NoSuchFileException e) {
173-
throw new N5NoSuchKeyException("No such file", e);
174-
} catch (IOException | UncheckedIOException e) {
175-
throw new N5IOException(e);
176-
}
177-
}
178-
179125
@Override
180126
public String[] listDirectories(final String normalPath) throws N5IOException {
181127

@@ -346,6 +292,17 @@ public void delete(final String normalPath) throws N5IOException {
346292
}
347293
}
348294

295+
protected static long size(final Path path) {
296+
297+
try {
298+
return Files.size(path);
299+
} catch (NoSuchFileException e) {
300+
throw new N5NoSuchKeyException("No such file", e);
301+
} catch (IOException | UncheckedIOException e) {
302+
throw new N5IOException(e);
303+
}
304+
}
305+
349306
protected static void tryDelete(final Path path) throws IOException {
350307

351308
try {
@@ -501,88 +458,4 @@ protected static void createAndCheckIsDirectory(
501458
throw x;
502459
}
503460
}
504-
505-
static class FileLazyRead implements LazyRead {
506-
507-
private static final Closeable NO_OP = () -> { };
508-
509-
private final Path path;
510-
private Closeable lock;
511-
512-
FileLazyRead(final Path path) throws IOException {
513-
this(path, true);
514-
}
515-
516-
FileLazyRead(final Path path, final boolean requireLock ) throws IOException {
517-
this.path = path;
518-
if (requireLock)
519-
lock = FILE_LOCK_MANAGER.lockForReading(path);
520-
else
521-
lock = NO_OP;
522-
}
523-
524-
@Override
525-
public long size() throws N5IOException {
526-
527-
if (lock == null) {
528-
throw new N5IOException("FileLazyRead is already closed.");
529-
}
530-
return FileSystemKeyValueAccess.size(path);
531-
}
532-
533-
@Override
534-
public ReadData materialize(final long offset, final long length) {
535-
536-
if (lock == null) {
537-
throw new N5IOException("FileLazyRead is already closed.");
538-
}
539-
540-
try (final FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) {
541-
542-
channel.position(offset);
543-
544-
final long channelSize = channel.size();
545-
if (!validBounds(channelSize, offset, length)) {
546-
throw new IndexOutOfBoundsException();
547-
}
548-
549-
final long size = length < 0 ? (channelSize - offset) : length;
550-
if (size > Integer.MAX_VALUE) {
551-
throw new IndexOutOfBoundsException("Attempt to materialize too large data");
552-
}
553-
554-
final byte[] data = new byte[(int) size];
555-
final ByteBuffer buf = ByteBuffer.wrap(data);
556-
channel.read(buf);
557-
return ReadData.from(data);
558-
559-
} catch (final NoSuchFileException e) {
560-
throw new N5NoSuchKeyException("No such file", e);
561-
} catch (IOException | UncheckedIOException e) {
562-
throw new N5IOException(e);
563-
}
564-
}
565-
566-
@Override
567-
public void close() throws IOException {
568-
569-
if (lock != null) {
570-
lock.close();
571-
lock = null;
572-
}
573-
}
574-
}
575-
576-
private static boolean validBounds(long channelSize, long offset, long length) {
577-
578-
if (offset < 0)
579-
return false;
580-
else if (channelSize > 0 && offset >= channelSize) // offset == 0 and channelSize == 0 is okay
581-
return false;
582-
else if (length >= 0 && offset + length > channelSize)
583-
return false;
584-
585-
return true;
586-
}
587-
588461
}

‎src/main/java/org/janelia/saalfeldlab/n5/FsIoPolicy.java‎

Lines changed: 89 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
package org.janelia.saalfeldlab.n5;
22

3+
import org.janelia.saalfeldlab.n5.readdata.LazyRead;
34
import org.janelia.saalfeldlab.n5.readdata.ReadData;
45
import org.janelia.saalfeldlab.n5.readdata.VolatileReadData;
56

7+
import java.io.Closeable;
68
import java.io.IOException;
9+
import java.io.UncheckedIOException;
10+
import java.nio.ByteBuffer;
711
import java.nio.channels.FileChannel;
812
import java.nio.file.*;
913

@@ -13,6 +17,18 @@ public class FsIoPolicy {
1317

1418
static final IoPolicy atomicWithFallback = IoPolicy.withFallback(new Atomic(), new Unsafe());
1519

20+
private static boolean validBounds(long channelSize, long offset, long length) {
21+
22+
if (offset < 0)
23+
return false;
24+
else if (channelSize > 0 && offset >= channelSize) // offset == 0 and channelSize == 0 is okay
25+
return false;
26+
else if (length >= 0 && offset + length > channelSize)
27+
return false;
28+
29+
return true;
30+
}
31+
1632
public static class Unsafe implements IoPolicy {
1733
@Override
1834
public void write(String key, ReadData readData) throws IOException {
@@ -23,7 +39,7 @@ public void write(String key, ReadData readData) throws IOException {
2339
@Override
2440
public VolatileReadData read(final String key) throws IOException {
2541
final Path path = Paths.get(key);
26-
FileSystemKeyValueAccess.FileLazyRead fileLazyRead = new FileSystemKeyValueAccess.FileLazyRead(path, false);
42+
FileLazyRead fileLazyRead = new FileLazyRead(path, false);
2743
return VolatileReadData.from(fileLazyRead);
2844
}
2945

@@ -46,7 +62,7 @@ public void write(String key, ReadData readData) throws IOException {
4662
@Override
4763
public VolatileReadData read(String key) throws IOException {
4864
final Path path = Paths.get(key);
49-
FileSystemKeyValueAccess.FileLazyRead fileLazyRead = new FileSystemKeyValueAccess.FileLazyRead(path, true);
65+
FileLazyRead fileLazyRead = new FileLazyRead(path, true);
5066
return VolatileReadData.from(fileLazyRead);
5167
}
5268

@@ -58,4 +74,75 @@ public void delete(final String key) throws IOException {
5874
}
5975
}
6076
}
77+
78+
static class FileLazyRead implements LazyRead {
79+
80+
private static final Closeable NO_OP = () -> { };
81+
82+
private final Path path;
83+
private Closeable lock;
84+
85+
FileLazyRead(final Path path) throws IOException {
86+
this(path, true);
87+
}
88+
89+
FileLazyRead(final Path path, final boolean requireLock ) throws IOException {
90+
this.path = path;
91+
if (requireLock)
92+
lock = FILE_LOCK_MANAGER.lockForReading(path);
93+
else
94+
lock = NO_OP;
95+
}
96+
97+
@Override
98+
public long size() throws N5Exception.N5IOException {
99+
100+
if (lock == null) {
101+
throw new N5Exception.N5IOException("FileLazyRead is already closed.");
102+
}
103+
return FileSystemKeyValueAccess.size(path);
104+
}
105+
106+
@Override
107+
public ReadData materialize(final long offset, final long length) {
108+
109+
if (lock == null) {
110+
throw new N5Exception.N5IOException("FileLazyRead is already closed.");
111+
}
112+
113+
try (final FileChannel channel = FileChannel.open(path, StandardOpenOption.READ)) {
114+
115+
channel.position(offset);
116+
117+
final long channelSize = channel.size();
118+
if (!validBounds(channelSize, offset, length)) {
119+
throw new IndexOutOfBoundsException();
120+
}
121+
122+
final long size = length < 0 ? (channelSize - offset) : length;
123+
if (size > Integer.MAX_VALUE) {
124+
throw new IndexOutOfBoundsException("Attempt to materialize too large data");
125+
}
126+
127+
final byte[] data = new byte[(int) size];
128+
final ByteBuffer buf = ByteBuffer.wrap(data);
129+
channel.read(buf);
130+
return ReadData.from(data);
131+
132+
} catch (final NoSuchFileException e) {
133+
throw new N5Exception.N5NoSuchKeyException("No such file", e);
134+
} catch (IOException | UncheckedIOException e) {
135+
throw new N5Exception.N5IOException(e);
136+
}
137+
}
138+
139+
@Override
140+
public void close() throws IOException {
141+
142+
if (lock != null) {
143+
lock.close();
144+
lock = null;
145+
}
146+
}
147+
}
61148
}

0 commit comments

Comments
 (0)