Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .git-blame-ignore-revs
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Format source code according to checkstyle configuration
a5da2cad2e1f73c18df90bbfbe5d06f2bbd3357e
5 changes: 5 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ plugins {
id 'java-library'
id 'eclipse'
id 'maven-publish'
id 'checkstyle'
}

group = 'com.glencoesoftware.omero'
Expand Down Expand Up @@ -87,3 +88,7 @@ jar {
)
}
}

checkstyle {
toolVersion = "10.26.1"
}
470 changes: 470 additions & 0 deletions config/checkstyle/checkstyle.xml

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,8 @@
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
package com.glencoesoftware.omero.zarr;

import java.net.URI;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

import org.slf4j.LoggerFactory;
package com.glencoesoftware.omero.zarr;

import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSCredentialsProviderChain;
Expand All @@ -36,7 +30,15 @@
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.util.AwsHostNameUtils;
import com.upplication.s3fs.AmazonS3ClientFactory;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import org.slf4j.LoggerFactory;

/**
* Subclass which maps an URI into a set of credentials to use for the client.
*/
public class OmeroAmazonS3ClientFactory extends AmazonS3ClientFactory {

private static final org.slf4j.Logger log =
Expand All @@ -48,11 +50,11 @@ public class OmeroAmazonS3ClientFactory extends AmazonS3ClientFactory {
protected AWSCredentialsProvider getCredentialsProvider(Properties props) {
// If AWS Environment or System Properties are set, throw an exception
// so users will know they are not supported
if (System.getenv("AWS_ACCESS_KEY_ID") != null ||
System.getenv("AWS_SECRET_ACCESS_KEY") != null ||
System.getenv("AWS_SESSION_TOKEN") != null ||
System.getProperty("aws.accessKeyId") != null ||
System.getProperty("aws.secretAccessKey") != null) {
if (System.getenv("AWS_ACCESS_KEY_ID") != null
|| System.getenv("AWS_SECRET_ACCESS_KEY") != null
|| System.getenv("AWS_SESSION_TOKEN") != null
|| System.getProperty("aws.accessKeyId") != null
|| System.getProperty("aws.secretAccessKey") != null) {
throw new RuntimeException("AWS credentials supplied by environment variables"
+ " or Java system properties are not supported."
+ " Please use either named profiles or instance"
Expand All @@ -77,6 +79,7 @@ protected AWSCredentialsProvider getCredentialsProvider(Properties props) {

/**
* Retrieves the bucket name from a given URI.
*
* @param uri The URI to handle
* @return The bucket name
*/
Expand All @@ -90,6 +93,7 @@ private String getBucketFromUri(URI uri) {

/**
* Retrieves the region from a given URI.
*
* @param uri The URI to handle
* @return The region
*/
Expand All @@ -103,6 +107,7 @@ private String getRegionFromUri(URI uri) {

/**
* Retrieves the endpoint from a given URI.
*
* @param uri The URI to handle
* @return The endpoint
*/
Expand All @@ -112,19 +117,20 @@ public String getEndPointFromUri(URI uri) {

@Override
public synchronized AmazonS3 getAmazonS3(URI uri, Properties props) {
//Check if we have a S3 client for this bucket
// Check if we have a S3 client for this bucket
String bucket = getBucketFromUri(uri);
if (bucketClientMap.containsKey(bucket)) {
log.info("Found bucket " + bucket);
return bucketClientMap.get(bucket);
}
log.info("Creating client for bucket " + bucket);
AmazonS3 client = AmazonS3ClientBuilder.standard()
.withCredentials(getCredentialsProvider(props))
.withClientConfiguration(getClientConfiguration(props))
.withMetricsCollector(getRequestMetricsCollector(props))
.withEndpointConfiguration(new EndpointConfiguration(getEndPointFromUri(uri), getRegionFromUri(uri)))
.build();
.withCredentials(getCredentialsProvider(props))
.withClientConfiguration(getClientConfiguration(props))
.withMetricsCollector(getRequestMetricsCollector(props))
.withEndpointConfiguration(
new EndpointConfiguration(getEndPointFromUri(uri), getRegionFromUri(uri)))
.build();
bucketClientMap.put(bucket, client);
return client;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,28 +15,30 @@
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
package com.glencoesoftware.omero.zarr;

import java.io.IOException;
package com.glencoesoftware.omero.zarr;

import com.amazonaws.services.s3.AmazonS3;
import com.upplication.s3fs.S3FileSystem;
import com.upplication.s3fs.S3FileSystemProvider;
import java.io.IOException;

/** Subclass of S3FileSystem with performance optimizations. */
public class OmeroS3FileSystem extends S3FileSystem {

/** Default constructor. */
public OmeroS3FileSystem(S3FileSystemProvider provider, String key,
AmazonS3 client, String endpoint) {
super(provider, key, client, endpoint);
}

@Override
public void close() throws IOException {
//No-op
// No-op
}

@Override
public boolean isOpen() {
return true; //Not possible to be closed
return true; // Not possible to be closed
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,14 @@
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
package com.glencoesoftware.omero.zarr;

package com.glencoesoftware.omero.zarr;

import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.upplication.s3fs.S3Path;
import com.upplication.s3fs.S3ReadOnlySeekableByteChannel;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
Expand All @@ -32,14 +37,8 @@
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

import org.perf4j.slf4j.Slf4JStopWatch;

import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.upplication.s3fs.S3Path;
import com.upplication.s3fs.S3ReadOnlySeekableByteChannel;

/**
* Overridden, hybrid version of the implementation from
Expand All @@ -61,15 +60,15 @@ public class OmeroS3ReadOnlySeekableByteChannel implements SeekableByteChannel {
* entire object in full from S3 without checks for length during
* object construction.
*/
public OmeroS3ReadOnlySeekableByteChannel(S3Path path, Set<? extends OpenOption> options) throws IOException {
public OmeroS3ReadOnlySeekableByteChannel(S3Path path, Set<? extends OpenOption> options)
throws IOException {
this.options = Collections.unmodifiableSet(new HashSet<>(options));

if (
this.options.contains(StandardOpenOption.WRITE) ||
this.options.contains(StandardOpenOption.CREATE) ||
this.options.contains(StandardOpenOption.CREATE_NEW) ||
this.options.contains(StandardOpenOption.APPEND)
) {
if (this.options.contains(StandardOpenOption.WRITE)
|| this.options.contains(StandardOpenOption.CREATE)
|| this.options.contains(StandardOpenOption.CREATE_NEW)
|| this.options.contains(StandardOpenOption.APPEND)
) {
throw new ReadOnlyFileSystemException();
}

Expand All @@ -92,10 +91,10 @@ public OmeroS3ReadOnlySeekableByteChannel(S3Path path, Set<? extends OpenOption>
// the stream closed as quickly as possible. See
// https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/S3Object.html#getObjectContent--
try (S3ObjectInputStream s3Stream = s3Object.getObjectContent()) {
byte[] read_buf = new byte[1024*1024];
int read_len = 0;
while ((read_len = s3Stream.read(read_buf)) > 0) {
outputStream.write(read_buf, 0, read_len);
byte[] readBuf = new byte[1024 * 1024];
int readLen = 0;
while ((readLen = s3Stream.read(readBuf)) > 0) {
outputStream.write(readBuf, 0, readLen);
}
}
this.data = outputStream.toByteArray();
Expand Down Expand Up @@ -124,7 +123,9 @@ public boolean isOpen() {
* private visibility.
*/
@Override
public long position() { return position; }
public long position() {
return position;
}

/**
* Overridden, hybrid version of the implementation from
Expand All @@ -133,8 +134,7 @@ public boolean isOpen() {
*/
@Override
public SeekableByteChannel position(long targetPosition)
throws IOException
{
throws IOException {
throw new UnsupportedOperationException();
}

Expand Down
Loading