From fd769af4aeb96541ee80354cec2763cb8fead86c Mon Sep 17 00:00:00 2001 From: Anton Wahyu Date: Sat, 5 Aug 2017 18:03:44 +0700 Subject: [PATCH 1/2] Add region poperty for working bucket --- README.md | 3 ++- grails-app/conf/BootStrap.groovy | 1 + src/java/com/netflix/ice/common/AwsUtils.java | 21 +++++++++++-------- src/java/com/netflix/ice/common/Config.java | 5 ++++- .../com/netflix/ice/common/IceOptions.java | 6 ++++++ src/java/sample.properties | 2 ++ 6 files changed, 27 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index eeb16d03..16fe438d 100644 --- a/README.md +++ b/README.md @@ -55,9 +55,10 @@ Using basic setup, you don't need any extra code change and you will use the pro ice.processor.localDir=/mnt/ice_processor - 1.3 In ice.properties, set up the working s3 bucket and working s3 bucket file prefix to upload the processed output files which will be read by reader. Ice must have read and write access to the s3 bucket. For example: + 1.3 In ice.properties, set up the working s3 bucket, working s3 region, and working s3 bucket file prefix to upload the processed output files which will be read by reader. Ice must have read and write access to the s3 bucket. For example: ice.work_s3bucketname=work_s3bucketname + ice.work_s3bucketregion=eu-west-1 ice.work_s3bucketprefix=work_s3bucketprefix/ 1.4 If running locally, set the following system properties at runtime. ice.s3AccessToken is optional. diff --git a/grails-app/conf/BootStrap.groovy b/grails-app/conf/BootStrap.groovy index 19d1986f..7c0bd58d 100644 --- a/grails-app/conf/BootStrap.groovy +++ b/grails-app/conf/BootStrap.groovy @@ -138,6 +138,7 @@ class BootStrap { else properties.setProperty(IceOptions.START_MILLIS, "" + new DateTime(DateTimeZone.UTC).withMillisOfDay(0).withDayOfMonth(1).getMillis()); properties.setProperty(IceOptions.WORK_S3_BUCKET_NAME, prop.getProperty(IceOptions.WORK_S3_BUCKET_NAME)); + properties.setProperty(IceOptions.WORK_S3_BUCKET_REGION, prop.getProperty(IceOptions.WORK_S3_BUCKET_REGION)); properties.setProperty(IceOptions.WORK_S3_BUCKET_PREFIX, prop.getProperty(IceOptions.WORK_S3_BUCKET_PREFIX)); properties.setProperty(IceOptions.CUSTOM_TAGS, prop.getProperty(IceOptions.CUSTOM_TAGS, "")); diff --git a/src/java/com/netflix/ice/common/AwsUtils.java b/src/java/com/netflix/ice/common/AwsUtils.java index f179a520..6e0c6d71 100755 --- a/src/java/com/netflix/ice/common/AwsUtils.java +++ b/src/java/com/netflix/ice/common/AwsUtils.java @@ -84,7 +84,7 @@ public static Credentials getAssumedCredentials(String accountId, String assumeR * This method must be called before all methods can be used. * @param credentialsProvider */ - public static void init(AWSCredentialsProvider credentialsProvider) { + public static void init(AWSCredentialsProvider credentialsProvider, String workS3BucketRegion) { awsCredentialsProvider = credentialsProvider; clientConfig = new ClientConfiguration(); String proxyHost = System.getProperty("https.proxyHost"); @@ -95,13 +95,11 @@ public static void init(AWSCredentialsProvider credentialsProvider) { } s3Client = new AmazonS3Client(awsCredentialsProvider, clientConfig); securityClient = new AWSSecurityTokenServiceClient(awsCredentialsProvider, clientConfig); - if (System.getProperty("EC2_REGION") != null && !"us-east-1".equals(System.getProperty("EC2_REGION"))) { - if ("global".equals(System.getProperty("EC2_REGION"))) { - s3Client.setEndpoint("s3.amazonaws.com"); - } - else { - s3Client.setEndpoint("s3-" + System.getProperty("EC2_REGION") + ".amazonaws.com"); - } + if("us-east-1".equals(workS3BucketRegion)) { + s3Client.setEndpoint("s3.amazonaws.com"); + } + else { + s3Client.setEndpoint("s3-" + workS3BucketRegion + ".amazonaws.com"); } } @@ -273,7 +271,12 @@ public static boolean downloadFileIfChangedSince(String bucketName, String bucke } if(bucketFileRegion != null && !bucketFileRegion.isEmpty()) { - s3Client.setEndpoint("s3-" + bucketFileRegion + ".amazonaws.com"); + if("us-east-1".equals(bucketFileRegion)) { + s3Client.setEndpoint("s3.amazonaws.com"); + } + else { + s3Client.setEndpoint("s3-" + bucketFileRegion + ".amazonaws.com"); + } } ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, bucketFilePrefix + file.getName()); diff --git a/src/java/com/netflix/ice/common/Config.java b/src/java/com/netflix/ice/common/Config.java index 73992e03..2d072404 100644 --- a/src/java/com/netflix/ice/common/Config.java +++ b/src/java/com/netflix/ice/common/Config.java @@ -26,6 +26,7 @@ public abstract class Config { public final String workS3BucketName; + public final String workS3BucketRegion; public final String workS3BucketPrefix; public final String localDir; public final AccountService accountService; @@ -56,10 +57,12 @@ public Config( DateTime startDate = new DateTime(Long.parseLong(properties.getProperty(IceOptions.START_MILLIS)), DateTimeZone.UTC); workS3BucketName = properties.getProperty(IceOptions.WORK_S3_BUCKET_NAME); + workS3BucketRegion = properties.getProperty(IceOptions.WORK_S3_BUCKET_REGION); workS3BucketPrefix = properties.getProperty(IceOptions.WORK_S3_BUCKET_PREFIX, "ice/"); localDir = properties.getProperty(IceOptions.LOCAL_DIR, "/mnt/ice"); if (workS3BucketName == null) throw new IllegalArgumentException("IceOptions.WORK_S3_BUCKET_NAME must be specified"); + if (workS3BucketRegion == null) throw new IllegalArgumentException("IceOptions.WORK_S3_BUCKET_REGION must be specified"); this.credentialsProvider = credentialsProvider; this.startDate = startDate; @@ -67,6 +70,6 @@ public Config( this.productService = productService; this.resourceService = resourceService; - AwsUtils.init(credentialsProvider); + AwsUtils.init(credentialsProvider, workS3BucketRegion); } } diff --git a/src/java/com/netflix/ice/common/IceOptions.java b/src/java/com/netflix/ice/common/IceOptions.java index bfb1ee87..3282d610 100644 --- a/src/java/com/netflix/ice/common/IceOptions.java +++ b/src/java/com/netflix/ice/common/IceOptions.java @@ -95,6 +95,12 @@ public class IceOptions { */ public static final String WORK_S3_BUCKET_NAME = "ice.work_s3bucketname"; + /** + * Region for output files s3 bucket. It should be specified for buckets using v4 validation. + * It must be specified in Config. + */ + public static final String WORK_S3_BUCKET_REGION = "ice.work_s3bucketregion"; + /** * Prefix of output files in output s3 bucket. It must be specified in Config. */ diff --git a/src/java/sample.properties b/src/java/sample.properties index e5b7077a..88637809 100644 --- a/src/java/sample.properties +++ b/src/java/sample.properties @@ -54,6 +54,8 @@ ice.companyName=Your Company Name # s3 bucket name where Ice can store output files. Ice must have read and write access to billing s3 bucket. ice.work_s3bucketname=work_s3bucketname +# location for the output files bucket. It should be specified for buckets using v4 validation +ice.work_s3bucketregion=eu-west-1 # prefix of Ice output files ice.work_s3bucketprefix=ice/ From c53801e91cc5adb31d38f6709fd572b0cd519f57 Mon Sep 17 00:00:00 2001 From: Anton Wahyu Date: Sat, 5 Aug 2017 20:55:31 +0700 Subject: [PATCH 2/2] make working bucket region configuration optional, take the same region than the first billing bucket as default --- README.md | 3 +-- src/java/com/netflix/ice/common/Config.java | 13 +++++++++++-- src/java/sample.properties | 4 ++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 16fe438d..eeb16d03 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,9 @@ Using basic setup, you don't need any extra code change and you will use the pro ice.processor.localDir=/mnt/ice_processor - 1.3 In ice.properties, set up the working s3 bucket, working s3 region, and working s3 bucket file prefix to upload the processed output files which will be read by reader. Ice must have read and write access to the s3 bucket. For example: + 1.3 In ice.properties, set up the working s3 bucket and working s3 bucket file prefix to upload the processed output files which will be read by reader. Ice must have read and write access to the s3 bucket. For example: ice.work_s3bucketname=work_s3bucketname - ice.work_s3bucketregion=eu-west-1 ice.work_s3bucketprefix=work_s3bucketprefix/ 1.4 If running locally, set the following system properties at runtime. ice.s3AccessToken is optional. diff --git a/src/java/com/netflix/ice/common/Config.java b/src/java/com/netflix/ice/common/Config.java index 2d072404..6d7dfbfb 100644 --- a/src/java/com/netflix/ice/common/Config.java +++ b/src/java/com/netflix/ice/common/Config.java @@ -57,12 +57,21 @@ public Config( DateTime startDate = new DateTime(Long.parseLong(properties.getProperty(IceOptions.START_MILLIS)), DateTimeZone.UTC); workS3BucketName = properties.getProperty(IceOptions.WORK_S3_BUCKET_NAME); - workS3BucketRegion = properties.getProperty(IceOptions.WORK_S3_BUCKET_REGION); + + String defaultRegion = properties.getProperty(IceOptions.BILLING_S3_BUCKET_REGION); + if (defaultRegion == null || defaultRegion.isEmpty()) throw new IllegalArgumentException("IceOptions.BILLING_S3_BUCKET_REGION must be specified"); + + String bucketRegion = properties.getProperty(IceOptions.WORK_S3_BUCKET_REGION, defaultRegion); + if (bucketRegion.isEmpty()) bucketRegion = defaultRegion; + if (bucketRegion.contains(",")) { + bucketRegion = bucketRegion.substring(0, bucketRegion.indexOf(",")); + } + workS3BucketRegion = bucketRegion; + workS3BucketPrefix = properties.getProperty(IceOptions.WORK_S3_BUCKET_PREFIX, "ice/"); localDir = properties.getProperty(IceOptions.LOCAL_DIR, "/mnt/ice"); if (workS3BucketName == null) throw new IllegalArgumentException("IceOptions.WORK_S3_BUCKET_NAME must be specified"); - if (workS3BucketRegion == null) throw new IllegalArgumentException("IceOptions.WORK_S3_BUCKET_REGION must be specified"); this.credentialsProvider = credentialsProvider; this.startDate = startDate; diff --git a/src/java/sample.properties b/src/java/sample.properties index 88637809..a9e94973 100644 --- a/src/java/sample.properties +++ b/src/java/sample.properties @@ -54,8 +54,8 @@ ice.companyName=Your Company Name # s3 bucket name where Ice can store output files. Ice must have read and write access to billing s3 bucket. ice.work_s3bucketname=work_s3bucketname -# location for the output files bucket. It should be specified for buckets using v4 validation -ice.work_s3bucketregion=eu-west-1 +# location for the output files bucket. The working bucket takes the same region than the first billing bucket if unspecified. +#ice.work_s3bucketregion=eu-east-1 # prefix of Ice output files ice.work_s3bucketprefix=ice/