From e6eb58f971a3f9d3dcc882abc0ed3190498acc89 Mon Sep 17 00:00:00 2001 From: "Lesko, Matthew (NIH/NLM/NCBI) [C]" Date: Wed, 4 Feb 2015 15:31:34 -0500 Subject: [PATCH 1/7] Change the System.getProperty calls to those reading from the configuration file ($ICE_HOME/ice.properties). In our environment, we want those credentials in the ice.properties files, plus it lets you stick everything in a .war for serving up by tomcat --- grails-app/conf/BootStrap.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/grails-app/conf/BootStrap.groovy b/grails-app/conf/BootStrap.groovy index 44e978bf..96bf1e15 100644 --- a/grails-app/conf/BootStrap.groovy +++ b/grails-app/conf/BootStrap.groovy @@ -74,24 +74,24 @@ class BootStrap { AWSCredentialsProvider credentialsProvider; - if (StringUtils.isEmpty(System.getProperty("ice.s3AccessKeyId")) || StringUtils.isEmpty(System.getProperty("ice.s3SecretKey"))) + if (StringUtils.isEmpty(prop.getProperty("ice.s3AccessKeyId")) || StringUtils.isEmpty(prop.getProperty("ice.s3SecretKey"))) credentialsProvider = new InstanceProfileCredentialsProvider(); else credentialsProvider = new AWSCredentialsProvider() { public AWSCredentials getCredentials() { - if (StringUtils.isEmpty(System.getProperty("ice.s3AccessToken"))) + if (StringUtils.isEmpty(prop.getProperty("ice.s3AccessToken"))) return new AWSCredentials() { public String getAWSAccessKeyId() { - return System.getProperty("ice.s3AccessKeyId"); + return prop.getProperty("ice.s3AccessKeyId"); } public String getAWSSecretKey() { - return System.getProperty("ice.s3SecretKey"); + return prop.getProperty("ice.s3SecretKey"); } }; else - return new BasicSessionCredentials(System.getProperty("ice.s3AccessKeyId"), System.getProperty("ice.s3SecretKey"), - System.getProperty("ice.s3AccessToken")); + return new BasicSessionCredentials(prop.getProperty("ice.s3AccessKeyId"), prop.getProperty("ice.s3SecretKey"), + prop.getProperty("ice.s3AccessToken")); } public void refresh() { From 8e066ea59b26eadd6ce3ece1db1dd7723a3d8173 Mon Sep 17 00:00:00 2001 From: "Lesko, Matthew (NIH/NLM/NCBI) [C]" Date: Thu, 12 Feb 2015 12:48:38 -0500 Subject: [PATCH 2/7] rewrite the code to allow both command-line (System.getProperties) & ICE_HOME definitions for the AWS credentials, first preference is given to those on the command line. --- grails-app/conf/BootStrap.groovy | 75 +++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 20 deletions(-) diff --git a/grails-app/conf/BootStrap.groovy b/grails-app/conf/BootStrap.groovy index 96bf1e15..11d86cc9 100644 --- a/grails-app/conf/BootStrap.groovy +++ b/grails-app/conf/BootStrap.groovy @@ -74,29 +74,64 @@ class BootStrap { AWSCredentialsProvider credentialsProvider; - if (StringUtils.isEmpty(prop.getProperty("ice.s3AccessKeyId")) || StringUtils.isEmpty(prop.getProperty("ice.s3SecretKey"))) + if (StringUtils.isEmpty(System.getProperty("ice.s3SecretKey")) || + StringUtils.isEmpty(prop.getProperty("ice.s3SecretKey")) || + StringUtils.isEmpty(System.getProperty("ice.s3AccessKeyId")) || + StringUtils.isEmpty(prop.getProperty("ice.s3SecretKey")) + ) { /* No credentials supplied? Use instance profile credentials */ credentialsProvider = new InstanceProfileCredentialsProvider(); - else + } + else { credentialsProvider = new AWSCredentialsProvider() { public AWSCredentials getCredentials() { - if (StringUtils.isEmpty(prop.getProperty("ice.s3AccessToken"))) - return new AWSCredentials() { - public String getAWSAccessKeyId() { - return prop.getProperty("ice.s3AccessKeyId"); - } - - public String getAWSSecretKey() { - return prop.getProperty("ice.s3SecretKey"); - } - }; - else - return new BasicSessionCredentials(prop.getProperty("ice.s3AccessKeyId"), prop.getProperty("ice.s3SecretKey"), - prop.getProperty("ice.s3AccessToken")); - } - - public void refresh() { - } - }; + // First - we were not given a token + if ( StringUtils.isEmpty(System.getProperty("ice.s3AccessToken")) || + StringUtils.isEmpty(prop.getProperty("ice.s3AccessToken")) + ) { + if ( + StringUtils.isNotEmpty(System.getProperty("ice.s3SecretKey")) && + StringUtils.isNotEmpty(System.getProperty("ice.s3AccessKeyId")) + ) { /* Command line (System.properties used */ + return new AWSCredentials() { + public String getAWSAccessKeyId() { + return system.getProperty("ice.s3AccessKeyId"); + } + + public String getAWSSecretKey() { + return system.getProperty("ice.s3SecretKey"); + } + }; + } else { /* System properties were empty, check ICE_HOME */ + return new AWSCredentials() { + public String getAWSAccessKeyId() { + return prop.getProperty("ice.s3AccessKeyId"); + } + + public String getAWSSecretKey() { + return prop.getProperty("ice.s3SecretKey"); + } + }; + } + } + else { /* Token _was_ given */ + if ( + StringUtils.isNotEmpty(System.getProperty("ice.s3SecretKey")) && + StringUtils.isNotEmpty(System.getProperty("ice.s3AccessKeyId")) + ) { /* Command line (System.properties used */ + return new BasicSessionCredentials(System.getProperty("ice.s3AccessKeyId"), System.getProperty("ice.s3SecretKey"), + System.getProperty("ice.s3AccessToken")); + } else { /* Check ICE_HOME */ + return new BasicSessionCredentials(prop.getProperty("ice.s3AccessKeyId"), prop.getProperty("ice.s3SecretKey"), + prop.getProperty("ice.s3AccessToken")); + } + } + + } /* end getCredentials() define */ + public void refresh() { + } + }; /* end provider credentials statement */ + } /* end way to long if statement */ + JSONConverter.register(); From bca2a5bfe77985c9701657b4dc3f033d5be14ff8 Mon Sep 17 00:00:00 2001 From: "Lesko, Matthew (NIH/NLM/NCBI) [C]" Date: Thu, 12 Feb 2015 13:30:02 -0500 Subject: [PATCH 3/7] fix typos (System, not system), and change to AND not OR when checking if properties are set or not --- grails-app/conf/BootStrap.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/grails-app/conf/BootStrap.groovy b/grails-app/conf/BootStrap.groovy index 11d86cc9..f7da69dc 100644 --- a/grails-app/conf/BootStrap.groovy +++ b/grails-app/conf/BootStrap.groovy @@ -74,9 +74,9 @@ class BootStrap { AWSCredentialsProvider credentialsProvider; - if (StringUtils.isEmpty(System.getProperty("ice.s3SecretKey")) || - StringUtils.isEmpty(prop.getProperty("ice.s3SecretKey")) || - StringUtils.isEmpty(System.getProperty("ice.s3AccessKeyId")) || + if (StringUtils.isEmpty(System.getProperty("ice.s3SecretKey")) && + StringUtils.isEmpty(prop.getProperty("ice.s3SecretKey")) && + StringUtils.isEmpty(System.getProperty("ice.s3AccessKeyId")) && StringUtils.isEmpty(prop.getProperty("ice.s3SecretKey")) ) { /* No credentials supplied? Use instance profile credentials */ credentialsProvider = new InstanceProfileCredentialsProvider(); @@ -85,7 +85,7 @@ class BootStrap { credentialsProvider = new AWSCredentialsProvider() { public AWSCredentials getCredentials() { // First - we were not given a token - if ( StringUtils.isEmpty(System.getProperty("ice.s3AccessToken")) || + if ( StringUtils.isEmpty(System.getProperty("ice.s3AccessToken")) && StringUtils.isEmpty(prop.getProperty("ice.s3AccessToken")) ) { if ( @@ -94,11 +94,11 @@ class BootStrap { ) { /* Command line (System.properties used */ return new AWSCredentials() { public String getAWSAccessKeyId() { - return system.getProperty("ice.s3AccessKeyId"); + return System.getProperty("ice.s3AccessKeyId"); } public String getAWSSecretKey() { - return system.getProperty("ice.s3SecretKey"); + return System.getProperty("ice.s3SecretKey"); } }; } else { /* System properties were empty, check ICE_HOME */ From 3589a2cdfe88ec68c38578b9d170cdbc79786753 Mon Sep 17 00:00:00 2001 From: "Lesko, Matthew (NIH/NLM/NCBI) [C]" Date: Thu, 12 Feb 2015 13:36:30 -0500 Subject: [PATCH 4/7] document AWS credentials change - ice.properties or command line works --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0238a00a..dba5265a 100644 --- a/README.md +++ b/README.md @@ -57,13 +57,13 @@ Using basic setup, you don't need any extra code change and you will use the pro ice.work_s3bucketname=work_s3bucketname ice.work_s3bucketprefix=work_s3bucketprefix/ - 1.4 If running locally, set the following system properties at runtime. ice.s3AccessToken is optional. + 1.4 If running from locally - "./grailsw runApp" or a .war in local Tomcat instance, set the following properties in ice.properties OR define them on the command line with "-Dice.s3AccessKey=accessKey", etc. Command-line definitions take precedence over the ice.properties definitions. ice.s3AccessKeyId= ice.s3SecretKey= ice.s3AccessToken= - If running on a ec2 instance and you want to use the credentials in the instance metadata, you can leave the above properties unset. + If running on a ec2 instance and you want to use the credentials in the instance metadata, you must leave the above properties unset. 1.5 In ice.properties, specify the start time in millis for the processor to start processing billing files. For example, if you want to start processing billing files from April 1, 2013. If this property is not set, Ice will set startmillis to be the beginning of current month. From 7f679c18c0673a1e98c01b75dbccf3335cebfbbb Mon Sep 17 00:00:00 2001 From: "Lesko, Matthew (NIH/NLM/NCBI) [C]" Date: Thu, 26 Feb 2015 13:00:47 -0500 Subject: [PATCH 5/7] comment out reservationService.init() - a new reservaration type has been added which causes the startup to fail with a NullPointerException. in our environment we do not need the reservation info --- src/java/com/netflix/ice/processor/ProcessorConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/com/netflix/ice/processor/ProcessorConfig.java b/src/java/com/netflix/ice/processor/ProcessorConfig.java index c009dc4e..56e2a3ae 100644 --- a/src/java/com/netflix/ice/processor/ProcessorConfig.java +++ b/src/java/com/netflix/ice/processor/ProcessorConfig.java @@ -90,7 +90,7 @@ public ProcessorConfig( ProcessorConfig.instance = this; - reservationService.init(); + /*reservationService.init();*/ if (resourceService != null) resourceService.init(); From 73b1b55abb4699a017306f4daee5a70b3357b2e0 Mon Sep 17 00:00:00 2001 From: Anthony Johnson Date: Wed, 25 Feb 2015 21:18:41 -0500 Subject: [PATCH 6/7] Add new South America AZ(sa-east-1c) and some extra debugging so that we get more than an ugly NullPointer exception --- src/java/com/netflix/ice/basic/BasicReservationService.java | 3 +++ src/java/com/netflix/ice/tag/Zone.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/java/com/netflix/ice/basic/BasicReservationService.java b/src/java/com/netflix/ice/basic/BasicReservationService.java index 0525fbe0..7a34ea4d 100644 --- a/src/java/com/netflix/ice/basic/BasicReservationService.java +++ b/src/java/com/netflix/ice/basic/BasicReservationService.java @@ -171,6 +171,9 @@ private void pollAPI() throws Exception { } } UsageType usageType = getUsageType(offer.getInstanceType(), offer.getProductDescription()); + // Unknown Zone + if (Zone.getZone(offer.getAvailabilityZone()) == null) + logger.error("No Zone for " + offer.getAvailabilityZone()); hasNewPrice = setPrice(utilization, currentTime, Zone.getZone(offer.getAvailabilityZone()).region, usageType, offer.getFixedPrice(), hourly) || hasNewPrice; diff --git a/src/java/com/netflix/ice/tag/Zone.java b/src/java/com/netflix/ice/tag/Zone.java index e6080c62..1eeb5347 100644 --- a/src/java/com/netflix/ice/tag/Zone.java +++ b/src/java/com/netflix/ice/tag/Zone.java @@ -57,6 +57,7 @@ private Zone (Region region, String name) { public static final Zone SA_EAST_1A = new Zone(Region.SA_EAST_1, "sa-east-1a"); public static final Zone SA_EAST_1B = new Zone(Region.SA_EAST_1, "sa-east-1b"); + public static final Zone SA_EAST_1C = new Zone(Region.SA_EAST_1, "sa-east-1c"); public static final Zone AP_NORTHEAST_1A = new Zone(Region.AP_NORTHEAST_1, "ap-northeast-1a"); public static final Zone AP_NORTHEAST_1B = new Zone(Region.AP_NORTHEAST_1, "ap-northeast-1b"); @@ -94,6 +95,7 @@ private Zone (Region region, String name) { zonesByName.put(SA_EAST_1A.name, SA_EAST_1A); zonesByName.put(SA_EAST_1B.name, SA_EAST_1B); + zonesByName.put(SA_EAST_1C.name, SA_EAST_1C); zonesByName.put(AP_NORTHEAST_1A.name, AP_NORTHEAST_1A); zonesByName.put(AP_NORTHEAST_1B.name, AP_NORTHEAST_1B); From 55b25392d3bc25a914c77b2f845b3dccb682b2ea Mon Sep 17 00:00:00 2001 From: "Lesko, Matthew (NIH/NLM/NCBI) [C]" Date: Thu, 26 Feb 2015 13:19:38 -0500 Subject: [PATCH 7/7] Upstream fixed Revert "comment out reservationService.init() - a new reservaration type has been added which causes the startup to fail with a NullPointerException. in our environment we do not need the reservation info" This reverts commit 11824b46941c47a77a8b45fcaba631f4a65f4f26. --- src/java/com/netflix/ice/processor/ProcessorConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/com/netflix/ice/processor/ProcessorConfig.java b/src/java/com/netflix/ice/processor/ProcessorConfig.java index 56e2a3ae..c009dc4e 100644 --- a/src/java/com/netflix/ice/processor/ProcessorConfig.java +++ b/src/java/com/netflix/ice/processor/ProcessorConfig.java @@ -90,7 +90,7 @@ public ProcessorConfig( ProcessorConfig.instance = this; - /*reservationService.init();*/ + reservationService.init(); if (resourceService != null) resourceService.init();