From fbb77029745797f70a1f77e4a1cb1d7e0c0e3ea4 Mon Sep 17 00:00:00 2001 From: Dominik Stadler Date: Tue, 27 Jan 2015 10:48:04 +0100 Subject: [PATCH 1/3] Improve/Add some error messages that we found useful when starting to use Ice, one null-check is too late in Config, and check for file.length()>0 as we had cases where zero-length files caused the product to fail --- .../com/netflix/ice/basic/BasicLineItemProcessor.java | 6 ++++-- .../netflix/ice/basic/BasicReservationService.java | 11 +++++++++-- src/java/com/netflix/ice/common/Config.java | 2 +- src/java/com/netflix/ice/processor/DataWriter.java | 5 ++++- .../com/netflix/ice/processor/TagGroupWriter.java | 9 +++++++-- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java index f16a88dd..b4a395c6 100644 --- a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java +++ b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java @@ -97,8 +97,10 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c return Result.ignore; Account account = config.accountService.getAccountById(items[accountIdIndex]); - if (account == null) - return Result.ignore; + if (account == null) { + logger.warn("Could not find account: " + items[accountIdIndex] + " in AccountService"); + return Result.ignore; + } double usageValue = Double.parseDouble(items[usageQuantityIndex]); double costValue = Double.parseDouble(items[costIndex]); diff --git a/src/java/com/netflix/ice/basic/BasicReservationService.java b/src/java/com/netflix/ice/basic/BasicReservationService.java index 0525fbe0..37f0a7d2 100644 --- a/src/java/com/netflix/ice/basic/BasicReservationService.java +++ b/src/java/com/netflix/ice/basic/BasicReservationService.java @@ -106,8 +106,7 @@ public void init() { pollAPI(); } catch (Exception e) { - logger.error("failed to poll reservation prices", e); - throw new RuntimeException("failed to poll reservation prices for " + e.getMessage()); + throw new RuntimeException("failed to poll reservation prices", e); } } else { @@ -318,6 +317,10 @@ public double getLatestHourlyTotalPrice( Ec2InstanceReservationPrice ec2Price = ec2InstanceReservationPrices.get(utilization).get(new Ec2InstanceReservationPrice.Key(region, usageType)); + if(ec2Price == null) { + throw new IllegalStateException("Did not find an EC2 price for region " + region + ", type " + usageType + " and utilization " + utilization); + } + double tier = getEc2Tier(time); return ec2Price.hourlyPrice.getPrice(null).getPrice(tier) + ec2Price.upfrontPrice.getPrice(null).getUpfrontAmortized(time, term, tier); @@ -367,6 +370,10 @@ public ReservationInfo getReservation( houlyCost = houlyCost / count; } + if(houlyCost == 0) { + throw new IllegalStateException("Did not find an EC2 price for region " + tagGroup.region + ", type " + tagGroup.usageType + " and utilization " + utilization); + } + return new ReservationInfo(count, upfrontAmortized, houlyCost); } diff --git a/src/java/com/netflix/ice/common/Config.java b/src/java/com/netflix/ice/common/Config.java index 73992e03..1493ccee 100644 --- a/src/java/com/netflix/ice/common/Config.java +++ b/src/java/com/netflix/ice/common/Config.java @@ -48,8 +48,8 @@ public Config( AccountService accountService, ProductService productService, ResourceService resourceService) { + if (properties == null) throw new IllegalArgumentException("properties must be specified"); if (properties.getProperty(IceOptions.START_MILLIS) == null) throw new IllegalArgumentException("IceOptions.START_MILLIS must be specified"); - if (properties == null) throw new IllegalArgumentException("properties must be specified"); if (credentialsProvider == null) throw new IllegalArgumentException("credentialsProvider must be specified"); if (accountService == null) throw new IllegalArgumentException("accountService must be specified"); if (productService == null) throw new IllegalArgumentException("productService must be specified"); diff --git a/src/java/com/netflix/ice/processor/DataWriter.java b/src/java/com/netflix/ice/processor/DataWriter.java index 875843d8..a0586899 100644 --- a/src/java/com/netflix/ice/processor/DataWriter.java +++ b/src/java/com/netflix/ice/processor/DataWriter.java @@ -44,11 +44,14 @@ public class DataWriter { AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, file); } - if (file.exists()) { + if (file.exists() && file.length() > 0) { DataInputStream in = new DataInputStream(new FileInputStream(file)); try { data = ReadWriteData.Serializer.deserialize(in); } + catch (EOFException e) { + throw new IllegalStateException("While handling file: " + file, e); + } finally { in.close(); } diff --git a/src/java/com/netflix/ice/processor/TagGroupWriter.java b/src/java/com/netflix/ice/processor/TagGroupWriter.java index 35d8a6ef..63d17068 100644 --- a/src/java/com/netflix/ice/processor/TagGroupWriter.java +++ b/src/java/com/netflix/ice/processor/TagGroupWriter.java @@ -46,7 +46,12 @@ public class TagGroupWriter { if (file.exists()) { DataInputStream in = new DataInputStream(new FileInputStream(file)); try { - tagGroups = TagGroup.Serializer.deserializeTagGroups(config, in); + try { + tagGroups = TagGroup.Serializer.deserializeTagGroups(config, in); + } catch (EOFException e) { + // how do we handle a corrupted file! + throw new IllegalStateException("While handling file " + file, e); + } } finally { if (in != null) @@ -69,7 +74,7 @@ void archive(Long monthMilli,Collection tagGroups) throws IOException out.close(); } - logger.info(dbName + " uploading to s3..."); + logger.info(dbName + " uploading to s3 bucket " + config.workS3BucketName + " with prefix '" + config.workS3BucketPrefix + "'..."); AwsUtils.upload(config.workS3BucketName, config.workS3BucketPrefix, config.localDir, dbName); logger.info(dbName + " uploading done."); } From 2b320837b34f8116c5e66f160852f25ddc0b1ca6 Mon Sep 17 00:00:00 2001 From: Dominik Stadler Date: Tue, 27 Jan 2015 10:49:30 +0100 Subject: [PATCH 2/3] Add .gitignore with a useful ignores for grails applications --- .gitignore | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e2b395da --- /dev/null +++ b/.gitignore @@ -0,0 +1,51 @@ +# .gitignore for Grails 1.2 and 1.3 + +# web application files +/web-app/WEB-INF + +# IDE support files +/.launch +/.settings +/*.launch +/*.tmproj +/ivy* +/eclipse +/bin + +# default HSQL database files for production mode +/prodDb.* + +# general HSQL database files +*Db.properties +*Db.script + +# logs +/stacktrace.log +/test/reports +/logs + +# project release file +/*.war + +# plugin release file +/*.zip + +# older plugin install locations +/plugins +/web-app/plugins +/web-app/WEB-INF/classes + +# "temporary" build files +/target + +# other +*.iws + +# local data +/ice_processor +/ice_reader +/work +/test + +# sensitive data not checked in +account.properties From 2e93a4d3900a0e8ff2e48be1bf6bc5387a1c845e Mon Sep 17 00:00:00 2001 From: Dominik Stadler Date: Tue, 27 Jan 2015 13:26:40 +0100 Subject: [PATCH 3/3] Use BufferedReader to speed up reading the file-data from disk --- src/java/com/netflix/ice/processor/BillingFileProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/com/netflix/ice/processor/BillingFileProcessor.java b/src/java/com/netflix/ice/processor/BillingFileProcessor.java index e5c18ce2..e92f7243 100644 --- a/src/java/com/netflix/ice/processor/BillingFileProcessor.java +++ b/src/java/com/netflix/ice/processor/BillingFileProcessor.java @@ -560,7 +560,7 @@ private void processBillingZipFile(File file, boolean withTags) throws IOExcepti private void processBillingFile(String fileName, InputStream tempIn, boolean withTags) { - CsvReader reader = new CsvReader(new InputStreamReader(tempIn), ','); + CsvReader reader = new CsvReader(new BufferedReader(new InputStreamReader(tempIn), 1024*1024), ','); long lineNumber = 0; List delayedItems = Lists.newArrayList();