Merge pull request #2 from gruntwork-io/improve-test-success-rate

Refine test conditions.

Author: josh-padnick

README.md

@@ -80,30 +80,7 @@ We have two types of tests:
 
 We use Go's built-in [package testing](https://golang.org/pkg/testing/) for tests.  Therefore, simply create a file ending in `_test.go` and write your test.  
 
-Here's a sample:
-
-```go
-func TestTerraformApplyMainFunction(t *testing.T) {
-	// Running Terraform tests requires various resources like EC2 Keypairs, a unique identifier, etc.
-    // This creates all necessary common resources with randomly chosen names in an randomly selected AWS region.
-	rand, err := main.CreateRandomResourceCollection()
-	defer main.DestroyRandomResourceCollection(rand)
-	if err != nil {
-		t.Errorf("Failed to create random resource collection: %s\n", err.Error())
-	}
-
-	// Populate your template's variables.
-    // Feel free to use string literals or get clever and try to compute a variable here.
-	vars := make(map[string]string)
-	vars["aws_region"] = rand.AwsRegion
-	vars["ec2_key_name"] = rand.KeyPair.Name
-	vars["ec2_instance_name"] = rand.UniqueId
-	vars["ec2_image"] = rand.AmiId
-
-	// This will run both "terraform apply" and "terraform destroy" for a full-cycle unit test.
-	main.TerraformApply("Integration Test - TestTerraformApplyMainFunction", "resources/minimal-example", vars, false)
-}
-```
+For samples, see the [terratest_test.go](_terratest_test.go) file which shows many examples of how to use terratest.
 
 ### Running unit tests
 

apply.go

@@ -10,48 +10,37 @@ import (
 
 // Apply handles all setup required for a Terraform Apply operation but does not perform a destroy operation or do any cleanup.
 // The caller of this function is expected to call Destroy to clean up the Terraform template when done.
-func Apply(ao *ApplyOptions) (string, *RandomResourceCollection, *ApplyOptions, error) {
+func Apply(ao *ApplyOptions) (string, error) {
 	logger := log.NewLogger(ao.TestName)
 	var output string
 
 	// SETUP
-
-	// Generate random values to allow tests to run in parallel
-	logger.Println("Generating random values needed for terraform apply...")
-	var rand *RandomResourceCollection
-
-	defer rand.DestroyResources()
-	rand, err := CreateRandomResourceCollection()
-	if err != nil {
-		return output, rand, ao, fmt.Errorf("Failed to create random resource collection: %s", err.Error())
-	}
-
 	// Configure terraform to use Remote State.
-	err = aws.AssertS3BucketExists(ao.TfRemoteStateS3BucketRegion, ao.TfRemoteStateS3BucketName)
+	err := aws.AssertS3BucketExists(ao.TfRemoteStateS3BucketRegion, ao.TfRemoteStateS3BucketName)
 	if err != nil {
-		return output, rand, ao, fmt.Errorf("Test failed because the S3 Bucket '%s' does not exist in the '%s' region.\n", ao.TfRemoteStateS3BucketName, ao.TfRemoteStateS3BucketRegion)
+		return output, fmt.Errorf("Test failed because the S3 Bucket '%s' does not exist in the '%s' region.\n", ao.TfRemoteStateS3BucketName, ao.TfRemoteStateS3BucketRegion)
 	}
 
-	terraform.ConfigureRemoteState(ao.TemplatePath, ao.TfRemoteStateS3BucketName, ao.generateTfStateFileName(rand), ao.TfRemoteStateS3BucketRegion, logger)
+	terraform.ConfigureRemoteState(ao.TemplatePath, ao.TfRemoteStateS3BucketName, ao.getTfStateFileName(), ao.TfRemoteStateS3BucketRegion, logger)
 
 	// TERRAFORM APPLY
 	// Download all the Terraform modules
 	logger.Println("Running terraform get...")
 	err = terraform.Get(ao.TemplatePath, logger)
 	if err != nil {
-		return output, rand, ao, fmt.Errorf("Failed to call terraform get successfully: %s\n", err.Error())
+		return output, fmt.Errorf("Failed to call terraform get successfully: %s\n", err.Error())
 	}
 
 	// Apply the Terraform template
 	logger.Println("Running terraform apply...")
 	if ao.AttemptTerraformRetry {
-		output, err = terraform.ApplyAndGetOutputWithRetry(ao.TemplatePath, ao.Vars, logger)
+		output, err = terraform.ApplyAndGetOutputWithRetry(ao.TemplatePath, ao.Vars, ao.RetryableTerraformErrors, logger)
 	} else {
 		output, err = terraform.ApplyAndGetOutput(ao.TemplatePath, ao.Vars, logger)
 	}
 	if err != nil {
-		return output, rand, ao, fmt.Errorf("Failed to terraform apply: %s\n", err.Error())
+		return output, fmt.Errorf("Failed to terraform apply: %s\n", err.Error())
 	}
 
-	return output, rand, ao, nil
+	return output, nil
 }

apply_and_destroy.go

@@ -1,57 +1,7 @@
 package terratest
 
-import (
-	"fmt"
-
-	"github.com/gruntwork-io/terratest/log"
-	"github.com/gruntwork-io/terratest/aws"
-	"github.com/gruntwork-io/terratest/terraform"
-)
-
 // ApplyAndDestroy wraps all setup and teardown required for a Terraform Apply operation. It returns the output of the terraform operations.
 func ApplyAndDestroy(ao *ApplyOptions) (string, error) {
-	logger := log.NewLogger(ao.TestName)
-	var output string
-
-	// SETUP
-	// Generate random values to allow tests to run in parallel
-	logger.Println("Generating random values needed for terraform apply...")
-	var rand *RandomResourceCollection
-
-	defer rand.DestroyResources()
-	rand, err := CreateRandomResourceCollection()
-	if err != nil {
-		return output, fmt.Errorf("Failed to create random resource collection: %s", err.Error())
-	}
-
-	// Configure terraform to use Remote State.
-	err = aws.AssertS3BucketExists(ao.TfRemoteStateS3BucketRegion, ao.TfRemoteStateS3BucketName)
-	if err != nil {
-		return output, fmt.Errorf("Test failed because the S3 Bucket '%s' does not exist in the '%s' region.\n", ao.TfRemoteStateS3BucketName, ao.TfRemoteStateS3BucketRegion)
-	}
-
-	terraform.ConfigureRemoteState(ao.TemplatePath, ao.TfRemoteStateS3BucketName, ao.generateTfStateFileName(rand), ao.TfRemoteStateS3BucketRegion, logger)
-
-	// TERRAFORM APPLY
-	// Download all the Terraform modules
-	logger.Println("Running terraform get...")
-	err = terraform.Get(ao.TemplatePath, logger)
-	if err != nil {
-		return output, fmt.Errorf("Failed to call terraform get successfully: %s\n", err.Error())
-	}
-
-	// Apply the Terraform template
-	logger.Println("Running terraform apply...")
-
-	defer destroyHelper(ao, ao.generateTfStateFileName(rand))
-	if ao.AttemptTerraformRetry {
-		output, err = terraform.ApplyAndGetOutputWithRetry(ao.TemplatePath, ao.Vars, logger)
-	} else {
-		output, err = terraform.ApplyAndGetOutput(ao.TemplatePath, ao.Vars, logger)
-	}
-	if err != nil {
-		return output, fmt.Errorf("Failed to terraform apply: %s\n", err.Error())
-	}
-
-	return output, nil
+	defer destroyHelper(ao, ao.getTfStateFileName())
+	return Apply(ao)
 }

apply_options.go

@@ -2,10 +2,12 @@ package terratest
 
 // The options to be passed into any terratest.Apply or Destroy function
 type ApplyOptions struct {
+	UniqueId		    string	      // A unique identifier for this terraform run.
 	TestName                    string            // the name of the test to run, for logging purposes.
 	TemplatePath                string            // the relative or absolute path to the terraform template to be applied.
 	Vars                        map[string]string // the vars to pass to the terraform template.
 	AttemptTerraformRetry       bool              // if true, if a known error message occurs, automatically attempt a retry.
+	RetryableTerraformErrors    map[string]string // a map of error messages which we expect on this template and which should invoke a second terraform apply, along with an additional message offering details on this error text.
 	TfRemoteStateS3BucketName   string            // S3 bucket name where terraform.tfstate files should be stored for running any terraform runs. Bucket should already exist.
 	TfRemoteStateS3BucketRegion string            // AWS Region where the TfRemoteStateS3BucketName exists.
 }
@@ -20,6 +22,6 @@ func NewApplyOptions() *ApplyOptions {
 
 // generateTfStateFileName creates a path and filename used to reference a terraform tfstate file. E.g. this is
 // useful with S3 for deciding where the tfstate file should be within a given bucket.
-func (ao *ApplyOptions) generateTfStateFileName(r *RandomResourceCollection) string {
-	return r.UniqueId + "/terraform.tfstate"
+func (ao *ApplyOptions) getTfStateFileName() string {
+	return ao.UniqueId + "/terraform.tfstate"
 }

aws/region.go

@@ -9,14 +9,14 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 )
 
-func GetForbiddenRegions() []string {
+func GetGloballyForbiddenRegions() []string {
 	return []string{
 		"us-west-2",
 	}
 }
 
-// Get a randomly chosen AWS region
-func GetRandomRegion() string {
+// Get a randomly chosen AWS region that's not in the forbiddenRegions list
+func GetRandomRegion(forbiddenRegions []string) string {
 
 	allRegions := []string{
 		"us-east-1",
@@ -40,7 +40,8 @@ func GetRandomRegion() string {
 		randomIndex = util.Random(0,len(allRegions))
 		randomIndexIsValid = true
 
-		for _, forbiddenRegion := range GetForbiddenRegions() {
+		// The ... allows append to be used to concatenate two slices
+		for _, forbiddenRegion := range append(GetGloballyForbiddenRegions(), forbiddenRegions...) {
 			if forbiddenRegion == allRegions[randomIndex] {
 				randomIndexIsValid = false
 			}

aws/region_test.go

@@ -8,8 +8,8 @@ func TestGetRandomRegionExcludesForbiddenRegions(t *testing.T) {
 	t.Parallel()
 
 	for i := 0; i < 1000; i++ {
-		randomRegion := GetRandomRegion()
-		for _, forbiddenRegion := range GetForbiddenRegions() {
+		randomRegion := GetRandomRegion(nil)
+		for _, forbiddenRegion := range GetGloballyForbiddenRegions() {
 			if forbiddenRegion == randomRegion {
 				t.Fatalf("Returned a forbidden AWS Region: %s", forbiddenRegion)
 			}

aws/s3_test.go

@@ -15,7 +15,7 @@ func TestCreateAndDestroyS3Bucket(t *testing.T) {
 	logger := log.NewLogger("TestCreateAndDestroyS3Bucket")
 
 	// SETUP
-	region := GetRandomRegion()
+	region := GetRandomRegion(nil)
 	id := util.UniqueId()
 	logger.Printf("Random values selected. Region = %s, Id = %s\n", region, id)
 
@@ -32,7 +32,7 @@ func TestAssertS3BucketExistsNoFalseNegative(t *testing.T) {
 	logger := log.NewLogger("TestAssertS3BucketExists")
 
 	// SETUP
-	region := GetRandomRegion()
+	region := GetRandomRegion(nil)
 	s3BucketName := "gruntwork-terratest-" + strings.ToLower(util.UniqueId())
 	logger.Printf("Random values selected. Region = %s, s3BucketName = %s\n", region, s3BucketName)
 
@@ -54,7 +54,7 @@ func TestAssertS3BucketExistsNoFalsePositive(t *testing.T) {
 	logger := log.NewLogger("TestAssertS3BucketExists")
 
 	// SETUP
-	region := GetRandomRegion()
+	region := GetRandomRegion(nil)
 	s3BucketName := "gruntwork-terratest-" + strings.ToLower(util.UniqueId())
 	logger.Printf("Random values selected. Region = %s, s3BucketName = %s\n", region, s3BucketName)
 

destroy.go

@@ -17,7 +17,7 @@ func Destroy(ao *ApplyOptions, rand *RandomResourceCollection) (string, error) {
 	}
 
 	logger.Println("Running terraform destroy...")
-	output, err := destroyHelper(ao, ao.generateTfStateFileName(rand))
+	output, err := destroyHelper(ao, ao.getTfStateFileName())
 	if err != nil {
 		return output, fmt.Errorf("Failed to terraform destroy: %s", err.Error())
 	}

glide.lock

@@ -33,21 +33,30 @@ func ApplyAndGetOutput(terraformPath string, vars map[string]string, logger *log
 
 // Regrettably Terraform has many bugs. Often, just re-running terraform apply will resolve the issue.
 // This function declares which Terraform error messages warrant an automatic retry and does the retry.
-func ApplyAndGetOutputWithRetry(terraformPath string, vars map[string]string, logger *log.Logger) (string, error) {
+func ApplyAndGetOutputWithRetry(terraformPath string, vars map[string]string, errors map[string]string, logger *log.Logger) (string, error) {
 	output, err := ApplyAndGetOutput(terraformPath, vars, logger)
 	if err != nil {
 		logger.Printf("Terraform apply failed with error: %s\n", err.Error())
 
-		// Check for all Terraform errors
+		// Check for terraform errors that apply to all terraform templates.
 		if strings.Contains(output, TF_ERROR_DIFFS_DIDNT_MATCH_DURING_APPLY) {
 			logger.Printf("Terraform apply failed with the error '%s'. %s\n", TF_ERROR_DIFFS_DIDNT_MATCH_DURING_APPLY, TF_ERROR_DIFFS_DIDNT_MATCH_DURING_APPLY_MSG)
-			return ApplyAndGetOutput(terraformPath, vars, logger)
-		} else if strings.Contains(output, TF_ERROR_EIP_DOES_NOT_HAVE_ATTRIBUTE_ID) {
-			logger.Printf("Terraform apply failed with the error '%s'. %s\n", TF_ERROR_EIP_DOES_NOT_HAVE_ATTRIBUTE_ID, TF_ERROR_EIP_DOES_NOT_HAVE_ATTRIBUTE_ID_MSG)
-			return ApplyAndGetOutput(terraformPath, vars, logger)
-		} else {
-			return output, err
+			retryOutput, err := ApplyAndGetOutput(terraformPath, vars, logger)
+			return output + "**TERRAFORM-RETRY**\n" + retryOutput, err
 		}
+
+		// Check for terraform errors that are specific to this template.
+		for errorText, errorTextMsg := range errors {
+			if strings.Contains(output, errorText) {
+				logger.Printf("Terraform apply failed with the error '%s' but this error was expected and warrants a terraform apply retry. Further details: %s\n", errorText, errorTextMsg)
+				retryOutput, err := ApplyAndGetOutput(terraformPath, vars, logger)
+				return output + "**TERRAFORM-RETRY**\n" + retryOutput, err
+
+			}
+		}
+
+		logger.Printf("Terraform failed with an error we didn't expect: %s", err.Error())
+		return output, err
 	}
 
 	return output, nil