Fix running aws cli on GH Actions/Azure, get modules dynamically (#309)

* Prevent aws cli using GH Actions Azure Metadata service
* Get modules dynamically
* Fix broken tests

Author: mbarrien

.github/workflows/ci.yml

@@ -1,4 +1,19 @@
+# To avoid https://github.com/aws/aws-cli/issues/5262
+env:
+  AWS_EC2_METADATA_DISABLED: true
+
 jobs:
+    get-modules:
+        runs-on: ubuntu-20.04
+        outputs:
+            matrix: ${{steps.list_dirs.outputs.matrix}}
+        # Modified from https://stackoverflow.com/a/67180216 to filter just directory names
+        # and to filter out scripts dir and to disables bless-ca
+        steps:
+            - uses: actions/checkout@v2
+            - id: list_dirs
+              run: echo "::set-output name=matrix::$(ls -d */|sed -e 's/\///'|grep -v 'bless-ca\|scripts'|jq -cnR '[inputs | select(length>0)]')"
+
     check-mod:
         name: check-mod
         runs-on: ubuntu-latest
@@ -26,75 +41,11 @@ jobs:
     test:
         name: test ${{ matrix.module }}
         runs-on: ubuntu-latest
+        needs: get-modules
         strategy:
             fail-fast: false
             matrix:
-                module:
-                    - aws-acm-cert
-                    - aws-aurora
-                    - aws-aurora-mysql
-                    - aws-aurora-postgres
-                    - aws-cloudfront-domain-redirect
-                    - aws-cloudfront-logs-bucket
-                    - aws-cloudwatch-log-group
-                    - aws-cloudwatch-log-retention-manager
-                    - aws-default-vpc-security
-                    - aws-ecs-job
-                    - aws-ecs-job-fargate
-                    - aws-ecs-service
-                    - aws-ecs-service-fargate
-                    - aws-efs-volume
-                    - aws-iam-ecs-task-role
-                    - aws-iam-group-assume-role
-                    - aws-iam-group-console-login
-                    - aws-iam-instance-profile
-                    - aws-iam-password-policy
-                    - aws-iam-policy-cwlogs
-                    - aws-iam-role
-                    - aws-iam-role-bless
-                    - aws-iam-role-cloudfront-poweruser
-                    - aws-iam-role-crossacct
-                    - aws-iam-role-ec2-poweruser
-                    - aws-iam-role-ecs-poweruser
-                    - aws-iam-role-infraci
-                    - aws-iam-role-poweruser
-                    - aws-iam-role-readonly
-                    - aws-iam-role-route53domains-poweruser
-                    - aws-iam-role-security-audit
-                    - aws-iam-secrets-reader-policy
-                    - aws-lambda-edge-add-security-headers
-                    - aws-lambda-function
-                    - aws-param
-                    - aws-params-reader-policy
-                    - aws-params-secrets-setup
-                    - aws-params-writer
-                    - aws-redis-node
-                    - aws-redis-replication-group
-                    - aws-s3-private-bucket
-                    - aws-s3-public-bucket
-                    - aws-single-page-static-site
-                    - aws-sns-lambda
-                    - aws-ssm-params
-                    - aws-ssm-params-writer
-                    # - bless-ca/test
-                    - github-webhooks-to-s3
-                    - module-template
-                    - snowflake-account-grant-all
-                    - snowflake-database-grant-all
-                    - snowflake-external-table-grant-all
-                    - snowflake-file-format-grant-all
-                    - snowflake-function-grant-all
-                    - snowflake-integration-grant-all
-                    - snowflake-materialized-view-grant-all
-                    - snowflake-procedure-grant-all
-                    - snowflake-resource-monitor-grant-all
-                    - snowflake-schema-grant-all
-                    - snowflake-sequence-grant-all
-                    - snowflake-stage-grant-all
-                    - snowflake-stream-grant-all
-                    - snowflake-table-grant-all
-                    - snowflake-view-grant-all
-                    - snowflake-warehouse-grant-all
+                module: ${{fromJson(needs.get-modules.outputs.matrix)}}
         steps:
             - uses: actions/checkout@v2
             - id: filter

aws-default-vpc-security/module_test.go

@@ -10,10 +10,14 @@ import (
 func TestAWSDefaultVPCSecurity(t *testing.T) {
 	test := tftest.Test{
 		Setup: func(t *testing.T) *terraform.Options {
-			return tftest.Options(
-				tftest.DefaultRegion,
-				map[string]interface{}{},
-			)
+			// Not using tftest.Options because module does not take standard arguments
+			return &terraform.Options{
+				TerraformDir: ".",
+
+				EnvVars: map[string]string{
+					"AWS_DEFAULT_REGION": tftest.DefaultRegion,
+				},
+			}
 		},
 
 		Mode: tftest.Plan,

aws-iam-group-console-login/module_test.go

@@ -12,14 +12,19 @@ import (
 func TestAWSIAMGroupConsoleLogin(t *testing.T) {
 	test := tftest.Test{
 		Setup: func(t *testing.T) *terraform.Options {
-			return tftest.Options(
-				tftest.IAMRegion,
+			// Not using tftest.Options because module does not take standard arguments
+			return &terraform.Options{
+				TerraformDir: ".",
 
-				map[string]interface{}{
+				EnvVars: map[string]string{
+					"AWS_DEFAULT_REGION": tftest.IAMRegion,
+				},
+
+				Vars: map[string]interface{}{
 					"group_name": random.UniqueId(),
 					"iam_path":   fmt.Sprintf("/%s/", random.UniqueId()),
 				},
-			)
+			}
 		},
 		Validate: func(t *testing.T, options *terraform.Options) {},
 	}

aws-iam-instance-profile/module_test.go

@@ -11,14 +11,20 @@ import (
 func TestAWSIAMInstanceProfile(t *testing.T) {
 	test := tftest.Test{
 		Setup: func(t *testing.T) *terraform.Options {
-			return tftest.Options(
-				tftest.IAMRegion,
-				map[string]interface{}{
+			// Not using tftest.Options because module does not take standard arguments
+			return &terraform.Options{
+				TerraformDir: ".",
+
+				EnvVars: map[string]string{
+					"AWS_DEFAULT_REGION": tftest.IAMRegion,
+				},
+
+				Vars: map[string]interface{}{
 					"name_prefix":      random.UniqueId(),
 					"iam_path":         "/foo/",
 					"role_description": random.UniqueId(),
 				},
-			)
+			}
 		},
 		Validate: func(t *testing.T, options *terraform.Options) {},
 	}

aws-iam-policy-cwlogs/module_test.go

@@ -6,19 +6,24 @@ import (
 
 	"github.com/chanzuckerberg/go-misc/tftest"
 	"github.com/gruntwork-io/terratest/modules/random"
+	"github.com/gruntwork-io/terratest/modules/terraform"
 )
 
 func TestAWSIAMPolicyCwlogs(t *testing.T) {
 	roleName := tftest.CreateRole(t)
 	defer tftest.DeleteRole(t, roleName) //nolint
 
-	terraformOptions := tftest.Options(
-		tftest.IAMRegion,
-		map[string]interface{}{
+	terraformOptions := &terraform.Options{
+		TerraformDir: ".",
+
+		EnvVars: map[string]string{
+			"AWS_DEFAULT_REGION": tftest.IAMRegion,
+		},
+		Vars: map[string]interface{}{
 			"role_name": roleName,
 			"iam_path":  fmt.Sprintf("/%s/", random.UniqueId()),
 		},
-	)
+	}
 
 	defer tftest.Cleanup(t, terraformOptions)
 

aws-iam-secrets-reader-policy/module_test.go

@@ -2,6 +2,7 @@ package test
 
 import (
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/service/secretsmanager"
 	"github.com/chanzuckerberg/go-misc/tftest"
@@ -14,18 +15,22 @@ func TestDefaults(t *testing.T) {
 	test := tftest.Test{
 		Setup: func(t *testing.T) *terraform.Options {
 			// vars are all encoded in the test terraform files
-			opt := tftest.Options(
-				tftest.DefaultRegion,
-				map[string]interface{}{},
-			)
-			opt.TerraformDir = "./test"
-			return opt
+			return &terraform.Options{
+				TerraformDir: "./test",
+
+				EnvVars: map[string]string{
+					"AWS_DEFAULT_REGION": tftest.DefaultRegion,
+				},
+			}
 		},
 
 		Validate: func(t *testing.T, options *terraform.Options) {
 			r := require.New(t)
 			secret := terraform.Output(t, options, "secret")
 			notSecret := terraform.Output(t, options, "not_secret")
+			// Need sleep to allow IAM time to catch up and recognize that
+			// test user is allowed to assume our roles.
+			time.Sleep(10 * time.Second)
 
 			{
 				roleArn := terraform.Output(t, options, "role")

aws-iam-secrets-reader-policy/test/main.tf

@@ -10,36 +10,25 @@ resource "random_string" "not" {
 
 data "aws_caller_identity" "cur" {}
 
+data "aws_iam_policy_document" "assume" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.cur.account_id}:root"]
+    }
+  }
+}
+
 resource "aws_iam_role" "role" {
   name               = random_string.name.result
-  assume_role_policy = <<JSON
-{
-		"Version": "2012-10-17",
-		"Statement": {
-			"Effect": "Allow",
-			"Principal": {
-			"AWS": "arn:aws:iam::${data.aws_caller_identity.cur.account_id}:root"
-			},
-			"Action": "sts:AssumeRole"
-		}
-		}
-JSON
+  assume_role_policy = data.aws_iam_policy_document.assume.json
 }
 
 resource "aws_iam_role" "not" {
   name               = random_string.not.result
-  assume_role_policy = <<JSON
-{
-		"Version": "2012-10-17",
-		"Statement": {
-			"Effect": "Allow",
-			"Principal": {
-			"AWS": "arn:aws:iam::${data.aws_caller_identity.cur.account_id}:root"
-			},
-			"Action": "sts:AssumeRole"
-		}
-		}
-JSON
+  assume_role_policy = data.aws_iam_policy_document.assume.json
 }
 
 resource "aws_secretsmanager_secret" "secret" {

aws-params-reader-policy/module_test.go

@@ -19,18 +19,15 @@ func TestAWSParamsSecretReaderPolicy(t *testing.T) {
 
 	log.Debug("SETUP ROLE")
 
-	setupTerraformOptions := &terraform.Options{
-		TerraformDir: "../aws-iam-role-crossacct",
-
-		Vars: map[string]interface{}{
+	setupTerraformOptions := tftest.Options(
+		tftest.IAMRegion,
+		map[string]interface{}{
 			"role_name":         random.UniqueId(),
 			"iam_path":          fmt.Sprintf("/%s/", random.UniqueId()),
 			"source_account_id": curAcct,
 		},
-		EnvVars: map[string]string{
-			"AWS_DEFAULT_REGION": tftest.IAMRegion,
-		},
-	}
+	)
+	setupTerraformOptions.TerraformDir = "../aws-iam-role-crossacct"
 
 	defer tftest.Cleanup(t, setupTerraformOptions)
 
@@ -62,13 +59,11 @@ func TestAWSParamsSecretReaderPolicy(t *testing.T) {
 	terraformOptions := tftest.Options(
 		tftest.IAMRegion,
 		map[string]interface{}{
-			"project":                   random.UniqueId(),
-			"env":                       random.UniqueId(),
-			"service":                   random.UniqueId(),
 			"role_name":                 roleName,
 			"parameter_store_key_alias": keyAlias,
 		},
 	)
+	delete(terraformOptions.Vars, "owner")
 
 	defer tftest.Cleanup(t, terraformOptions)
 

aws-s3-account-public-access-block/module_test.go

@@ -3,17 +3,24 @@ package test
 import (
 	"testing"
 
-	"github.com/chanzuckerberg/go-misc/tftest"
 	"github.com/gruntwork-io/terratest/modules/terraform"
 	"github.com/stretchr/testify/require"
 )
 
 func TestAll(t *testing.T) {
 	r := require.New(t)
 
-	terraformOptions := tftest.Options("us-east-1", map[string]interface{}{
-		"restrict": "all",
-	})
+	terraformOptions := &terraform.Options{
+		TerraformDir: ".",
+
+		EnvVars: map[string]string{
+			"AWS_DEFAULT_REGION": "us-east-1",
+		},
+
+		Vars: map[string]interface{}{
+			"restrict": "all",
+		},
+	}
 
 	defer terraform.Destroy(t, terraformOptions)
 	terraform.InitAndApply(t, terraformOptions)
@@ -32,10 +39,17 @@ func TestAll(t *testing.T) {
 func TestNone(t *testing.T) {
 	r := require.New(t)
 
-	terraformOptions := tftest.Options("us-east-1", map[string]interface{}{
-		"restrict": "none",
-	})
+	terraformOptions := &terraform.Options{
+		TerraformDir: ".",
+
+		EnvVars: map[string]string{
+			"AWS_DEFAULT_REGION": "us-east-1",
+		},
 
+		Vars: map[string]interface{}{
+			"restrict": "none",
+		},
+	}
 	defer terraform.Destroy(t, terraformOptions)
 	terraform.InitAndApply(t, terraformOptions)
 
@@ -51,9 +65,17 @@ func TestNone(t *testing.T) {
 func TestNew(t *testing.T) {
 	r := require.New(t)
 
-	terraformOptions := tftest.Options("us-east-1", map[string]interface{}{
-		"restrict": "new",
-	})
+	terraformOptions := &terraform.Options{
+		TerraformDir: ".",
+
+		EnvVars: map[string]string{
+			"AWS_DEFAULT_REGION": "us-east-1",
+		},
+
+		Vars: map[string]interface{}{
+			"restrict": "new",
+		},
+	}
 
 	defer terraform.Destroy(t, terraformOptions)
 	terraform.InitAndApply(t, terraformOptions)

aws-single-page-static-site/module_test.go

@@ -11,7 +11,6 @@ func TestAwsSinglePageStaticSite(t *testing.T) {
 	t.Parallel()
 
 	test := tftest.Test{
-		SkipDestroy: true,
 		Setup: func(t *testing.T) *terraform.Options {
 			subdomain := tftest.UniqueID()
 			route53ZoneID := tftest.EnvVar(tftest.EnvRoute53ZoneID)