From 46ffb85fe79f596a62d3cf09866b2c88725a4b0e Mon Sep 17 00:00:00 2001 From: matttrach Date: Thu, 26 Jun 2025 10:32:25 -0500 Subject: [PATCH 1/6] fix: split test files and fix GitHub release test Signed-off-by: matttrach --- .github/workflows/release.yaml | 54 +++++++- modules/deploy/create.sh.tpl | 53 +++++++ modules/deploy/destroy.sh.tpl | 9 ++ modules/deploy/main.tf | 104 ++++---------- .../{downstream_test.go => basic_test.go} | 118 ---------------- test/tests/downstream/splitrole_test.go | 131 ++++++++++++++++++ test/tests/one/{one_test.go => basic_test.go} | 2 +- .../prod/{prod_test.go => basic_test.go} | 2 +- .../three/{three_test.go => basic_test.go} | 6 +- 9 files changed, 274 insertions(+), 205 deletions(-) create mode 100644 modules/deploy/create.sh.tpl create mode 100644 modules/deploy/destroy.sh.tpl rename test/tests/downstream/{downstream_test.go => basic_test.go} (53%) create mode 100644 test/tests/downstream/splitrole_test.go rename test/tests/one/{one_test.go => basic_test.go} (99%) rename test/tests/prod/{prod_test.go => basic_test.go} (99%) rename test/tests/three/{three_test.go => basic_test.go} (95%) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f0b67db..3a371b4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -157,6 +157,49 @@ jobs: run: | ./run_tests.sh -t TestProdBasic + test_TestThreeBasic: + needs: + - release + - test_TestOneBasic + if: needs.release.outputs.release_pr + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + token: ${{secrets.GITHUB_TOKEN}} + fetch-depth: 0 + - id: aws-creds + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{env.AWS_ROLE}} + role-session-name: ${{github.run_id}} + aws-region: ${{env.AWS_REGION}} + role-duration-seconds: 14400 # 4 hours + output-credentials: true + - name: install-nix + run: | + curl -L https://nixos.org/nix/install | sh + source /home/runner/.nix-profile/etc/profile.d/nix.sh + nix --version + which nix + - name: run_tests + shell: '/home/runner/.nix-profile/bin/nix develop --ignore-environment --extra-experimental-features nix-command --extra-experimental-features flakes --keep HOME --keep SSH_AUTH_SOCK --keep IDENTIFIER --keep GITHUB_TOKEN --keep GITHUB_OWNER --keep ZONE --keep AWS_ROLE --keep AWS_REGION --keep AWS_DEFAULT_REGION --keep AWS_ACCESS_KEY_ID --keep AWS_SECRET_ACCESS_KEY --keep AWS_SESSION_TOKEN --keep UPDATECLI_GPGTOKEN --keep UPDATECLI_GITHUB_TOKEN --keep UPDATECLI_GITHUB_ACTOR --keep GPG_SIGNING_KEY --keep NIX_SSL_CERT_FILE --keep NIX_ENV_LOADED --keep TERM --command bash -e {0}' + env: + AWS_ACCESS_KEY_ID: ${{ steps.aws-creds.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.aws-creds.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.aws-creds.outputs.aws-session-token }} + AWS_MAX_ATTEMPTS: 100 + AWS_RETRY_MODE: adaptive + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + GITHUB_OWNER: rancher + IDENTIFIER: ${{github.run_id}} + ZONE: ${{secrets.ZONE}} + ACME_SERVER_URL: https://acme-v02.api.letsencrypt.org/directory + RANCHER_INSECURE: false + run: | + ./run_tests.sh -t TestThreeBasic + + test_TestDownstreamBasic: needs: - release @@ -200,11 +243,12 @@ jobs: run: | ./run_tests.sh -t TestDownstreamBasic - test_TestDownstreamRoles: + test_TestDownstreamSplitrole: needs: - release - test_TestOneBasic - test_TestProdBasic + - test_TestThreeBasic - test_TestDownstreamBasic if: needs.release.outputs.release_pr runs-on: ubuntu-latest @@ -242,15 +286,16 @@ jobs: ACME_SERVER_URL: https://acme-v02.api.letsencrypt.org/directory RANCHER_INSECURE: false run: | - ./run_tests.sh -t TestDownstreamRoles + ./run_tests.sh -t TestDownstreamSplitrole test_Cleanup: needs: - release - test_TestOneBasic - test_TestProdBasic + - test_TestThreeBasic - test_TestDownstreamBasic - - test_TestDownstreamRoles + - test_TestDownstreamSplitrole if: always() && needs.release.outputs.release_pr runs-on: ubuntu-latest steps: @@ -293,8 +338,9 @@ jobs: - release - test_TestOneBasic - test_TestProdBasic + - test_TestThreeBasic - test_TestDownstreamBasic - - test_TestDownstreamRoles + - test_TestDownstreamSplitrole - test_Cleanup if: success() && needs.release.outputs.release_pr #Ensure the test jobs succeeded, and that a release PR was created. runs-on: ubuntu-latest diff --git a/modules/deploy/create.sh.tpl b/modules/deploy/create.sh.tpl new file mode 100644 index 0000000..e360831 --- /dev/null +++ b/modules/deploy/create.sh.tpl @@ -0,0 +1,53 @@ +${export_contents} +cd ${deploy_path} +export TF_DATA_DIR="${tf_data_dir}" + +${init_script} + +MAX=${attempts} +EXITCODE=1 +ATTEMPTS=0 +E=1 +E1=0 +while [ $EXITCODE -gt 0 ] && [ $ATTEMPTS -lt $MAX ]; do + A=0 + while [ $E -gt 0 ] && [ $A -lt $MAX ]; do + timeout -k 1m ${timeout} terraform apply -var-file="${deploy_path}/inputs.tfvars" -auto-approve -state="${deploy_path}/tfstate" + E=$? + if [ $E -eq 124 ]; then echo "Apply timed out after ${timeout}"; fi + A=$((A+1)) + done + # don't destroy if the last attempt fails + if [ $E -gt 0 ] && [ $ATTEMPTS != $((MAX-1)) ]; then + A1=0 + while [ $E1 -gt 0 ] && [ $A1 -lt $MAX ]; do + timeout -k 1m ${timeout} terraform destroy -var-file="${deploy_path}/inputs.tfvars" -auto-approve -state="${deploy_path}/tfstate" + E1=$? + if [ $E1 -eq 124 ]; then echo "Apply timed out after ${timeout}"; fi + A1=$((A1+1)) + done + fi + if [ $E -gt 0 ]; then + echo "apply failed..." + fi + if [ $E1 -gt 0 ]; then + echo "destroy failed..." + fi + if [ $E -gt 0 ] || [ $E1 -gt 0 ]; then + EXITCODE=1 + else + EXITCODE=0 + fi + ATTEMPTS=$((ATTEMPTS+1)) + if [ $EXITCODE -gt 0 ] && [ $ATTEMPTS -lt $MAX ]; then + echo "wait ${interval} seconds between attempts..." + sleep ${interval} + fi +done +if [ $ATTEMPTS -eq $MAX ]; then echo "max attempts reached..."; fi +if [ $EXITCODE -ne 0 ]; then echo "failure, exit code $EXITCODE..."; fi +if [ $EXITCODE -eq 0 ]; then + echo "success..."; + terraform output -json -state="${deploy_path}/tfstate" > ${deploy_path}/outputs.json +fi +exit $EXITCODE diff --git a/modules/deploy/destroy.sh.tpl b/modules/deploy/destroy.sh.tpl new file mode 100644 index 0000000..7b7f50f --- /dev/null +++ b/modules/deploy/destroy.sh.tpl @@ -0,0 +1,9 @@ +${export_contents} +cd ${deploy_path} +export TF_DATA_DIR="${tf_data_dir}" +if [ -z "${skip_destroy}" ]; then + timeout -k 1m ${timeout} terraform init -upgrade + timeout -k 1m ${timeout} terraform destroy -var-file="${deploy_path}/inputs.tfvars" -auto-approve -state="${deploy_path}/tfstate" || true +else + echo "Not destroying deployed module, it will no longer be managed here." +fi diff --git a/modules/deploy/main.tf b/modules/deploy/main.tf index fac12cc..e8e4813 100644 --- a/modules/deploy/main.tf +++ b/modules/deploy/main.tf @@ -74,18 +74,14 @@ resource "terraform_data" "destroy" { sd = local.skip_destroy } provisioner "local-exec" { - when = destroy - command = <<-EOT - ${self.triggers_replace.ec} - cd ${self.triggers_replace.dp} - export TF_DATA_DIR="${self.triggers_replace.dd}" - if [ -z "${self.triggers_replace.sd}" ]; then - timeout -k 1m ${self.triggers_replace.to} terraform init -upgrade - timeout -k 1m ${self.triggers_replace.to} terraform destroy -var-file="${self.triggers_replace.dp}/inputs.tfvars" -auto-approve -state="${self.triggers_replace.dp}/tfstate" || true - else - echo "Not destroying deployed module, it will no longer be managed here." - fi - EOT + when = destroy + command = templatefile("${path.module}/destroy.sh.tpl", { + export_hash = self.triggers_replace.env + tf_data_dir = self.triggers_replace.dd + deploy_path = self.triggers_replace.dp + skip_destroy = self.triggers_replace.sd + timeout = self.triggers_replace.to + }) } } @@ -101,61 +97,15 @@ resource "terraform_data" "create" { env = local.export_hash } provisioner "local-exec" { - command = <<-EOT - ${local.export_contents} - cd ${local.deploy_path} - export TF_DATA_DIR="${local.tf_data_dir}" - - ${local.init_script} - - MAX=${local.attempts} - EXITCODE=1 - ATTEMPTS=0 - E=1 - E1=0 - while [ $EXITCODE -gt 0 ] && [ $ATTEMPTS -lt $MAX ]; do - A=0 - while [ $E -gt 0 ] && [ $A -lt $MAX ]; do - timeout -k 1m ${local.timeout} terraform apply -var-file="${local.deploy_path}/inputs.tfvars" -auto-approve -state="${local.deploy_path}/tfstate" - E=$? - if [ $E -eq 124 ]; then echo "Apply timed out after ${local.timeout}"; fi - A=$((A+1)) - done - # don't destroy if the last attempt fails - if [ $E -gt 0 ] && [ $ATTEMPTS != $((MAX-1)) ]; then - A1=0 - while [ $E1 -gt 0 ] && [ $A1 -lt $MAX ]; do - timeout -k 1m ${local.timeout} terraform destroy -var-file="${local.deploy_path}/inputs.tfvars" -auto-approve -state="${local.deploy_path}/tfstate" - E1=$? - if [ $E1 -eq 124 ]; then echo "Apply timed out after ${local.timeout}"; fi - A1=$((A1+1)) - done - fi - if [ $E -gt 0 ]; then - echo "apply failed..." - fi - if [ $E1 -gt 0 ]; then - echo "destroy failed..." - fi - if [ $E -gt 0 ] || [ $E1 -gt 0 ]; then - EXITCODE=1 - else - EXITCODE=0 - fi - ATTEMPTS=$((ATTEMPTS+1)) - if [ $EXITCODE -gt 0 ] && [ $ATTEMPTS -lt $MAX ]; then - echo "wait ${local.interval} seconds between attempts..." - sleep ${local.interval} - fi - done - if [ $ATTEMPTS -eq $MAX ]; then echo "max attempts reached..."; fi - if [ $EXITCODE -ne 0 ]; then echo "failure, exit code $EXITCODE..."; fi - if [ $EXITCODE -eq 0 ]; then - echo "success..."; - terraform output -json -state="${local.deploy_path}/tfstate" > ${local.deploy_path}/outputs.json - fi - exit $EXITCODE - EOT + command = templatefile("${path.module}/create.sh.tpl", { + export_contents = local.export_contents + deploy_path = local.deploy_path + tf_data_dir = local.tf_data_dir + init_script = local.init_script + attempts = local.attempts + timeout = local.timeout + interval = local.interval + }) } } @@ -205,17 +155,13 @@ resource "terraform_data" "destroy_end" { sd = local.skip_destroy } provisioner "local-exec" { - when = destroy - command = <<-EOT - ${self.triggers_replace.ec} - cd ${self.triggers_replace.dp} - export TF_DATA_DIR="${self.triggers_replace.dd}" - if [ -z "${self.triggers_replace.sd}" ]; then - timeout -k 1m ${self.triggers_replace.to} terraform init -upgrade - timeout -k 1m ${self.triggers_replace.to} terraform destroy -var-file="${self.triggers_replace.dp}/inputs.tfvars" -auto-approve -state="${self.triggers_replace.dp}/tfstate" || true - else - echo "Not destroying deployed module, it will no longer be managed here." - fi - EOT + when = destroy + command = templatefile("${path.module}/destroy.sh.tpl", { + export_hash = self.triggers_replace.env + tf_data_dir = self.triggers_replace.dd + deploy_path = self.triggers_replace.dp + skip_destroy = self.triggers_replace.sd + timeout = self.triggers_replace.to + }) } } diff --git a/test/tests/downstream/downstream_test.go b/test/tests/downstream/basic_test.go similarity index 53% rename from test/tests/downstream/downstream_test.go rename to test/tests/downstream/basic_test.go index ba34a8d..f31a0c6 100644 --- a/test/tests/downstream/downstream_test.go +++ b/test/tests/downstream/basic_test.go @@ -131,121 +131,3 @@ func TestDownstreamBasic(t *testing.T) { os.Remove(exampleDir + "/.terraform.lock.hcl") sshAgent.Stop() } - -func TestDownstreamSplitrole(t *testing.T) { - t.Parallel() - id := util.GetId() - region := util.GetRegion() - accessKey := util.GetAwsAccessKey() - secretKey := util.GetAwsSecretKey() - sessionToken := util.GetAwsSessionToken() - directory := "downstream_splitrole" - owner := "terraform-ci@suse.com" - util.SetAcmeServer() - - repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) - if err != nil { - t.Fatalf("Error getting git root directory: %v", err) - } - - exampleDir := repoRoot + "/examples/" + directory - testDir := repoRoot + "/test/tests/data/" + id - - err = util.CreateTestDirectories(t, id) - if err != nil { - os.RemoveAll(testDir) - t.Fatalf("Error creating test data directories: %s", err) - } - keyPair, err := util.CreateKeypair(t, region, owner, id) - if err != nil { - os.RemoveAll(testDir) - t.Fatalf("Error creating test key pair: %s", err) - } - - err = os.WriteFile(testDir+"/id_rsa", []byte(keyPair.KeyPair.PrivateKey), 0600) - if err != nil { - os.RemoveAll(testDir) - t.Fatalf("Error creating test key pair: %s", err) - } - - sshAgent := ssh.SshAgentWithKeyPair(t, keyPair.KeyPair) - t.Logf("Key %s created and added to agent", keyPair.Name) - - // use oldest RKE2, remember it releases much more than Rancher - _, _, rke2Version, err := util.GetRke2Releases() - if err != nil { - os.RemoveAll(testDir) - aws.DeleteEC2KeyPair(t, keyPair) - sshAgent.Stop() - t.Fatalf("Error getting Rke2 release version: %s", err) - } - - rancherVersion := os.Getenv("RANCHER_VERSION") - if rancherVersion == "" { - // use stable version if not specified - // using stable prevents problems where the Rancher provider hasn't released to fit the latest Rancher - _, rancherVersion, _, err = util.GetRancherReleases() - } - if err != nil { - os.RemoveAll(testDir) - aws.DeleteEC2KeyPair(t, keyPair) - sshAgent.Stop() - t.Fatalf("Error getting Rancher release version: %s", err) - } - - terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ - TerraformDir: exampleDir, - // Variables to pass to our Terraform code using -var options - Vars: map[string]interface{}{ - "identifier": id, - "owner": owner, - "key_name": keyPair.Name, - "key": keyPair.KeyPair.PublicKey, - "zone": os.Getenv("ZONE"), - "rke2_version": rke2Version, - "rancher_version": rancherVersion, - "file_path": testDir, - "aws_access_key_id": accessKey, - "aws_secret_access_key": secretKey, - "aws_session_token": sessionToken, - "aws_region": region, - }, - // Environment variables to set when running Terraform - EnvVars: map[string]string{ - "AWS_DEFAULT_REGION": region, - "AWS_REGION": region, - "TF_DATA_DIR": testDir, - "TF_IN_AUTOMATION": "1", - "KUBECONFIG": testDir + "/kubeconfig", - "KUBE_CONFIG_PATH": testDir, - "TF_CLI_ARGS_plan": "-no-color -state=" + testDir + "/tfstate", - "TF_CLI_ARGS_apply": "-no-color -state=" + testDir + "/tfstate", - "TF_CLI_ARGS_destroy": "-no-color -state=" + testDir + "/tfstate", - "TF_CLI_ARGS_output": "-no-color -state=" + testDir + "/tfstate", - }, - RetryableTerraformErrors: util.GetRetryableTerraformErrors(), - NoColor: true, - SshAgent: sshAgent, - Upgrade: true, - }) - - _, err = terraform.InitAndApplyE(t, terraformOptions) - if err != nil { - t.Log("Test failed, tearing down...") - util.GetErrorLogs(t, testDir+"/kubeconfig") - util.Teardown(t, testDir, terraformOptions, keyPair) - os.Remove(exampleDir + ".terraform.lock.hcl") - sshAgent.Stop() - t.Fatalf("Error creating cluster: %s", err) - } - util.CheckReady(t, testDir+"/kubeconfig") - util.CheckRunning(t, testDir+"/kubeconfig") - if t.Failed() { - t.Log("Test failed...") - } else { - t.Log("Test passed...") - } - util.Teardown(t, testDir, terraformOptions, keyPair) - os.Remove(exampleDir + "/.terraform.lock.hcl") - sshAgent.Stop() -} diff --git a/test/tests/downstream/splitrole_test.go b/test/tests/downstream/splitrole_test.go new file mode 100644 index 0000000..0d7763a --- /dev/null +++ b/test/tests/downstream/splitrole_test.go @@ -0,0 +1,131 @@ +package downstream + +import ( + "os" + "path/filepath" + "testing" + + aws "github.com/gruntwork-io/terratest/modules/aws" + g "github.com/gruntwork-io/terratest/modules/git" + "github.com/gruntwork-io/terratest/modules/ssh" + "github.com/gruntwork-io/terratest/modules/terraform" + util "github.com/rancher/terraform-rancher2-aws/test/tests" +) + +func TestDownstreamSplitrole(t *testing.T) { + t.Parallel() + id := util.GetId() + region := util.GetRegion() + accessKey := util.GetAwsAccessKey() + secretKey := util.GetAwsSecretKey() + sessionToken := util.GetAwsSessionToken() + directory := "downstream_splitrole" + owner := "terraform-ci@suse.com" + util.SetAcmeServer() + + repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) + if err != nil { + t.Fatalf("Error getting git root directory: %v", err) + } + + exampleDir := repoRoot + "/examples/" + directory + testDir := repoRoot + "/test/tests/data/" + id + + err = util.CreateTestDirectories(t, id) + if err != nil { + os.RemoveAll(testDir) + t.Fatalf("Error creating test data directories: %s", err) + } + keyPair, err := util.CreateKeypair(t, region, owner, id) + if err != nil { + os.RemoveAll(testDir) + t.Fatalf("Error creating test key pair: %s", err) + } + + err = os.WriteFile(testDir+"/id_rsa", []byte(keyPair.KeyPair.PrivateKey), 0600) + if err != nil { + os.RemoveAll(testDir) + t.Fatalf("Error creating test key pair: %s", err) + } + + sshAgent := ssh.SshAgentWithKeyPair(t, keyPair.KeyPair) + t.Logf("Key %s created and added to agent", keyPair.Name) + + // use oldest RKE2, remember it releases much more than Rancher + _, _, rke2Version, err := util.GetRke2Releases() + if err != nil { + os.RemoveAll(testDir) + aws.DeleteEC2KeyPair(t, keyPair) + sshAgent.Stop() + t.Fatalf("Error getting Rke2 release version: %s", err) + } + + rancherVersion := os.Getenv("RANCHER_VERSION") + if rancherVersion == "" { + // use stable version if not specified + // using stable prevents problems where the Rancher provider hasn't released to fit the latest Rancher + _, rancherVersion, _, err = util.GetRancherReleases() + } + if err != nil { + os.RemoveAll(testDir) + aws.DeleteEC2KeyPair(t, keyPair) + sshAgent.Stop() + t.Fatalf("Error getting Rancher release version: %s", err) + } + + terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ + TerraformDir: exampleDir, + // Variables to pass to our Terraform code using -var options + Vars: map[string]interface{}{ + "identifier": id, + "owner": owner, + "key_name": keyPair.Name, + "key": keyPair.KeyPair.PublicKey, + "zone": os.Getenv("ZONE"), + "rke2_version": rke2Version, + "rancher_version": rancherVersion, + "file_path": testDir, + "aws_access_key_id": accessKey, + "aws_secret_access_key": secretKey, + "aws_session_token": sessionToken, + "aws_region": region, + }, + // Environment variables to set when running Terraform + EnvVars: map[string]string{ + "AWS_DEFAULT_REGION": region, + "AWS_REGION": region, + "TF_DATA_DIR": testDir, + "TF_IN_AUTOMATION": "1", + "KUBECONFIG": testDir + "/kubeconfig", + "KUBE_CONFIG_PATH": testDir, + "TF_CLI_ARGS_plan": "-no-color -state=" + testDir + "/tfstate", + "TF_CLI_ARGS_apply": "-no-color -state=" + testDir + "/tfstate", + "TF_CLI_ARGS_destroy": "-no-color -state=" + testDir + "/tfstate", + "TF_CLI_ARGS_output": "-no-color -state=" + testDir + "/tfstate", + }, + RetryableTerraformErrors: util.GetRetryableTerraformErrors(), + NoColor: true, + SshAgent: sshAgent, + Upgrade: true, + }) + + _, err = terraform.InitAndApplyE(t, terraformOptions) + if err != nil { + t.Log("Test failed, tearing down...") + util.GetErrorLogs(t, testDir+"/kubeconfig") + util.Teardown(t, testDir, terraformOptions, keyPair) + os.Remove(exampleDir + ".terraform.lock.hcl") + sshAgent.Stop() + t.Fatalf("Error creating cluster: %s", err) + } + util.CheckReady(t, testDir+"/kubeconfig") + util.CheckRunning(t, testDir+"/kubeconfig") + if t.Failed() { + t.Log("Test failed...") + } else { + t.Log("Test passed...") + } + util.Teardown(t, testDir, terraformOptions, keyPair) + os.Remove(exampleDir + "/.terraform.lock.hcl") + sshAgent.Stop() +} diff --git a/test/tests/one/one_test.go b/test/tests/one/basic_test.go similarity index 99% rename from test/tests/one/one_test.go rename to test/tests/one/basic_test.go index e830db7..e3e8ce3 100644 --- a/test/tests/one/one_test.go +++ b/test/tests/one/basic_test.go @@ -12,7 +12,7 @@ import ( util "github.com/rancher/terraform-rancher2-aws/test/tests" ) -func TestOne(t *testing.T) { +func TestOneBasic(t *testing.T) { t.Parallel() id := util.GetId() region := util.GetRegion() diff --git a/test/tests/prod/prod_test.go b/test/tests/prod/basic_test.go similarity index 99% rename from test/tests/prod/prod_test.go rename to test/tests/prod/basic_test.go index c86f113..8e96406 100644 --- a/test/tests/prod/prod_test.go +++ b/test/tests/prod/basic_test.go @@ -12,7 +12,7 @@ import ( util "github.com/rancher/terraform-rancher2-aws/test/tests" ) -func TestProd(t *testing.T) { +func TestProdBasic(t *testing.T) { t.Parallel() id := util.GetId() region := util.GetRegion() diff --git a/test/tests/three/three_test.go b/test/tests/three/basic_test.go similarity index 95% rename from test/tests/three/three_test.go rename to test/tests/three/basic_test.go index 0451cb0..788816d 100644 --- a/test/tests/three/three_test.go +++ b/test/tests/three/basic_test.go @@ -12,7 +12,7 @@ import ( util "github.com/rancher/terraform-rancher2-aws/test/tests" ) -func TestThree(t *testing.T) { +func TestThreeBasic(t *testing.T) { t.Parallel() id := util.GetId() region := util.GetRegion() @@ -44,9 +44,11 @@ func TestThree(t *testing.T) { os.RemoveAll(testDir) t.Fatalf("Error creating test key pair: %s", err) } - sshAgent := ssh.SshAgentWithKeyPair(t, keyPair.KeyPair) + sshAgent := ssh.SshAgentWithKeyPair(t, keyPair.KeyPair) t.Logf("Key %s created and added to agent", keyPair.Name) + // here we need to create an object storage to hold the remote state + // use oldest RKE2, remember it releases much more than Rancher _, _, rke2Version, err := util.GetRke2Releases() if err != nil { From e7a8693a47605c38f5f71cb1bb782ac4df3f1e18 Mon Sep 17 00:00:00 2001 From: matttrach Date: Thu, 26 Jun 2025 10:50:02 -0500 Subject: [PATCH 2/6] fix: commit message validation Signed-off-by: matttrach --- .github/workflows/validate.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 20cc754..8c2c292 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -87,17 +87,19 @@ jobs: # This format enables automatic generation of changelogs and versioning filter() { COMMIT="$1" - output="$(echo "$COMMIT" | grep -e '^fix: ' -e '^feature: ' -e '^feat: ' -e 'refactor!: ' -e 'feature!: ' -e 'feat!: ' -e '^chore(main): ')" + output="$(echo "$COMMIT" | grep -e '^fix: ' -e '^feature: ' -e '^feat: ' -e '^refactor!: ' -e '^feature!: ' -e '^feat!: ' -e '^chore(main): ')" echo "$output" } prefix_check() { message="$1" - if [ "" != "$(filter "$message")" ]; then - echo "...Commit message does not start with the required prefix. + if [ -z "$(filter "$message")" ]; then + cat <<-EOT + ...Commit message does not start with the required prefix. Please use one of the following prefixes: fix:, feature:, feat:, refactor!:, feature!:, feat:!. 'chore(main): ' is also allowed for release PRs. This enables release-please to automatically determine the type of release (major, minor, patch) based on the commit message. - $message" + $message + EOT exit 1 else echo "...Commit message starts with the required prefix." @@ -105,7 +107,7 @@ jobs: } empty_check() { message="$1" - if [ "" == "$message" ]; then + if [ -z "$message" ]; then echo "...Empty commit message." exit 1 else @@ -141,9 +143,9 @@ jobs: while read -r message; do echo "checking message ^$message\$" - prefix_check "$message" empty_check "$message" length_check "$message" + prefix_check "$message" spell_check "$message" echo "message ^$message\$ passed all checks" done <<<"$COMMIT_MESSAGES" From 42e310464f15f10d5ea88e8d22f142261f737aa6 Mon Sep 17 00:00:00 2001 From: matttrach Date: Thu, 26 Jun 2025 10:57:15 -0500 Subject: [PATCH 3/6] fix: validate Signed-off-by: matttrach --- .github/workflows/validate.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 8c2c292..5eb505b 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -93,13 +93,12 @@ jobs: prefix_check() { message="$1" if [ -z "$(filter "$message")" ]; then - cat <<-EOT + echo " ...Commit message does not start with the required prefix. Please use one of the following prefixes: fix:, feature:, feat:, refactor!:, feature!:, feat:!. 'chore(main): ' is also allowed for release PRs. This enables release-please to automatically determine the type of release (major, minor, patch) based on the commit message. - $message - EOT + $message" exit 1 else echo "...Commit message starts with the required prefix." From 23da134eb8d1c4505c5eccaab6ddf7c286dc9c16 Mon Sep 17 00:00:00 2001 From: matttrach Date: Sat, 28 Jun 2025 00:15:04 -0500 Subject: [PATCH 4/6] fix: improve cleanup, add remote example Signed-off-by: matttrach --- .gitignore | 1 + examples/backend_s3/main.tf | 24 ++++++++++ examples/backend_s3/variables.tf | 14 ++++++ examples/backend_s3/versions.tf | 9 ++++ examples/three/main.tf | 10 ++++ modules/deploy/create.sh.tpl | 2 + modules/deploy/destroy.sh.tpl | 2 + modules/deploy/main.tf | 55 ++++++++++++++++----- modules/install_cert_manager/main.tf | 1 + modules/rancher_bootstrap/main.tf | 1 + run_tests.sh | 45 +++++++++++++++++ test/tests/downstream/basic_test.go | 10 ++-- test/tests/downstream/splitrole_test.go | 10 ++-- test/tests/one/basic_test.go | 11 ++--- test/tests/prod/basic_test.go | 10 ++-- test/tests/three/basic_test.go | 46 ++++++++++-------- test/tests/util.go | 64 +++++++++++++++++++++---- 17 files changed, 249 insertions(+), 66 deletions(-) create mode 100644 examples/backend_s3/main.tf create mode 100644 examples/backend_s3/variables.tf create mode 100644 examples/backend_s3/versions.tf diff --git a/.gitignore b/.gitignore index 4456694..3f19536 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ run.sh *.test static_files.txt * copy +*.tfbackend diff --git a/examples/backend_s3/main.tf b/examples/backend_s3/main.tf new file mode 100644 index 0000000..c6fbf58 --- /dev/null +++ b/examples/backend_s3/main.tf @@ -0,0 +1,24 @@ +provider "aws" { + default_tags { + tags = { + Id = local.identifier + Owner = local.owner + } + } +} + +locals { + identifier = var.identifier + owner = var.owner +} + +module "s3_bucket" { + source = "terraform-aws-modules/s3-bucket/aws" + version = "5.0.2" + bucket = lower(local.identifier) + force_destroy = true + versioning = { + status = true + mfa_delete = false + } +} diff --git a/examples/backend_s3/variables.tf b/examples/backend_s3/variables.tf new file mode 100644 index 0000000..6b96837 --- /dev/null +++ b/examples/backend_s3/variables.tf @@ -0,0 +1,14 @@ +variable "identifier" { + type = string + description = <<-EOT + Unique ID for the resource, a tag will be added to the resource. + This helps with identifying and cleaning up resources. + EOT +} +variable "owner" { + type = string + description = <<-EOT + Owner tag to be added to the resource, helps when identifying and cleaning up resources. + Often this is an email address, so that someone can see and contact the person who generated the object. + EOT +} diff --git a/examples/backend_s3/versions.tf b/examples/backend_s3/versions.tf new file mode 100644 index 0000000..43fc7d3 --- /dev/null +++ b/examples/backend_s3/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.5.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.11" + } + } +} diff --git a/examples/three/main.tf b/examples/three/main.tf index 54e0727..65eda74 100644 --- a/examples/three/main.tf +++ b/examples/three/main.tf @@ -22,6 +22,16 @@ provider "rancher2" { timeout = "300s" } +terraform { + backend "s3" { + # This needs to be set in the backend configs on the command line. + # bucket = local.identifier + # https://developer.hashicorp.com/terraform/language/backend/s3 + # https://developer.hashicorp.com/terraform/language/backend#partial-configuration + key = "tfstate" + } +} + resource "rancher2_bootstrap" "authenticate" { depends_on = [ module.rancher, diff --git a/modules/deploy/create.sh.tpl b/modules/deploy/create.sh.tpl index e360831..a68ff92 100644 --- a/modules/deploy/create.sh.tpl +++ b/modules/deploy/create.sh.tpl @@ -1,6 +1,8 @@ ${export_contents} cd ${deploy_path} export TF_DATA_DIR="${tf_data_dir}" +TF_CLI_ARGS_init="" +TF_CLI_ARGS_apply="" ${init_script} diff --git a/modules/deploy/destroy.sh.tpl b/modules/deploy/destroy.sh.tpl index 7b7f50f..23010d7 100644 --- a/modules/deploy/destroy.sh.tpl +++ b/modules/deploy/destroy.sh.tpl @@ -1,5 +1,7 @@ ${export_contents} cd ${deploy_path} +TF_CLI_ARGS_init="" +TF_CLI_ARGS_apply="" export TF_DATA_DIR="${tf_data_dir}" if [ -z "${skip_destroy}" ]; then timeout -k 1m ${timeout} terraform init -upgrade diff --git a/modules/deploy/main.tf b/modules/deploy/main.tf index e8e4813..59b56c3 100644 --- a/modules/deploy/main.tf +++ b/modules/deploy/main.tf @@ -76,11 +76,11 @@ resource "terraform_data" "destroy" { provisioner "local-exec" { when = destroy command = templatefile("${path.module}/destroy.sh.tpl", { - export_hash = self.triggers_replace.env - tf_data_dir = self.triggers_replace.dd - deploy_path = self.triggers_replace.dp - skip_destroy = self.triggers_replace.sd - timeout = self.triggers_replace.to + export_contents = self.triggers_replace.ec + tf_data_dir = self.triggers_replace.dd + deploy_path = self.triggers_replace.dp + skip_destroy = self.triggers_replace.sd + timeout = self.triggers_replace.to }) } } @@ -92,9 +92,7 @@ resource "terraform_data" "create" { terraform_data.destroy, ] triggers_replace = { - inputs = local.inputs_hash - files = local.template_files_hash - env = local.export_hash + files = local.template_files_hash } provisioner "local-exec" { command = templatefile("${path.module}/create.sh.tpl", { @@ -135,6 +133,36 @@ module "persist_outputs" { recreate = terraform_data.create.id } +# during initial create this should be an extra apply that has no effect +# when the inputs change and the template needs to be rebuilt this will allow the persist +# to rebuild the template before running the create script +resource "terraform_data" "create_after_persist" { + depends_on = [ + module.persist_template, + module.persist_inputs, + terraform_data.destroy, + terraform_data.create, + module.persist_state, + module.persist_outputs, + ] + triggers_replace = { + inputs = local.inputs_hash + files = local.template_files_hash + env = local.export_hash + } + provisioner "local-exec" { + command = templatefile("${path.module}/create.sh.tpl", { + export_contents = local.export_contents + deploy_path = local.deploy_path + tf_data_dir = local.tf_data_dir + init_script = local.init_script + attempts = local.attempts + timeout = local.timeout + interval = local.interval + }) + } +} + resource "terraform_data" "destroy_end" { depends_on = [ module.persist_template, @@ -143,6 +171,7 @@ resource "terraform_data" "destroy_end" { terraform_data.create, module.persist_state, module.persist_outputs, + terraform_data.create_after_persist, ] triggers_replace = { inputs = local.inputs_hash @@ -157,11 +186,11 @@ resource "terraform_data" "destroy_end" { provisioner "local-exec" { when = destroy command = templatefile("${path.module}/destroy.sh.tpl", { - export_hash = self.triggers_replace.env - tf_data_dir = self.triggers_replace.dd - deploy_path = self.triggers_replace.dp - skip_destroy = self.triggers_replace.sd - timeout = self.triggers_replace.to + export_contents = self.triggers_replace.ec + tf_data_dir = self.triggers_replace.dd + deploy_path = self.triggers_replace.dp + skip_destroy = self.triggers_replace.sd + timeout = self.triggers_replace.to }) } } diff --git a/modules/install_cert_manager/main.tf b/modules/install_cert_manager/main.tf index b42bb5a..27a5a0c 100644 --- a/modules/install_cert_manager/main.tf +++ b/modules/install_cert_manager/main.tf @@ -24,6 +24,7 @@ module "deploy_cert_manager" { deploy_path = local.deploy_path data_path = local.deploy_path template_path = local.cert_manager_path + skip_destroy = true # this is a one way operation, uninstall is not supported environment_variables = { KUBE_CONFIG_PATH = "${abspath(local.path)}/kubeconfig" KUBECONFIG = "${abspath(local.path)}/kubeconfig" diff --git a/modules/rancher_bootstrap/main.tf b/modules/rancher_bootstrap/main.tf index 9c48db1..8e968bb 100644 --- a/modules/rancher_bootstrap/main.tf +++ b/modules/rancher_bootstrap/main.tf @@ -23,6 +23,7 @@ module "deploy_rancher" { deploy_path = local.deploy_path data_path = local.deploy_path template_path = local.rancher_path + skip_destroy = true # this is a one way operation, uninstall not supported environment_variables = { KUBECONFIG = "${local.path}/kubeconfig" KUBE_CONFIG_PATH = "${local.path}/kubeconfig" diff --git a/run_tests.sh b/run_tests.sh index 541828a..724c827 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -156,6 +156,7 @@ if [ -n "$IDENTIFIER" ]; then echo "Warning: Failed to clear all resources after 3 attempts." fi + # remove key pairs attempts=0 # shellcheck disable=SC2143 while [ -n "$(leftovers -d --iaas=aws --aws-region="$AWS_REGION" --type="ec2-key-pair" --filter="terraform-ci-$IDENTIFIER" | grep -v 'AccessDenied')" ] && [ $attempts -lt 3 ]; do @@ -167,6 +168,50 @@ if [ -n "$IDENTIFIER" ]; then if [ $attempts -eq 3 ]; then echo "Warning: Failed to clear all EC2 key pairs after 3 attempts." fi + + # remove s3 storage + attempts=0 + # shellcheck disable=SC2143 + ID="$(aws s3 ls | grep -i "$IDENTIFIER" | awk '{print $3}')" + while [ -n "$(aws s3 ls | grep -i "$IDENTIFIER")" ] && [ $attempts -lt 3 ]; do + echo "found s3 bucket $ID, removing..." + while read -r v; do + if [ -z "$v" ]; then continue; fi; + aws s3api delete-object --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" --key "tfstate" --version-id="$v" + done <<<"$( + aws s3api list-object-versions --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" | jq -r '.Versions[]?.VersionId' + )" + + while read -r v; do + if [ -z "$v" ]; then continue; fi; + aws s3api delete-object --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" --key "tfstate" --version-id="$v"; + done <<<"$( + aws s3api list-object-versions --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" | jq -r '.DeleteMarkers[]?.VersionId' + )" + + aws s3api delete-bucket --bucket "$(echo "$ID" | tr '[:upper:]' '[:lower:]')" + + sleep 10 + attempts=$((attempts + 1)) + done + + # remove load balancer target groups + attempts=0 + # shellcheck disable=SC2143 + while [ $attempts -lt 3 ]; do + while read -r line; do + if [ -z "$line" ]; then continue; fi + echo "removing load balancer target group, $line..." + aws elbv2 delete-target-group --target-group-arn $line; + done <<<"$( + while read -r line; do + if [ -z "$line" ]; then continue; fi + aws elbv2 describe-tags --resource-arns $line | jq -r --arg id "$IDENTIFIER" '.TagDescriptions[] | select(any(.Tags[]; .Key == "Id" and .Value == $id)) | .ResourceArn // ""'; + done <<<"$(aws elbv2 describe-target-groups | jq -r '.TargetGroups[]?.TargetGroupArn')" + )" + sleep 10 + attempts=$((attempts + 1)) + done fi if [ -f "/tmp/${IDENTIFIER}_failed_tests.txt" ]; then diff --git a/test/tests/downstream/basic_test.go b/test/tests/downstream/basic_test.go index f31a0c6..45fbc99 100644 --- a/test/tests/downstream/basic_test.go +++ b/test/tests/downstream/basic_test.go @@ -111,13 +111,13 @@ func TestDownstreamBasic(t *testing.T) { Upgrade: true, }) + var tfOptions []*terraform.Options + tfOptions = append(tfOptions, terraformOptions) _, err = terraform.InitAndApplyE(t, terraformOptions) if err != nil { t.Log("Test failed, tearing down...") util.GetErrorLogs(t, testDir+"/kubeconfig") - util.Teardown(t, testDir, terraformOptions, keyPair) - os.Remove(exampleDir + ".terraform.lock.hcl") - sshAgent.Stop() + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) t.Fatalf("Error creating cluster: %s", err) } util.CheckReady(t, testDir+"/kubeconfig") @@ -127,7 +127,5 @@ func TestDownstreamBasic(t *testing.T) { } else { t.Log("Test passed...") } - util.Teardown(t, testDir, terraformOptions, keyPair) - os.Remove(exampleDir + "/.terraform.lock.hcl") - sshAgent.Stop() + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) } diff --git a/test/tests/downstream/splitrole_test.go b/test/tests/downstream/splitrole_test.go index 0d7763a..c07d2bf 100644 --- a/test/tests/downstream/splitrole_test.go +++ b/test/tests/downstream/splitrole_test.go @@ -109,13 +109,13 @@ func TestDownstreamSplitrole(t *testing.T) { Upgrade: true, }) + var tfOptions []*terraform.Options + tfOptions = append(tfOptions, terraformOptions) _, err = terraform.InitAndApplyE(t, terraformOptions) if err != nil { t.Log("Test failed, tearing down...") util.GetErrorLogs(t, testDir+"/kubeconfig") - util.Teardown(t, testDir, terraformOptions, keyPair) - os.Remove(exampleDir + ".terraform.lock.hcl") - sshAgent.Stop() + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) t.Fatalf("Error creating cluster: %s", err) } util.CheckReady(t, testDir+"/kubeconfig") @@ -125,7 +125,5 @@ func TestDownstreamSplitrole(t *testing.T) { } else { t.Log("Test passed...") } - util.Teardown(t, testDir, terraformOptions, keyPair) - os.Remove(exampleDir + "/.terraform.lock.hcl") - sshAgent.Stop() + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) } diff --git a/test/tests/one/basic_test.go b/test/tests/one/basic_test.go index e3e8ce3..0c2fa67 100644 --- a/test/tests/one/basic_test.go +++ b/test/tests/one/basic_test.go @@ -98,14 +98,13 @@ func TestOneBasic(t *testing.T) { SshAgent: sshAgent, Upgrade: true, }) - + var tfOptions []*terraform.Options + tfOptions = append(tfOptions, terraformOptions) _, err = terraform.InitAndApplyE(t, terraformOptions) if err != nil { t.Log("Test failed, tearing down...") util.GetErrorLogs(t, testDir+"/kubeconfig") - util.Teardown(t, testDir, terraformOptions, keyPair) - os.Remove(exampleDir + ".terraform.lock.hcl") - sshAgent.Stop() + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) t.Fatalf("Error creating cluster: %s", err) } util.CheckReady(t, testDir+"/kubeconfig") @@ -115,7 +114,5 @@ func TestOneBasic(t *testing.T) { } else { t.Log("Test passed...") } - util.Teardown(t, testDir, terraformOptions, keyPair) - os.Remove(exampleDir + "/.terraform.lock.hcl") - sshAgent.Stop() + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) } diff --git a/test/tests/prod/basic_test.go b/test/tests/prod/basic_test.go index 8e96406..0fae31d 100644 --- a/test/tests/prod/basic_test.go +++ b/test/tests/prod/basic_test.go @@ -99,13 +99,13 @@ func TestProdBasic(t *testing.T) { SshAgent: sshAgent, Upgrade: true, }) + var tfOptions []*terraform.Options + tfOptions = append(tfOptions, terraformOptions) _, err = terraform.InitAndApplyE(t, terraformOptions) if err != nil { t.Log("Test failed, tearing down...") util.GetErrorLogs(t, testDir+"/kubeconfig") - util.Teardown(t, testDir, terraformOptions, keyPair) - os.Remove(exampleDir + ".terraform.lock.hcl") - sshAgent.Stop() + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) t.Fatalf("Error creating cluster: %s", err) } util.CheckReady(t, testDir+"/kubeconfig") @@ -115,7 +115,5 @@ func TestProdBasic(t *testing.T) { } else { t.Log("Test passed...") } - util.Teardown(t, testDir, terraformOptions, keyPair) - os.Remove(exampleDir + "/.terraform.lock.hcl") - sshAgent.Stop() + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) } diff --git a/test/tests/three/basic_test.go b/test/tests/three/basic_test.go index 788816d..831b581 100644 --- a/test/tests/three/basic_test.go +++ b/test/tests/three/basic_test.go @@ -3,6 +3,7 @@ package one import ( "os" "path/filepath" + "strings" "testing" aws "github.com/gruntwork-io/terratest/modules/aws" @@ -38,23 +39,31 @@ func TestThreeBasic(t *testing.T) { os.RemoveAll(testDir) t.Fatalf("Error creating test key pair: %s", err) } - err = os.WriteFile(testDir+"/id_rsa", []byte(keyPair.KeyPair.PrivateKey), 0600) if err != nil { + err = aws.DeleteEC2KeyPairE(t, keyPair) + if err != nil { + t.Logf("Failed to destroy key pair: %v", err) + } os.RemoveAll(testDir) t.Fatalf("Error creating test key pair: %s", err) } - sshAgent := ssh.SshAgentWithKeyPair(t, keyPair.KeyPair) + sshAgent := ssh.SshAgentWithKeyPair(t, keyPair.KeyPair) t.Logf("Key %s created and added to agent", keyPair.Name) - // here we need to create an object storage to hold the remote state + var tfOptions []*terraform.Options + backendTerraformOptions, err := util.CreateObjectStorageBackend(t, testDir, id, owner, region) + tfOptions = append(tfOptions, backendTerraformOptions) + if err != nil { + t.Log("Test failed, tearing down...") + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) + t.Fatalf("Error creating cluster: %s", err) + } // use oldest RKE2, remember it releases much more than Rancher _, _, rke2Version, err := util.GetRke2Releases() if err != nil { - os.RemoveAll(testDir) - aws.DeleteEC2KeyPair(t, keyPair) - sshAgent.Stop() + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) t.Fatalf("Error getting Rke2 release version: %s", err) } @@ -65,9 +74,7 @@ func TestThreeBasic(t *testing.T) { _, rancherVersion, _, err = util.GetRancherReleases() } if err != nil { - os.RemoveAll(testDir) - aws.DeleteEC2KeyPair(t, keyPair) - sshAgent.Stop() + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) t.Fatalf("Error getting Rancher release version: %s", err) } @@ -90,24 +97,25 @@ func TestThreeBasic(t *testing.T) { "AWS_REGION": region, "TF_DATA_DIR": testDir, "TF_IN_AUTOMATION": "1", - "TF_CLI_ARGS_plan": "-no-color -state=" + testDir + "/tfstate", - "TF_CLI_ARGS_apply": "-no-color -state=" + testDir + "/tfstate", - "TF_CLI_ARGS_destroy": "-no-color -state=" + testDir + "/tfstate", - "TF_CLI_ARGS_output": "-no-color -state=" + testDir + "/tfstate", + "TF_CLI_ARGS_init": "-backend-config=\"bucket=" + strings.ToLower(id) + "\"", + "TF_CLI_ARGS_plan": "-no-color", + "TF_CLI_ARGS_apply": "-no-color", + "TF_CLI_ARGS_destroy": "-no-color", + "TF_CLI_ARGS_output": "-no-color", }, RetryableTerraformErrors: util.GetRetryableTerraformErrors(), NoColor: true, SshAgent: sshAgent, Upgrade: true, }) - + // we need to prepend the main options because we need to destroy it before the backend + tfOptions = append([]*terraform.Options{terraformOptions}, tfOptions...) + t.Logf("tfOptions: %v", tfOptions) _, err = terraform.InitAndApplyE(t, terraformOptions) if err != nil { t.Log("Test failed, tearing down...") util.GetErrorLogs(t, testDir+"/kubeconfig") - util.Teardown(t, testDir, terraformOptions, keyPair) - os.Remove(exampleDir + ".terraform.lock.hcl") - sshAgent.Stop() + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) t.Fatalf("Error creating cluster: %s", err) } util.CheckReady(t, testDir+"/kubeconfig") @@ -117,7 +125,5 @@ func TestThreeBasic(t *testing.T) { } else { t.Log("Test passed...") } - util.Teardown(t, testDir, terraformOptions, keyPair) - os.Remove(exampleDir + "/.terraform.lock.hcl") - sshAgent.Stop() + util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) } diff --git a/test/tests/util.go b/test/tests/util.go index 19513ce..03bac3f 100644 --- a/test/tests/util.go +++ b/test/tests/util.go @@ -17,6 +17,7 @@ import ( g "github.com/gruntwork-io/terratest/modules/git" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/shell" + "github.com/gruntwork-io/terratest/modules/ssh" "github.com/gruntwork-io/terratest/modules/terraform" "golang.org/x/oauth2" ) @@ -375,6 +376,7 @@ func CreateTestDirectories(t *testing.T, id string) error { paths := []string{ filepath.Join(fwd, "test/tests/data"), filepath.Join(fwd, "test/tests/data", id), + filepath.Join(fwd, "test/tests/data", id, "backend"), filepath.Join(fwd, "test/tests/data", id, "data"), } for _, path := range paths { @@ -386,26 +388,37 @@ func CreateTestDirectories(t *testing.T, id string) error { return nil } -func Teardown(t *testing.T, directory string, options *terraform.Options, keyPair *aws.Ec2Keypair) { +func Teardown(t *testing.T, dataDir string, exampleDir string, options []*terraform.Options, keyPair *aws.Ec2Keypair, agent *ssh.SshAgent) { directoryExists := true - _, err := os.Stat(directory) + _, err := os.Stat(dataDir) if err != nil { if os.IsNotExist(err) { directoryExists = false } } if directoryExists { - _, err := terraform.DestroyE(t, options) - if err != nil { - t.Logf("Failed to destroy: %v", err) + for _, option := range options { + t.Logf("Tearing down %v", option.TerraformDir) + _, err = terraform.InitE(t, option) + if err != nil { + t.Logf("Failed to init for destroy: %v", err) + } + _, err = terraform.DestroyE(t, option) + if err != nil { + t.Logf("Failed to destroy: %v", err) + } } - - err = os.RemoveAll(directory) + err = os.RemoveAll(dataDir) if err != nil { t.Logf("Failed to delete test data directory: %v", err) } } - aws.DeleteEC2KeyPair(t, keyPair) + agent.Stop() + err = aws.DeleteEC2KeyPairE(t, keyPair) + if err != nil { + t.Logf("Failed to destroy key pair: %v", err) + } + os.Remove(exampleDir + "/.terraform.lock.hcl") } func GetErrorLogs(t *testing.T, kubeconfigPath string) { @@ -482,3 +495,38 @@ func CheckRunning(t *testing.T, kubeconfigPath string) { } t.Logf("Ready script output: %s", out) } + +func CreateObjectStorageBackend(t *testing.T, testDir string, id string, owner string, region string) (*terraform.Options, error) { + repoRoot, err := filepath.Abs(g.GetRepoRoot(t)) + if err != nil { + t.Fatalf("Error getting git root directory: %v", err) + } + exampleDir := repoRoot + "/examples/backend_s3" + + terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ + TerraformDir: exampleDir, + // Variables to pass to our Terraform code using -var options + Vars: map[string]interface{}{ + "identifier": id, + "owner": owner, + }, + // Environment variables to set when running Terraform + EnvVars: map[string]string{ + "AWS_DEFAULT_REGION": region, + "AWS_REGION": region, + "TF_DATA_DIR": testDir, + "TF_IN_AUTOMATION": "1", + "TF_CLI_ARGS_init": "-reconfigure", + "TF_CLI_ARGS_plan": "-state=" + testDir + "/backend/tfstate", + "TF_CLI_ARGS_apply": "-state=" + testDir + "/backend/tfstate", + "TF_CLI_ARGS_destroy": "-state=" + testDir + "/backend/tfstate", + "TF_CLI_ARGS_output": "-state=" + testDir + "/backend/tfstate", + }, + RetryableTerraformErrors: GetRetryableTerraformErrors(), + NoColor: true, + Upgrade: true, + }) + + _, err = terraform.InitAndApplyE(t, terraformOptions) + return terraformOptions, err +} From 1c47ba50c995838f953b15339368acc47f5db7e8 Mon Sep 17 00:00:00 2001 From: matttrach Date: Mon, 30 Jun 2025 10:02:42 -0500 Subject: [PATCH 5/6] fix: shell style Signed-off-by: matttrach --- run_tests.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 724c827..dd70717 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -21,6 +21,7 @@ EOT esac done +# shellcheck disable=SC2143 if [ -n "$cleanup_id" ]; then export IDENTIFIER="$cleanup_id" fi @@ -72,11 +73,13 @@ EOF fi local specific_test_flag="" + # shellcheck disable=SC2143 if [ -n "$specific_test" ]; then specific_test_flag="-run=$specific_test" fi local package_pattern="" + # shellcheck disable=SC2143 if [ -n "$specific_package" ]; then package_pattern="$specific_package" else @@ -143,6 +146,7 @@ fi echo "Clearing leftovers with Id $IDENTIFIER in $AWS_REGION..." +# shellcheck disable=SC2143 if [ -n "$IDENTIFIER" ]; then attempts=0 # shellcheck disable=SC2143 @@ -171,8 +175,8 @@ if [ -n "$IDENTIFIER" ]; then # remove s3 storage attempts=0 - # shellcheck disable=SC2143 ID="$(aws s3 ls | grep -i "$IDENTIFIER" | awk '{print $3}')" + # shellcheck disable=SC2143 while [ -n "$(aws s3 ls | grep -i "$IDENTIFIER")" ] && [ $attempts -lt 3 ]; do echo "found s3 bucket $ID, removing..." while read -r v; do @@ -202,11 +206,11 @@ if [ -n "$IDENTIFIER" ]; then while read -r line; do if [ -z "$line" ]; then continue; fi echo "removing load balancer target group, $line..." - aws elbv2 delete-target-group --target-group-arn $line; + aws elbv2 delete-target-group --target-group-arn "$line"; done <<<"$( while read -r line; do if [ -z "$line" ]; then continue; fi - aws elbv2 describe-tags --resource-arns $line | jq -r --arg id "$IDENTIFIER" '.TagDescriptions[] | select(any(.Tags[]; .Key == "Id" and .Value == $id)) | .ResourceArn // ""'; + aws elbv2 describe-tags --resource-arns "$line" | jq -r --arg id "$IDENTIFIER" '.TagDescriptions[] | select(any(.Tags[]; .Key == "Id" and .Value == $id)) | .ResourceArn // ""'; done <<<"$(aws elbv2 describe-target-groups | jq -r '.TargetGroups[]?.TargetGroupArn')" )" sleep 10 From 0b4635ea32212fabed4501d7c76ecea7e50d89e0 Mon Sep 17 00:00:00 2001 From: matttrach Date: Mon, 30 Jun 2025 17:08:40 -0500 Subject: [PATCH 6/6] fix: back end data directory Signed-off-by: matttrach --- flake.lock | 6 +++--- modules/deploy/destroy.sh.tpl | 2 +- modules/deploy/main.tf | 2 +- test/tests/three/basic_test.go | 25 ++++++++++--------------- test/tests/util.go | 10 ++++++++-- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/flake.lock b/flake.lock index 963cb0f..69bc1d0 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1750811787, - "narHash": "sha256-rD/978c35JXz6JLAzciTIOCMenPumF6zrQOj4rVZeHE=", + "lastModified": 1751180975, + "narHash": "sha256-BKk4yDiXr4LdF80OTVqYJ53Q74rOcA/82EClXug8xsY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "992f916556fcfaa94451ebc7fc6e396134bbf5b1", + "rev": "a48741b083d4f36dd79abd9f760c84da6b4dc0e5", "type": "github" }, "original": { diff --git a/modules/deploy/destroy.sh.tpl b/modules/deploy/destroy.sh.tpl index 23010d7..66f912c 100644 --- a/modules/deploy/destroy.sh.tpl +++ b/modules/deploy/destroy.sh.tpl @@ -4,7 +4,7 @@ TF_CLI_ARGS_init="" TF_CLI_ARGS_apply="" export TF_DATA_DIR="${tf_data_dir}" if [ -z "${skip_destroy}" ]; then - timeout -k 1m ${timeout} terraform init -upgrade + timeout -k 1m ${timeout} terraform init -upgrade -reconfigure timeout -k 1m ${timeout} terraform destroy -var-file="${deploy_path}/inputs.tfvars" -auto-approve -state="${deploy_path}/tfstate" || true else echo "Not destroying deployed module, it will no longer be managed here." diff --git a/modules/deploy/main.tf b/modules/deploy/main.tf index 59b56c3..caf3371 100644 --- a/modules/deploy/main.tf +++ b/modules/deploy/main.tf @@ -34,7 +34,7 @@ locals { interval = var.interval timeout = var.timeout init = var.init - init_script = (local.init ? "terraform init -upgrade" : "") + init_script = (local.init ? "terraform init -reconfigure -upgrade" : "") tf_data_dir = var.data_path != null ? var.data_path : path.root skip_destroy = (var.skip_destroy ? "true" : "") } diff --git a/test/tests/three/basic_test.go b/test/tests/three/basic_test.go index 831b581..46e7006 100644 --- a/test/tests/three/basic_test.go +++ b/test/tests/three/basic_test.go @@ -51,9 +51,8 @@ func TestThreeBasic(t *testing.T) { sshAgent := ssh.SshAgentWithKeyPair(t, keyPair.KeyPair) t.Logf("Key %s created and added to agent", keyPair.Name) - var tfOptions []*terraform.Options backendTerraformOptions, err := util.CreateObjectStorageBackend(t, testDir, id, owner, region) - tfOptions = append(tfOptions, backendTerraformOptions) + tfOptions := []*terraform.Options{backendTerraformOptions} if err != nil { t.Log("Test failed, tearing down...") util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) @@ -93,29 +92,25 @@ func TestThreeBasic(t *testing.T) { }, // Environment variables to set when running Terraform EnvVars: map[string]string{ - "AWS_DEFAULT_REGION": region, - "AWS_REGION": region, - "TF_DATA_DIR": testDir, - "TF_IN_AUTOMATION": "1", - "TF_CLI_ARGS_init": "-backend-config=\"bucket=" + strings.ToLower(id) + "\"", - "TF_CLI_ARGS_plan": "-no-color", - "TF_CLI_ARGS_apply": "-no-color", - "TF_CLI_ARGS_destroy": "-no-color", - "TF_CLI_ARGS_output": "-no-color", + "AWS_DEFAULT_REGION": region, + "AWS_REGION": region, + "TF_DATA_DIR": testDir, + "TF_IN_AUTOMATION": "1", + "TF_CLI_ARGS_init": "-backend-config=\"bucket=" + strings.ToLower(id) + "\"", }, RetryableTerraformErrors: util.GetRetryableTerraformErrors(), NoColor: true, SshAgent: sshAgent, + Reconfigure: true, Upgrade: true, }) // we need to prepend the main options because we need to destroy it before the backend - tfOptions = append([]*terraform.Options{terraformOptions}, tfOptions...) - t.Logf("tfOptions: %v", tfOptions) + newTfOptions := []*terraform.Options{terraformOptions, backendTerraformOptions} _, err = terraform.InitAndApplyE(t, terraformOptions) if err != nil { t.Log("Test failed, tearing down...") util.GetErrorLogs(t, testDir+"/kubeconfig") - util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) + util.Teardown(t, testDir, exampleDir, newTfOptions, keyPair, sshAgent) t.Fatalf("Error creating cluster: %s", err) } util.CheckReady(t, testDir+"/kubeconfig") @@ -125,5 +120,5 @@ func TestThreeBasic(t *testing.T) { } else { t.Log("Test passed...") } - util.Teardown(t, testDir, exampleDir, tfOptions, keyPair, sshAgent) + util.Teardown(t, testDir, exampleDir, newTfOptions, keyPair, sshAgent) } diff --git a/test/tests/util.go b/test/tests/util.go index 03bac3f..dfd5faa 100644 --- a/test/tests/util.go +++ b/test/tests/util.go @@ -3,6 +3,7 @@ package tests import ( "cmp" "context" + "encoding/json" "errors" "fmt" "os" @@ -399,6 +400,11 @@ func Teardown(t *testing.T, dataDir string, exampleDir string, options []*terraf if directoryExists { for _, option := range options { t.Logf("Tearing down %v", option.TerraformDir) + jsonOptions, err := json.Marshal(option) + if err != nil { + t.Logf("Failed to marshal options for destroy log: %v", err) + } + fmt.Println(string(jsonOptions)) _, err = terraform.InitE(t, option) if err != nil { t.Logf("Failed to init for destroy: %v", err) @@ -514,15 +520,15 @@ func CreateObjectStorageBackend(t *testing.T, testDir string, id string, owner s EnvVars: map[string]string{ "AWS_DEFAULT_REGION": region, "AWS_REGION": region, - "TF_DATA_DIR": testDir, + "TF_DATA_DIR": testDir + "/backend", "TF_IN_AUTOMATION": "1", - "TF_CLI_ARGS_init": "-reconfigure", "TF_CLI_ARGS_plan": "-state=" + testDir + "/backend/tfstate", "TF_CLI_ARGS_apply": "-state=" + testDir + "/backend/tfstate", "TF_CLI_ARGS_destroy": "-state=" + testDir + "/backend/tfstate", "TF_CLI_ARGS_output": "-state=" + testDir + "/backend/tfstate", }, RetryableTerraformErrors: GetRetryableTerraformErrors(), + Reconfigure: true, NoColor: true, Upgrade: true, })