feat(clean): reap leaked AMP integ-test workspaces

Integration tests create one AMP (Amazon Managed Service for Prometheus)
workspace per /amp run (alias cwagent-integ-test-<id>) and destroy it via
terraform. Workspaces left behind by cancelled or hard-killed jobs, whose
terraform state dies with the ephemeral runner, accumulate to the per-region
75-workspace quota. Once at the cap every new run fails at CreateWorkspace with
ServiceQuotaExceededException before the Go test binary runs (observed on the
al2:selinux_amp_test job). The existing if:cancelled()||failure() terraform
destroy step cannot reclaim them because their state is gone.

Add tool/clean/clean_amp/clean_amp.go (modeled on clean_eks): list AMP
workspaces and delete ACTIVE ones whose alias has the cwagent-integ-test-
prefix and were created longer ago than KeepDurationOneDay. Supports -dry-run.
Add a clean-amp-workspaces job to clean-aws-resources.yml using the same OIDC
role and daily schedule as the other cleaners.

Adding service/amp pulled an aligned aws-sdk-go-v2 core bump in the isolated
tool/clean module. Verified: go build and go vet are clean for clean_amp; a
dry-run against the test account matches an independent audit (targets only
cwagent-integ-test-* workspaces and spares non-test workspaces).

Note: the clean OIDC role (vars.TERRAFORM_AWS_ASSUME_ROLE) needs
amp:ListWorkspaces and amp:DeleteWorkspace permissions for the scheduled job.

Author: olowosulu

.github/workflows/clean-aws-resources.yml

@@ -311,3 +311,22 @@ jobs:
           set -e
           go run ./clean_security_group/clean_security_group.go || { echo "Failed to clean security groups"; exit 1; }
 
+  clean-amp-workspaces:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - uses: actions/setup-go@7b8cf10d4e4a01d4992d18a89f4d7dc5a3e6d6f4 # v4.3.0
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
+        with:
+          role-to-assume: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }}
+          aws-region: us-west-2
+
+      - name: Clean old amp workspaces
+        working-directory: tool/clean
+        run: go run ./clean_amp/clean_amp.go
+

tool/clean/clean_amp/clean_amp.go

@@ -0,0 +1,107 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/amp"
+	"github.com/aws/aws-sdk-go-v2/service/amp/types"
+
+	"github.com/aws/amazon-cloudwatch-agent/tool/clean"
+)
+
+// WorkspacesToClean lists the alias prefixes of AMP (Amazon Managed Service for
+// Prometheus) workspaces created by the integration-test harness. The harness sets
+// workspace_alias to "cwagent-integ-test-<testing_id>" in
+// terraform/ec2/linux/main.tf (module "amp").
+var WorkspacesToClean = []string{
+	"cwagent-integ-test-",
+}
+
+var dryRun bool
+
+// Clean AMP workspaces created by the integration tests if they have been open
+// longer than one day. A test run creates a workspace, uses it for a few minutes,
+// then destroys it via terraform. Workspaces left behind by cancelled or
+// hard-killed jobs (whose terraform state is lost with the ephemeral runner)
+// accumulate to the per-region 75-workspace quota; once at the cap every new run
+// fails at CreateWorkspace with ServiceQuotaExceededException. The existing
+// terraform destroy cleanup step cannot reclaim these because their state is gone,
+// so this cleaner deletes them out-of-band via the AMP API.
+func main() {
+	flag.BoolVar(&dryRun, "dry-run", false, "Enable dry-run mode (no actual deletion)")
+	flag.Parse()
+
+	if err := cleanWorkspaces(); err != nil {
+		log.Fatalf("errors cleaning %v", err)
+	}
+}
+
+func cleanWorkspaces() error {
+	log.Print("Begin to clean AMP workspaces")
+	ctx := context.Background()
+	defaultConfig, err := config.LoadDefaultConfig(ctx)
+	if err != nil {
+		return err
+	}
+	client := amp.NewFromConfig(defaultConfig)
+	return terminateWorkspaces(ctx, client)
+}
+
+func aliasMatchesWorkspacesToClean(alias string, workspacesToClean []string) bool {
+	for _, workspaceToClean := range workspacesToClean {
+		if strings.HasPrefix(alias, workspaceToClean) {
+			return true
+		}
+	}
+	return false
+}
+
+func terminateWorkspaces(ctx context.Context, client *amp.Client) error {
+	expirationDate := time.Now().UTC().Add(clean.KeepDurationOneDay)
+
+	paginator := amp.NewListWorkspacesPaginator(client, &amp.ListWorkspacesInput{})
+	for paginator.HasMorePages() {
+		page, err := paginator.NextPage(ctx)
+		if err != nil {
+			return err
+		}
+		for _, workspace := range page.Workspaces {
+			workspaceID := aws.ToString(workspace.WorkspaceId)
+			alias := aws.ToString(workspace.Alias)
+
+			if workspace.Status == nil || workspace.Status.StatusCode != types.WorkspaceStatusCodeActive {
+				log.Printf("Ignoring workspace %s (alias %q) since it is not ACTIVE", workspaceID, alias)
+				continue
+			}
+			if workspace.CreatedAt == nil || !expirationDate.After(*workspace.CreatedAt) {
+				log.Printf("Ignoring workspace %s (alias %q) with create-date %v since it was created in the last %s",
+					workspaceID, alias, aws.ToTime(workspace.CreatedAt), clean.KeepDurationOneDay)
+				continue
+			}
+			if !aliasMatchesWorkspacesToClean(alias, WorkspacesToClean) {
+				log.Printf("Ignoring workspace %s since alias %q does not match any clean prefix", workspaceID, alias)
+				continue
+			}
+			if dryRun {
+				log.Printf("Dry-Run: would delete workspace %s (alias %q, create-date %v)",
+					workspaceID, alias, aws.ToTime(workspace.CreatedAt))
+				continue
+			}
+			log.Printf("Try to delete workspace %s (alias %q, create-date %v)",
+				workspaceID, alias, aws.ToTime(workspace.CreatedAt))
+			if _, err := client.DeleteWorkspace(ctx, &amp.DeleteWorkspaceInput{WorkspaceId: workspace.WorkspaceId}); err != nil {
+				log.Printf("could not delete workspace %s err %v", workspaceID, err)
+			}
+		}
+	}
+	return nil
+}

tool/clean/go.mod

@@ -1,30 +1,29 @@
 module github.com/aws/amazon-cloudwatch-agent/tool/clean
 
-go 1.22
-
-toolchain go1.23.6
+go 1.24
 
 require (
 	github.com/aws/aws-sdk-go v1.48.14
-	github.com/aws/aws-sdk-go-v2 v1.36.2
+	github.com/aws/aws-sdk-go-v2 v1.43.4
 	github.com/aws/aws-sdk-go-v2/config v1.25.12
+	github.com/aws/aws-sdk-go-v2/service/amp v1.48.1
 	github.com/aws/aws-sdk-go-v2/service/autoscaling v1.36.3
 	github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.45.14
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.140.0
 	github.com/aws/aws-sdk-go-v2/service/ecs v1.35.3
 	github.com/aws/aws-sdk-go-v2/service/efs v1.26.3
 	github.com/aws/aws-sdk-go-v2/service/eks v1.35.3
 	github.com/aws/aws-sdk-go-v2/service/iam v1.28.3
-	github.com/aws/smithy-go v1.22.2
+	github.com/aws/smithy-go v1.27.6
 	github.com/stretchr/testify v1.8.4
 )
 
 require (
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.16.10 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 // indirect

tool/clean/go.sum

@@ -1,7 +1,7 @@
 github.com/aws/aws-sdk-go v1.48.14 h1:nVLrp+F84SG+xGiFMfe1TE6ZV6smF+42tuuNgYGV30s=
 github.com/aws/aws-sdk-go v1.48.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
-github.com/aws/aws-sdk-go-v2 v1.36.2 h1:Ub6I4lq/71+tPb/atswvToaLGVMxKZvjYDVOWEExOcU=
-github.com/aws/aws-sdk-go-v2 v1.36.2/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
+github.com/aws/aws-sdk-go-v2 v1.43.4 h1:b9FTvbRwy+JCsfp2Wp6wV/KbOx3Aj7nkoFb2cRX0IhE=
+github.com/aws/aws-sdk-go-v2 v1.43.4/go.mod h1:70vwSy16txshwG+g55WkpgPKDIByzHI8ccBsOteo3bQ=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
 github.com/aws/aws-sdk-go-v2/config v1.25.12 h1:mF4cMuNh/2G+d19nWnm1vJ/ak0qK6SbqF0KtSX9pxu0=
@@ -10,12 +10,14 @@ github.com/aws/aws-sdk-go-v2/credentials v1.16.10 h1:VmRkuoKaGl2ZDNGkkRQgw80Hxj1
 github.com/aws/aws-sdk-go-v2/credentials v1.16.10/go.mod h1:WEn22lpd50buTs/TDqywytW5xQ2zPOMbYipIlqI6xXg=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 h1:FZVFahMyZle6WcogZCOxo6D/lkDA2lqKIn4/ueUmVXw=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9/go.mod h1:kjq7REMIkxdtcEC9/4BVXjOsNY5isz6jQbEgk6osRTU=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33 h1:knLyPMw3r3JsU8MFHWctE4/e2qWbPaxDYLlohPvnY8c=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33/go.mod h1:EBp2HQ3f+XCB+5J+IoEbGhoV7CpJbnrsd4asNXmTL0A=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33 h1:K0+Ne08zqti8J9jwENxZ5NoUyBnaFDTu3apwQJWrwwA=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33/go.mod h1:K97stwwzaWzmqxO8yLGHhClbVW1tC6VT1pDLk1pGrq4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35 h1:kzVuGlatQtYinwBJEEyLAbggepCoavosiaHHX9+fD+c=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35/go.mod h1:0yLx0yEI+SfqeJMPvOtIEFoZbiQYXMGszBueiutQyaI=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35 h1:WK6CjihTuLisCjSKKbildJ79sGZZgbBz3iNa7VsKIhU=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35/go.mod h1:KYleN57luLoe97R7vTnx8PMcVrr9gAcRECtOjl91DNg=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/service/amp v1.48.1 h1:TfAuwl0nIPXZGitfHDE/cDalB0F4ifkf1UIp2vFyRcY=
+github.com/aws/aws-sdk-go-v2/service/amp v1.48.1/go.mod h1:61MYCp543XUJM6lhxz9IBQHR5rTZFU5mAWva5IuStl4=
 github.com/aws/aws-sdk-go-v2/service/autoscaling v1.36.3 h1:16TRfDZhx5aX90VsvG0yJ5XNlDNHMVDj2DBpVMwDxzc=
 github.com/aws/aws-sdk-go-v2/service/autoscaling v1.36.3/go.mod h1:dJD5FZKnDClUVIcCwfu676Y72h4GytmMoZHzf1nTW8Q=
 github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.45.14 h1:Xc90sglbEnAC1X4d4ui422Ppw0HWjyNoqGAE1Dq+Rcg=
@@ -40,8 +42,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.3 h1:CxAHBS0BWSUqI7qzXHc2ZpTe
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.3/go.mod h1:7Lt5mjQ8x5rVdKqg+sKKDeuwoszDJIIPmkd8BVsEdS0=
 github.com/aws/aws-sdk-go-v2/service/sts v1.26.3 h1:KfREzajmHCSYjCaMRtdLr9boUMA7KPpoPApitPlbNeo=
 github.com/aws/aws-sdk-go-v2/service/sts v1.26.3/go.mod h1:7Ld9eTqocTvJqqJ5K/orbSDwmGcpRdlDiLjz2DO+SL8=
-github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
-github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/aws/smithy-go v1.27.6 h1:0zjT8jgK3jbrTT7JJ3EE6JsMhX8JTrZ+f1sEndYDXrA=
+github.com/aws/smithy-go v1.27.6/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=