diff --git a/.github/workflows/clean-aws-resources.yml b/.github/workflows/clean-aws-resources.yml index a2ec733a3c..5185a9924c 100644 --- a/.github/workflows/clean-aws-resources.yml +++ b/.github/workflows/clean-aws-resources.yml @@ -311,3 +311,22 @@ jobs: set -e go run ./clean_security_group/clean_security_group.go || { echo "Failed to clean security groups"; exit 1; } + clean-amp-workspaces: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/setup-go@7b8cf10d4e4a01d4992d18a89f4d7dc5a3e6d6f4 # v4.3.0 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 + with: + role-to-assume: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} + aws-region: us-west-2 + + - name: Clean old amp workspaces + working-directory: tool/clean + run: go run ./clean_amp/clean_amp.go + diff --git a/tool/clean/clean_amp/clean_amp.go b/tool/clean/clean_amp/clean_amp.go new file mode 100644 index 0000000000..9d754397f6 --- /dev/null +++ b/tool/clean/clean_amp/clean_amp.go @@ -0,0 +1,115 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package main + +import ( + "context" + "errors" + "flag" + "log" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/amp" + "github.com/aws/aws-sdk-go-v2/service/amp/types" + + "github.com/aws/amazon-cloudwatch-agent/tool/clean" +) + +// WorkspacesToClean lists the alias prefixes of AMP (Amazon Managed Service for +// Prometheus) workspaces created by the integration-test harness. The harness sets +// workspace_alias to "cwagent-integ-test-" in +// terraform/ec2/linux/main.tf (module "amp"). +var WorkspacesToClean = []string{ + "cwagent-integ-test-", +} + +var dryRun bool + +// Clean AMP workspaces created by the integration tests if they have been open +// longer than one day. A test run creates a workspace, uses it for a few minutes, +// then destroys it via terraform. Workspaces left behind by cancelled or +// hard-killed jobs (whose terraform state is lost with the ephemeral runner) +// accumulate to the per-region 75-workspace quota; once at the cap every new run +// fails at CreateWorkspace with ServiceQuotaExceededException. The existing +// terraform destroy cleanup step cannot reclaim these because their state is gone, +// so this cleaner deletes them out-of-band via the AMP API. +func main() { + flag.BoolVar(&dryRun, "dry-run", false, "Enable dry-run mode (no actual deletion)") + flag.Parse() + + if err := cleanWorkspaces(); err != nil { + log.Fatalf("errors cleaning %v", err) + } +} + +func cleanWorkspaces() error { + log.Print("Begin to clean AMP workspaces") + ctx := context.Background() + defaultConfig, err := config.LoadDefaultConfig(ctx) + if err != nil { + return err + } + client := amp.NewFromConfig(defaultConfig) + return terminateWorkspaces(ctx, client) +} + +func aliasMatchesWorkspacesToClean(alias string, workspacesToClean []string) bool { + for _, workspaceToClean := range workspacesToClean { + if workspaceToClean == "" { + continue + } + if strings.HasPrefix(alias, workspaceToClean) { + return true + } + } + return false +} + +func terminateWorkspaces(ctx context.Context, client *amp.Client) error { + expirationDate := time.Now().UTC().Add(clean.KeepDurationOneDay) + + var deleteErrors []error + paginator := amp.NewListWorkspacesPaginator(client, &.ListWorkspacesInput{}) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return err + } + for _, workspace := range page.Workspaces { + workspaceID := aws.ToString(workspace.WorkspaceId) + alias := aws.ToString(workspace.Alias) + + if workspace.Status == nil || + (workspace.Status.StatusCode != types.WorkspaceStatusCodeActive && + workspace.Status.StatusCode != types.WorkspaceStatusCodeCreationFailed) { + log.Printf("Ignoring workspace %s (alias %q) since it is not ACTIVE or CREATION_FAILED", workspaceID, alias) + continue + } + if workspace.CreatedAt == nil || !expirationDate.After(*workspace.CreatedAt) { + log.Printf("Ignoring workspace %s (alias %q) with create-date %v since it was created in the last %s", + workspaceID, alias, aws.ToTime(workspace.CreatedAt), -clean.KeepDurationOneDay) + continue + } + if !aliasMatchesWorkspacesToClean(alias, WorkspacesToClean) { + log.Printf("Ignoring workspace %s since alias %q does not match any clean prefix", workspaceID, alias) + continue + } + if dryRun { + log.Printf("Dry-Run: would delete workspace %s (alias %q, create-date %v)", + workspaceID, alias, aws.ToTime(workspace.CreatedAt)) + continue + } + log.Printf("Try to delete workspace %s (alias %q, create-date %v)", + workspaceID, alias, aws.ToTime(workspace.CreatedAt)) + if _, err := client.DeleteWorkspace(ctx, &.DeleteWorkspaceInput{WorkspaceId: workspace.WorkspaceId}); err != nil { + log.Printf("could not delete workspace %s err %v", workspaceID, err) + deleteErrors = append(deleteErrors, err) + } + } + } + return errors.Join(deleteErrors...) +} diff --git a/tool/clean/go.mod b/tool/clean/go.mod index e2209f0071..6b68b524f8 100644 --- a/tool/clean/go.mod +++ b/tool/clean/go.mod @@ -1,13 +1,12 @@ module github.com/aws/amazon-cloudwatch-agent/tool/clean -go 1.22 - -toolchain go1.23.6 +go 1.24 require ( github.com/aws/aws-sdk-go v1.48.14 - github.com/aws/aws-sdk-go-v2 v1.36.2 + github.com/aws/aws-sdk-go-v2 v1.43.4 github.com/aws/aws-sdk-go-v2/config v1.25.12 + github.com/aws/aws-sdk-go-v2/service/amp v1.48.1 github.com/aws/aws-sdk-go-v2/service/autoscaling v1.36.3 github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.45.14 github.com/aws/aws-sdk-go-v2/service/ec2 v1.140.0 @@ -15,7 +14,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/efs v1.26.3 github.com/aws/aws-sdk-go-v2/service/eks v1.35.3 github.com/aws/aws-sdk-go-v2/service/iam v1.28.3 - github.com/aws/smithy-go v1.22.2 + github.com/aws/smithy-go v1.27.6 github.com/stretchr/testify v1.8.4 ) @@ -23,8 +22,8 @@ require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.16.10 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 // indirect diff --git a/tool/clean/go.sum b/tool/clean/go.sum index ec10d06adb..e4085855af 100644 --- a/tool/clean/go.sum +++ b/tool/clean/go.sum @@ -1,7 +1,7 @@ github.com/aws/aws-sdk-go v1.48.14 h1:nVLrp+F84SG+xGiFMfe1TE6ZV6smF+42tuuNgYGV30s= github.com/aws/aws-sdk-go v1.48.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/aws-sdk-go-v2 v1.36.2 h1:Ub6I4lq/71+tPb/atswvToaLGVMxKZvjYDVOWEExOcU= -github.com/aws/aws-sdk-go-v2 v1.36.2/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2 v1.43.4 h1:b9FTvbRwy+JCsfp2Wp6wV/KbOx3Aj7nkoFb2cRX0IhE= +github.com/aws/aws-sdk-go-v2 v1.43.4/go.mod h1:70vwSy16txshwG+g55WkpgPKDIByzHI8ccBsOteo3bQ= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= github.com/aws/aws-sdk-go-v2/config v1.25.12 h1:mF4cMuNh/2G+d19nWnm1vJ/ak0qK6SbqF0KtSX9pxu0= @@ -10,12 +10,14 @@ github.com/aws/aws-sdk-go-v2/credentials v1.16.10 h1:VmRkuoKaGl2ZDNGkkRQgw80Hxj1 github.com/aws/aws-sdk-go-v2/credentials v1.16.10/go.mod h1:WEn22lpd50buTs/TDqywytW5xQ2zPOMbYipIlqI6xXg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 h1:FZVFahMyZle6WcogZCOxo6D/lkDA2lqKIn4/ueUmVXw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9/go.mod h1:kjq7REMIkxdtcEC9/4BVXjOsNY5isz6jQbEgk6osRTU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33 h1:knLyPMw3r3JsU8MFHWctE4/e2qWbPaxDYLlohPvnY8c= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33/go.mod h1:EBp2HQ3f+XCB+5J+IoEbGhoV7CpJbnrsd4asNXmTL0A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33 h1:K0+Ne08zqti8J9jwENxZ5NoUyBnaFDTu3apwQJWrwwA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33/go.mod h1:K97stwwzaWzmqxO8yLGHhClbVW1tC6VT1pDLk1pGrq4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35 h1:kzVuGlatQtYinwBJEEyLAbggepCoavosiaHHX9+fD+c= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.35/go.mod h1:0yLx0yEI+SfqeJMPvOtIEFoZbiQYXMGszBueiutQyaI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35 h1:WK6CjihTuLisCjSKKbildJ79sGZZgbBz3iNa7VsKIhU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.35/go.mod h1:KYleN57luLoe97R7vTnx8PMcVrr9gAcRECtOjl91DNg= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/service/amp v1.48.1 h1:TfAuwl0nIPXZGitfHDE/cDalB0F4ifkf1UIp2vFyRcY= +github.com/aws/aws-sdk-go-v2/service/amp v1.48.1/go.mod h1:61MYCp543XUJM6lhxz9IBQHR5rTZFU5mAWva5IuStl4= github.com/aws/aws-sdk-go-v2/service/autoscaling v1.36.3 h1:16TRfDZhx5aX90VsvG0yJ5XNlDNHMVDj2DBpVMwDxzc= github.com/aws/aws-sdk-go-v2/service/autoscaling v1.36.3/go.mod h1:dJD5FZKnDClUVIcCwfu676Y72h4GytmMoZHzf1nTW8Q= github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.45.14 h1:Xc90sglbEnAC1X4d4ui422Ppw0HWjyNoqGAE1Dq+Rcg= @@ -40,8 +42,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.3 h1:CxAHBS0BWSUqI7qzXHc2ZpTe github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.3/go.mod h1:7Lt5mjQ8x5rVdKqg+sKKDeuwoszDJIIPmkd8BVsEdS0= github.com/aws/aws-sdk-go-v2/service/sts v1.26.3 h1:KfREzajmHCSYjCaMRtdLr9boUMA7KPpoPApitPlbNeo= github.com/aws/aws-sdk-go-v2/service/sts v1.26.3/go.mod h1:7Ld9eTqocTvJqqJ5K/orbSDwmGcpRdlDiLjz2DO+SL8= -github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= -github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/smithy-go v1.27.6 h1:0zjT8jgK3jbrTT7JJ3EE6JsMhX8JTrZ+f1sEndYDXrA= +github.com/aws/smithy-go v1.27.6/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=