Clean up lingering clusters after test runs

.github/scripts/clean-clusters.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+if [ -z "${IS_CI}" ]; then
+  echo "Error: This script is intended to run only in CI environments."
+  echo "Running it locally may delete clusters in your account."
+  echo "Set the IS_CI environment variable to run this script."
+  exit 1
+fi
+
+REGIONS=("us-east-1" "us-east-2")
+
+# Region and cluster ID can be extracted from ARN
+# ARN format: arn:aws:dsql:REGION:ACCOUNT:cluster/CLUSTER_ID
+declare -a ARNS=()
+declare -a FILTERED_ARNS=()
+
+# Get clusters from each region and extract ARNs
+for region in "${REGIONS[@]}"; do
+  echo "Listing clusters in $region..."
+
+  region_arns=$(aws dsql list-clusters --region "$region" | jq -r '.clusters[].arn')
+
+  # Add ARNs to the array if any were found
+  if [ -n "$region_arns" ]; then
+    while IFS= read -r arn; do
+      ARNS+=("$arn")
+    done <<< "$region_arns"
+  fi
+done
+
+echo -e "\nFound ${#ARNS[@]} cluster(s) across all regions:"
+printf '%s\n' "${ARNS[@]}"
+
+echo -e "\nFiltering clusters..."
+for arn in "${ARNS[@]}"; do
+  region=$(echo "$arn" | cut -d':' -f4)
+  cluster_id=$(echo "$arn" | cut -d'/' -f2)
+
+  echo "Checking cluster $cluster_id in region $region..."
+
+  cluster_details=$(aws dsql get-cluster --region "$region" --identifier "$cluster_id")
+
+  status=$(echo "$cluster_details" | jq -r '.status')
+  repo_tag=$(echo "$cluster_details" | jq -r '.tags.Repo // empty')
+
+  # We only want clusters that are not already deleting, and have the specific repo tag
+  if [[ "$status" != "DELETED" && "$status" != "DELETING" && "$repo_tag" == "aws-samples/aurora-dsql-samples" ]]; then
+    echo "Cluster $cluster_id qualifies for update (Status: $status, Repo tag: $repo_tag)"
+    FILTERED_ARNS+=("$arn")
+  else
+    echo "Skipping cluster $cluster_id (Status: $status, Repo tag: $repo_tag)"
+  fi
+done
+
+echo -e "\nFound ${#FILTERED_ARNS[@]} cluster(s) that will be updated and deleted:"
+printf '%s\n' "${FILTERED_ARNS[@]}"
+
+# Early exit if no clusters to update
+if [ ${#FILTERED_ARNS[@]} -eq 0 ]; then
+  echo -e "\nNo clusters to update or delete. Exiting."
+  exit 0
+fi
+
+echo -e "\nUpdating filtered clusters to disable deletion protection..."
+for arn in "${FILTERED_ARNS[@]}"; do
+  region=$(echo "$arn" | cut -d':' -f4)
+  cluster_id=$(echo "$arn" | cut -d'/' -f2)
+
+  echo "Updating cluster $cluster_id in region $region..."
+  aws dsql update-cluster --region "$region" --identifier "$cluster_id" --no-deletion-protection-enabled
+  echo "Cluster $cluster_id updated successfully."
+done
+
+echo -e "\nDeleting filtered clusters..."
+for arn in "${FILTERED_ARNS[@]}"; do
+  region=$(echo "$arn" | cut -d':' -f4)
+  cluster_id=$(echo "$arn" | cut -d'/' -f2)
+
+  echo "Deleting cluster $cluster_id in region $region..."
+  aws dsql delete-cluster --region "$region" --identifier "$cluster_id"
+  echo "Deletion initiated for cluster $cluster_id."
+done

.github/workflows/clean-clusters.yml

@@ -0,0 +1,45 @@
+name: Clean up Aurora DSQL Clusters
+
+on:
+  workflow_call:
+    inputs:
+      aws_region:
+        required: false
+        type: string
+        default: 'us-east-1'
+        description: 'Default AWS region for credentials, does not limit access to other regions'
+    secrets:
+      AWS_IAM_ROLE:
+        required: true
+        description: 'AWS IAM role to assume for cluster cleanup'
+
+jobs:
+  cleanup:
+    name: Clean up Aurora DSQL Clusters
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Update AWS CLI to latest version
+        run: |
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+          unzip -q awscliv2.zip
+          sudo ./aws/install --update
+          aws --version
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
+          aws-region: ${{ inputs.aws_region }}
+
+      - name: Run cluster cleanup script
+        env:
+          IS_CI: "true"
+        run: .github/scripts/clean-clusters.sh

.github/workflows/cpp-cm-integ-tests.yml

@@ -77,4 +77,16 @@ jobs:
         CLUSTER_2_REGION: us-east-2
         WITNESS_REGION: us-west-2
       run: |
-        ./example
+        ./example
+
+  cleanup:
+    if: always()
+    needs: cpp-cm-integ-test
+    uses: ./.github/workflows/clean-clusters.yml
+    with:
+      aws_region: 'us-east-1'
+    secrets:
+      AWS_IAM_ROLE: ${{ secrets.CPP_IAM_ROLE }}
+    permissions:
+      id-token: write
+      contents: read

.github/workflows/dotnet-cm-integ-tests.yml

@@ -68,3 +68,15 @@ jobs:
         working-directory: ./dotnet/cluster_management
         run: |
           dotnet test
+
+  cleanup:
+    if: always()
+    needs: test
+    uses: ./.github/workflows/clean-clusters.yml
+    with:
+      aws_region: 'us-east-1'
+    secrets:
+      AWS_IAM_ROLE: ${{ secrets.DOTNET_IAM_ROLE }}
+    permissions:
+      id-token: write
+      contents: read

.github/workflows/go-cm-integ-tests.yml

@@ -90,3 +90,15 @@ jobs:
       run: |
         go env -w GOPROXY=direct
         go test
+
+  cleanup:
+    if: always()
+    needs: build
+    uses: ./.github/workflows/clean-clusters.yml
+    with:
+      aws_region: 'us-east-1'
+    secrets:
+      AWS_IAM_ROLE: ${{ secrets.GO_IAM_ROLE }}
+    permissions:
+      id-token: write
+      contents: read

.github/workflows/java-cm-integ-tests.yml

@@ -52,10 +52,20 @@ jobs:
 
     - name: Configure and run integration for cluster management
       working-directory: ./java/cluster_management
-      env:
-        IS_CI: "true"
       run: |
         mvn validate
         mvn initialize
         mvn clean compile assembly:single
         mvn test
+
+  cleanup:
+    if: always()
+    needs: test
+    uses: ./.github/workflows/clean-clusters.yml
+    with:
+      aws_region: 'us-east-1'
+    secrets:
+      AWS_IAM_ROLE: ${{ secrets.JAVA_IAM_ROLE }}
+    permissions:
+      id-token: write
+      contents: read

.github/workflows/javascript-cm-integ-tests.yml

@@ -52,3 +52,15 @@ jobs:
         run: |
           npm install
           npm test
+
+  cleanup:
+    if: always()
+    needs: test
+    uses: ./.github/workflows/clean-clusters.yml
+    with:
+      aws_region: 'us-east-1'
+    secrets:
+      AWS_IAM_ROLE: ${{ secrets.JAVASCRIPT_IAM_ROLE }}
+    permissions:
+      id-token: write
+      contents: read

.github/workflows/python-cm-integ-tests.yml

@@ -55,8 +55,6 @@ jobs:
 
     - name: Configure and run integration for cluster management
       working-directory: ./python/cluster_management
-      env:
-        IS_CI: "TRUE"
       run: |
         python3 -m venv cm-integ
         source cm-integ/bin/activate
@@ -67,4 +65,15 @@ jobs:
         pip list
         echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH
         pytest -v test/
-
+
+  cleanup:
+    if: always()
+    needs: test
+    uses: ./.github/workflows/clean-clusters.yml
+    with:
+      aws_region: 'us-east-1'
+    secrets:
+      AWS_IAM_ROLE: ${{ secrets.PYTHON_IAM_ROLE }}
+    permissions:
+      id-token: write
+      contents: read

.github/workflows/ruby-cm-integ-tests.yml

@@ -48,8 +48,18 @@ jobs:
 
     - name: Configure and run integration for cluster management
       working-directory: ./ruby/cluster_management
-      env:
-        IS_CI: "TRUE"
       run: |
         bundle install
         rspec
+
+  cleanup:
+    if: always()
+    needs: test
+    uses: ./.github/workflows/clean-clusters.yml
+    with:
+      aws_region: 'us-east-1'
+    secrets:
+      AWS_IAM_ROLE: ${{ secrets.RUBY_IAM_ROLE }}
+    permissions:
+      id-token: write
+      contents: read

.github/workflows/rust-cm-integ-tests.yml

@@ -65,3 +65,15 @@ jobs:
         working-directory: ./rust/cluster_management
         run: |
           cargo test -- --nocapture
+
+  cleanup:
+    if: always()
+    needs: test
+    uses: ./.github/workflows/clean-clusters.yml
+    with:
+      aws_region: 'us-east-1'
+    secrets:
+      AWS_IAM_ROLE: ${{ secrets.RUST_IAM_ROLE }}
+    permissions:
+      id-token: write
+      contents: read

java/cluster_management/src/test/java/org/example/DsqlClusterManagementTest.java

@@ -42,21 +42,8 @@ static void setup() {
 
     @AfterAll
     static void teardown() {
-        try {
-            if (System.getenv().containsKey("IS_CI")) {
-                logger.info("This is a CI run. Scanning for leaked clusters");
-                logger.info("Deleting test clusters in " + region1);
-                deleteTestsClusters(client1);
-
-                logger.info("Deleting test clusters in " + region2);
-                deleteTestsClusters(client2);
-            } else {
-                logger.info("This is not a CI run. Skipping leaked cluster cleanup");
-            }
-        } finally {
-            client1.close();
-            client2.close();
-        }
+        client1.close();
+        client2.close();
     }
 
     @Test
@@ -94,42 +81,6 @@ public void multiRegionClusterLifecycle() {
         logger.info("Finished multi region cluster lifecycle run");
     }
 
-    /**
-     * Delete all clusters that are:
-     * <ol>
-     *     <li>Not already deleting; and,</li>
-     *     <li>Tagged with 'Repo=aws-samples/aurora-dsql-samples'; and,</li>
-     *     <li>Tagged with 'Name=java *'</li>
-     * </ol>
-     */
-    static void deleteTestsClusters(DsqlClient client) {
-        List<GetClusterResponse> clustersToDelete = client
-                .listClustersPaginator(SdkBuilder::build)
-                .clusters()
-                .stream()
-                .map(summary -> client.getCluster(r -> r.identifier(summary.identifier())))
-                .filter(c -> !Set.of(ClusterStatus.DELETED, ClusterStatus.DELETING).contains(c.status()))
-                .filter(GetClusterResponse::hasTags)
-                .filter(c -> {
-                    var tags = c.tags();
-                    boolean isTestCluster = tags.getOrDefault("Repo", "").equals("aws-samples/aurora-dsql-samples") &&
-                            tags.getOrDefault("Name", "").startsWith("java ");
-                    return isTestCluster;
-                })
-                .toList();
-
-        logger.info(String.format("Found %d clusters to delete", clustersToDelete.size()));
-        for (GetClusterResponse cluster : clustersToDelete) {
-            if (cluster.deletionProtectionEnabled()) {
-                logger.info("Disabling deletion protection on " + cluster.arn());
-                client.updateCluster(r -> r.identifier(cluster.identifier()).deletionProtectionEnabled(false));
-            }
-            logger.info("Deleting " + cluster);
-            client.deleteCluster(r -> r.identifier(cluster.identifier()));
-            logger.info("Deleted " + cluster.arn());
-        }
-    }
-
     static DsqlClient createClient(Region region) {
         return DsqlClient.builder()
                 .region(region)