Skip to content

Commit 2e3e103

Browse files
danielfrankcomdanielfrankcom
authored
Clean up lingering clusters after test runs (#144)
Co-authored-by: Daniel Frankcom <frankcom@amazon.com>
1 parent 3a77c01 commit 2e3e103

12 files changed

Lines changed: 226 additions & 122 deletions

‎.github/scripts/clean-clusters.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/bin/bash
2+
3+
if [ -z "${IS_CI}" ]; then
4+
echo "Error: This script is intended to run only in CI environments."
5+
echo "Running it locally may delete clusters in your account."
6+
echo "Set the IS_CI environment variable to run this script."
7+
exit 1
8+
fi
9+
10+
REGIONS=("us-east-1" "us-east-2")
11+
12+
# Region and cluster ID can be extracted from ARN
13+
# ARN format: arn:aws:dsql:REGION:ACCOUNT:cluster/CLUSTER_ID
14+
declare -a ARNS=()
15+
declare -a FILTERED_ARNS=()
16+
17+
# Get clusters from each region and extract ARNs
18+
for region in "${REGIONS[@]}"; do
19+
echo "Listing clusters in $region..."
20+
21+
region_arns=$(aws dsql list-clusters --region "$region" | jq -r '.clusters[].arn')
22+
23+
# Add ARNs to the array if any were found
24+
if [ -n "$region_arns" ]; then
25+
while IFS= read -r arn; do
26+
ARNS+=("$arn")
27+
done <<< "$region_arns"
28+
fi
29+
done
30+
31+
echo -e "\nFound ${#ARNS[@]} cluster(s) across all regions:"
32+
printf '%s\n' "${ARNS[@]}"
33+
34+
echo -e "\nFiltering clusters..."
35+
for arn in "${ARNS[@]}"; do
36+
region=$(echo "$arn" | cut -d':' -f4)
37+
cluster_id=$(echo "$arn" | cut -d'/' -f2)
38+
39+
echo "Checking cluster $cluster_id in region $region..."
40+
41+
cluster_details=$(aws dsql get-cluster --region "$region" --identifier "$cluster_id")
42+
43+
status=$(echo "$cluster_details" | jq -r '.status')
44+
repo_tag=$(echo "$cluster_details" | jq -r '.tags.Repo // empty')
45+
46+
# We only want clusters that are not already deleting, and have the specific repo tag
47+
if [[ "$status" != "DELETED" && "$status" != "DELETING" && "$repo_tag" == "aws-samples/aurora-dsql-samples" ]]; then
48+
echo "Cluster $cluster_id qualifies for update (Status: $status, Repo tag: $repo_tag)"
49+
FILTERED_ARNS+=("$arn")
50+
else
51+
echo "Skipping cluster $cluster_id (Status: $status, Repo tag: $repo_tag)"
52+
fi
53+
done
54+
55+
echo -e "\nFound ${#FILTERED_ARNS[@]} cluster(s) that will be updated and deleted:"
56+
printf '%s\n' "${FILTERED_ARNS[@]}"
57+
58+
# Early exit if no clusters to update
59+
if [ ${#FILTERED_ARNS[@]} -eq 0 ]; then
60+
echo -e "\nNo clusters to update or delete. Exiting."
61+
exit 0
62+
fi
63+
64+
echo -e "\nUpdating filtered clusters to disable deletion protection..."
65+
for arn in "${FILTERED_ARNS[@]}"; do
66+
region=$(echo "$arn" | cut -d':' -f4)
67+
cluster_id=$(echo "$arn" | cut -d'/' -f2)
68+
69+
echo "Updating cluster $cluster_id in region $region..."
70+
aws dsql update-cluster --region "$region" --identifier "$cluster_id" --no-deletion-protection-enabled
71+
echo "Cluster $cluster_id updated successfully."
72+
done
73+
74+
echo -e "\nDeleting filtered clusters..."
75+
for arn in "${FILTERED_ARNS[@]}"; do
76+
region=$(echo "$arn" | cut -d':' -f4)
77+
cluster_id=$(echo "$arn" | cut -d'/' -f2)
78+
79+
echo "Deleting cluster $cluster_id in region $region..."
80+
aws dsql delete-cluster --region "$region" --identifier "$cluster_id"
81+
echo "Deletion initiated for cluster $cluster_id."
82+
done

‎.github/workflows/clean-clusters.yml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
name: Clean up Aurora DSQL Clusters
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
aws_region:
7+
required: false
8+
type: string
9+
default: 'us-east-1'
10+
description: 'Default AWS region for credentials, does not limit access to other regions'
11+
secrets:
12+
AWS_IAM_ROLE:
13+
required: true
14+
description: 'AWS IAM role to assume for cluster cleanup'
15+
16+
jobs:
17+
cleanup:
18+
name: Clean up Aurora DSQL Clusters
19+
runs-on: ubuntu-latest
20+
timeout-minutes: 10
21+
permissions:
22+
id-token: write
23+
contents: read
24+
25+
steps:
26+
- name: Checkout code
27+
uses: actions/checkout@v4
28+
29+
- name: Update AWS CLI to latest version
30+
run: |
31+
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
32+
unzip -q awscliv2.zip
33+
sudo ./aws/install --update
34+
aws --version
35+
36+
- name: Configure AWS Credentials
37+
uses: aws-actions/configure-aws-credentials@v4
38+
with:
39+
role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
40+
aws-region: ${{ inputs.aws_region }}
41+
42+
- name: Run cluster cleanup script
43+
env:
44+
IS_CI: "true"
45+
run: .github/scripts/clean-clusters.sh

‎.github/workflows/cpp-cm-integ-tests.yml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,4 +77,16 @@ jobs:
7777
CLUSTER_2_REGION: us-east-2
7878
WITNESS_REGION: us-west-2
7979
run: |
80-
./example
80+
./example
81+
82+
cleanup:
83+
if: always()
84+
needs: cpp-cm-integ-test
85+
uses: ./.github/workflows/clean-clusters.yml
86+
with:
87+
aws_region: 'us-east-1'
88+
secrets:
89+
AWS_IAM_ROLE: ${{ secrets.CPP_IAM_ROLE }}
90+
permissions:
91+
id-token: write
92+
contents: read

‎.github/workflows/dotnet-cm-integ-tests.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,15 @@ jobs:
6868
working-directory: ./dotnet/cluster_management
6969
run: |
7070
dotnet test
71+
72+
cleanup:
73+
if: always()
74+
needs: test
75+
uses: ./.github/workflows/clean-clusters.yml
76+
with:
77+
aws_region: 'us-east-1'
78+
secrets:
79+
AWS_IAM_ROLE: ${{ secrets.DOTNET_IAM_ROLE }}
80+
permissions:
81+
id-token: write
82+
contents: read

‎.github/workflows/go-cm-integ-tests.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,3 +90,15 @@ jobs:
9090
run: |
9191
go env -w GOPROXY=direct
9292
go test
93+
94+
cleanup:
95+
if: always()
96+
needs: build
97+
uses: ./.github/workflows/clean-clusters.yml
98+
with:
99+
aws_region: 'us-east-1'
100+
secrets:
101+
AWS_IAM_ROLE: ${{ secrets.GO_IAM_ROLE }}
102+
permissions:
103+
id-token: write
104+
contents: read

‎.github/workflows/java-cm-integ-tests.yml‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,20 @@ jobs:
5252

5353
- name: Configure and run integration for cluster management
5454
working-directory: ./java/cluster_management
55-
env:
56-
IS_CI: "true"
5755
run: |
5856
mvn validate
5957
mvn initialize
6058
mvn clean compile assembly:single
6159
mvn test
60+
61+
cleanup:
62+
if: always()
63+
needs: test
64+
uses: ./.github/workflows/clean-clusters.yml
65+
with:
66+
aws_region: 'us-east-1'
67+
secrets:
68+
AWS_IAM_ROLE: ${{ secrets.JAVA_IAM_ROLE }}
69+
permissions:
70+
id-token: write
71+
contents: read

‎.github/workflows/javascript-cm-integ-tests.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,15 @@ jobs:
5252
run: |
5353
npm install
5454
npm test
55+
56+
cleanup:
57+
if: always()
58+
needs: test
59+
uses: ./.github/workflows/clean-clusters.yml
60+
with:
61+
aws_region: 'us-east-1'
62+
secrets:
63+
AWS_IAM_ROLE: ${{ secrets.JAVASCRIPT_IAM_ROLE }}
64+
permissions:
65+
id-token: write
66+
contents: read

‎.github/workflows/python-cm-integ-tests.yml‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,6 @@ jobs:
5555

5656
- name: Configure and run integration for cluster management
5757
working-directory: ./python/cluster_management
58-
env:
59-
IS_CI: "TRUE"
6058
run: |
6159
python3 -m venv cm-integ
6260
source cm-integ/bin/activate
@@ -67,4 +65,15 @@ jobs:
6765
pip list
6866
echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH
6967
pytest -v test/
70-
68+
69+
cleanup:
70+
if: always()
71+
needs: test
72+
uses: ./.github/workflows/clean-clusters.yml
73+
with:
74+
aws_region: 'us-east-1'
75+
secrets:
76+
AWS_IAM_ROLE: ${{ secrets.PYTHON_IAM_ROLE }}
77+
permissions:
78+
id-token: write
79+
contents: read

‎.github/workflows/ruby-cm-integ-tests.yml‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,18 @@ jobs:
4848

4949
- name: Configure and run integration for cluster management
5050
working-directory: ./ruby/cluster_management
51-
env:
52-
IS_CI: "TRUE"
5351
run: |
5452
bundle install
5553
rspec
54+
55+
cleanup:
56+
if: always()
57+
needs: test
58+
uses: ./.github/workflows/clean-clusters.yml
59+
with:
60+
aws_region: 'us-east-1'
61+
secrets:
62+
AWS_IAM_ROLE: ${{ secrets.RUBY_IAM_ROLE }}
63+
permissions:
64+
id-token: write
65+
contents: read

‎.github/workflows/rust-cm-integ-tests.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,15 @@ jobs:
6565
working-directory: ./rust/cluster_management
6666
run: |
6767
cargo test -- --nocapture
68+
69+
cleanup:
70+
if: always()
71+
needs: test
72+
uses: ./.github/workflows/clean-clusters.yml
73+
with:
74+
aws_region: 'us-east-1'
75+
secrets:
76+
AWS_IAM_ROLE: ${{ secrets.RUST_IAM_ROLE }}
77+
permissions:
78+
id-token: write
79+
contents: read

0 commit comments

Comments
 (0)