diff --git a/template-only-bin/cleanup-test-resources b/template-only-bin/cleanup-test-resources index 0d905ab8..8776e212 100755 --- a/template-only-bin/cleanup-test-resources +++ b/template-only-bin/cleanup-test-resources @@ -25,6 +25,13 @@ # ----------------------------------------------------------------------------- set -euo pipefail +export AWS_PAGER="" + +# add any executables next to this script to the PATH for easy calling +SCRIPT_PATH=$(dirname "$(realpath -s "$0")") +PATH=${SCRIPT_PATH}:${PATH} +export PATH + # Default values DRY_RUN=false PROJECT_NAME="" @@ -85,7 +92,7 @@ cleanup_project() { # Check if terraform state bucket exists (informational only) local bucket_name="${project}-${AWS_ACCOUNT_ID}-${AWS_REGION}-tf" - if ! aws s3api head-bucket --bucket "${bucket_name}" 2>/dev/null; then + if ! aws s3api head-bucket --bucket "${bucket_name}" &>/dev/null; then echo "Note: No terraform state bucket found for ${project}, but checking for remaining resources..." fi @@ -96,7 +103,7 @@ cleanup_project() { --region "${AWS_REGION}" \ --tag-filters Key=project,Values="${project}" \ --query 'ResourceTagMappingList[].ResourceARN' \ - --output text) + --output text | tr '\t' '\n') if [ -z "${resources}" ]; then echo "No resources found for project ${project}" @@ -104,78 +111,55 @@ cleanup_project() { fi local resource_count - resource_count=$(echo "${resources}" | wc -w) + resource_count=$(echo "${resources}" | wc -l) echo "Found ${resource_count} resources" if [ "${DRY_RUN}" = true ]; then echo "Would delete the following resources:" - echo "${resources}" | tr '\t' '\n' + echo "${resources}" return 0 fi + # Note: Some resources need to be deleted in specific order due to dependencies echo "Deleting resources..." - # Delete Route 53 hosted zones (the main blocker) + # Delete Route 53 hosted zones echo "Cleaning up Route 53 hosted zones..." - local zones - zones=$(aws route53 list-hosted-zones \ - --query "HostedZones[].Id" \ - --output text || echo "") - - for zone_id in ${zones}; do - # Extract just the ID - zone_id="${zone_id#/hostedzone/}" - - # Check if this zone belongs to our project - local zone_tags - zone_tags=$(aws route53 list-tags-for-resource \ - --resource-type hostedzone \ - --resource-id "${zone_id}" \ - --query "ResourceTagSet.Tags[?Key=='project'].Value" \ - --output text || echo "") - - if [ "${zone_tags}" = "${project}" ]; then - echo "Deleting hosted zone: ${zone_id}" - aws route53 delete-hosted-zone --id "${zone_id}" || echo "Failed to delete zone ${zone_id}" - fi - done + local hosted_zone_arns + hosted_zone_arns=$(echo "${resources}" | grep 'arn:aws:route53:.*:hostedzone/' || echo "") - # Delete other resources using AWS CLI - # Note: Some resources need to be deleted in specific order due to dependencies + for hosted_zone_arn in ${hosted_zone_arns}; do + hosted_zone_id=$(echo "${hosted_zone_arn}" | awk -F'/' '{print $NF}') + + aws route53 delete-hosted-zone --id "${hosted_zone_id}" || echo "Failed to delete hosted zone ${hosted_zone_id}" + done # Delete ECS services first echo "Cleaning up ECS services and clusters..." - local clusters - clusters=$(aws ecs list-clusters --region "${AWS_REGION}" --query 'clusterArns[]' --output text || echo "") - for cluster_arn in ${clusters}; do - local cluster_tags - cluster_tags=$(aws ecs list-tags-for-resource \ - --resource-arn "${cluster_arn}" \ - --query "tags[?key=='project'].value" \ - --output text || echo "") - - if [ "${cluster_tags}" = "${project}" ]; then - local cluster_name - cluster_name=$(echo "${cluster_arn}" | awk -F/ '{print $NF}') - echo "Deleting ECS cluster: ${cluster_name}" - - # Delete services in cluster first - local services - services=$(aws ecs list-services --cluster "${cluster_name}" --region "${AWS_REGION}" --query 'serviceArns[]' --output text || echo "") - for service in ${services}; do - aws ecs delete-service --cluster "${cluster_name}" --service "${service}" --force --region "${AWS_REGION}" || echo "Failed to delete service" - done - - # Then delete cluster - aws ecs delete-cluster --cluster "${cluster_name}" --region "${AWS_REGION}" || echo "Failed to delete cluster" - fi + local cluster_arns + cluster_arns=$(echo "${resources}" | grep 'arn:aws:ecs:.*:cluster/' || echo "") + + for cluster_arn in ${cluster_arns}; do + local cluster_name + cluster_name=$(echo "${cluster_arn}" | awk -F/ '{print $NF}') + echo "Deleting ECS cluster: ${cluster_name}" + + # Delete services in cluster first + local services + services=$(aws ecs list-services --cluster "${cluster_name}" --region "${AWS_REGION}" --query 'serviceArns[]' --output text || echo "") + for service in ${services}; do + aws ecs delete-service --cluster "${cluster_name}" --service "${service}" --force --region "${AWS_REGION}" || echo "Failed to delete service" + done + + # Then delete cluster + aws ecs delete-cluster --cluster "${cluster_name}" --region "${AWS_REGION}" || echo "Failed to delete cluster" done # Delete ECS task definitions # Task definitions are tagged by the Resource Groups Tagging API echo "Cleaning up ECS task definitions..." local task_def_arns - task_def_arns=$(echo "${resources}" | tr '\t' '\n' | grep 'task-definition' || echo "") + task_def_arns=$(echo "${resources}" | grep 'arn:aws:ecs:.*:task-definition/' || echo "") for task_def_arn in ${task_def_arns}; do # Check current status @@ -204,16 +188,11 @@ cleanup_project() { # Delete load balancers echo "Cleaning up load balancers..." - local lbs - lbs=$(aws elbv2 describe-load-balancers --region "${AWS_REGION}" --query 'LoadBalancers[].LoadBalancerArn' --output text || echo "") - for lb_arn in ${lbs}; do - local lb_tags - lb_tags=$(aws elbv2 describe-tags --resource-arns "${lb_arn}" --query "TagDescriptions[0].Tags[?Key=='project'].Value" --output text || echo "") - - if [ "${lb_tags}" = "${project}" ]; then - echo "Deleting load balancer: ${lb_arn}" - aws elbv2 delete-load-balancer --load-balancer-arn "${lb_arn}" --region "${AWS_REGION}" || echo "Failed to delete LB" - fi + local lb_arns + lb_arns=$(echo "${resources}" | grep 'arn:aws:elasticloadbalancing:.*:loadbalancer/' || echo "") + for lb_arn in ${lb_arns}; do + echo "Deleting load balancer: ${lb_arn}" + aws elbv2 delete-load-balancer --load-balancer-arn "${lb_arn}" --region "${AWS_REGION}" || echo "Failed to delete LB" done # Wait a bit for LB deletion @@ -222,7 +201,7 @@ cleanup_project() { # Delete target groups echo "Cleaning up target groups..." local tg_arns - tg_arns=$(echo "${resources}" | tr '\t' '\n' | grep 'targetgroup' || echo "") + tg_arns=$(echo "${resources}" | grep 'arn:aws:elasticloadbalancing:.*:targetgroup/' || echo "") for tg_arn in ${tg_arns}; do echo "Deleting target group: ${tg_arn}" @@ -232,7 +211,7 @@ cleanup_project() { # Delete S3 buckets echo "Cleaning up S3 buckets..." local s3_arns - s3_arns=$(echo "${resources}" | tr '\t' '\n' | grep 'arn:aws:s3:::' || echo "") + s3_arns=$(echo "${resources}" | grep 'arn:aws:s3:::' || echo "") for s3_arn in ${s3_arns}; do local bucket_name @@ -240,7 +219,7 @@ cleanup_project() { echo "Deleting S3 bucket: ${bucket_name}" # Empty bucket first (required before deletion) - aws s3 rm "s3://${bucket_name}" --recursive --region "${AWS_REGION}" 2>/dev/null || echo "Bucket already empty or inaccessible" + empty-s3-bucket "${bucket_name}" "${AWS_REGION}" # Delete bucket aws s3api delete-bucket --bucket "${bucket_name}" --region "${AWS_REGION}" || echo "Failed to delete bucket ${bucket_name}" @@ -249,7 +228,7 @@ cleanup_project() { # Delete DynamoDB tables echo "Cleaning up DynamoDB tables..." local dynamodb_arns - dynamodb_arns=$(echo "${resources}" | tr '\t' '\n' | grep 'dynamodb' || echo "") + dynamodb_arns=$(echo "${resources}" | grep 'dynamodb' || echo "") for table_arn in ${dynamodb_arns}; do local table_name @@ -258,18 +237,180 @@ cleanup_project() { aws dynamodb delete-table --table-name "${table_name}" --region "${AWS_REGION}" || echo "Failed to delete table ${table_name}" done + echo "Cleaning up Bedrock Data Automation..." + local bedrock_da_arns + bedrock_da_arns=$(echo "${resources}" | grep 'arn:aws:bedrock:.*:data-automation-project/' || echo "") + + for dba_project_arn in ${bedrock_da_arns}; do + echo "Deleting Bedrock Data Automation project: ${dba_project_arn}" + aws bedrock-data-automation delete-data-automation-project --project-arn "${dba_project_arn}" --region "${AWS_REGION}" || echo "Failed to delete BDA ${dba_project_arn}" + done + + local bedrock_blueprint_arns + bedrock_blueprint_arns=$(echo "${resources}" | grep 'arn:aws:bedrock:.*:blueprint/' || echo "") + + for bedrock_blueprint_arn in ${bedrock_blueprint_arns}; do + echo "Deleting BDA Blueprint: ${bedrock_blueprint_arn}" + aws bedrock-data-automation delete-blueprint --blueprint-arn "${bedrock_blueprint_arn}" --region "${AWS_REGION}" || echo "Failed to delete BDA Blueprint ${bedrock_blueprint_arn}" + done + + # Security Groups need to be deleted before the associated VPC can be deleted + echo "Cleaning up Security Groups..." + local security_group_arns + security_group_arns=$(echo "${resources}" | grep 'arn:aws:ec2:.*:security-group/' || echo "") + + for security_group_arn in ${security_group_arns}; do + local security_group_id + security_group_id=$(echo "${security_group_arn}" | awk -F'/' '{print $NF}') + + # Check if this is a default security group, which we can't delete + # separately, will be removed with the VPC itself + local security_group_description + security_group_description=$(aws ec2 describe-security-groups --group-ids "${security_group_id}" --query='SecurityGroups[0].Description' --output text 2>&1) + if [[ "${security_group_description}" = "default VPC security group" ]]; then + echo "Default security group, can't delete individually: ${security_group_id}" + continue + fi + + if [[ "${security_group_description}" = *InvalidGroup.NotFound* ]]; then + echo "Security group not found/already queued for deletion: ${security_group_id}" + continue + fi + + echo "Deleting Security Group: ${security_group_id}" + aws ec2 delete-security-group --group-id "${security_group_id}" --region "${AWS_REGION}" || echo "Failed to delete Security Group ${security_group_id}" + done + + # Subnets need to be deleted before the associated VPC can be deleted + echo "Cleaning up Subnets..." + local subnet_arns + subnet_arns=$(echo "${resources}" | grep 'arn:aws:ec2:.*:subnet/' || echo "") + + for subnet_arn in ${subnet_arns}; do + local subnet_id + subnet_id=$(echo "${subnet_arn}" | awk -F'/' '{print $NF}') + + echo "Deleting Subnet: ${subnet_id}" + aws ec2 delete-subnet --subnet-id "${subnet_id}" --region "${AWS_REGION}" || echo "Failed to delete Subnet ${subnet_id}" + + # Give it a second to delete before proceeding + sleep 1 + done + + # Internet Gateways need to be deleted before the associated VPC can be deleted + echo "Cleaning up Internet Gateways..." + local igw_arns + igw_arns=$(echo "${resources}" | grep 'arn:aws:ec2:.*:internet-gateway/' || echo "") + + for igw_arn in ${igw_arns}; do + local igw_id + igw_id=$(echo "${igw_arn}" | awk -F'/' '{print $NF}') + + # Need to detach the gateway before deleting + igw_vpc_ids=$(aws ec2 describe-internet-gateways --internet-gateway-ids "${igw_id}" --query 'InternetGateways[0].Attachments[*].VpcId' --output text | tr '\t' '\n') + for igw_vpc_id in ${igw_vpc_ids}; do + echo "Detaching Internet Gateway ${igw_id} from VPC ${igw_vpc_id}" + aws ec2 detach-internet-gateway --internet-gateway-id "${igw_id}" --vpc-id "${igw_vpc_id}" --region "${AWS_REGION}" || echo "Failed to detach Internet Gateway ${igw_id}" + done + + echo "Deleting Internet Gateway: ${igw_id}" + aws ec2 delete-internet-gateway --internet-gateway-id "${igw_id}" --region "${AWS_REGION}" || echo "Failed to delete Internet Gateway ${igw_id}" + + # Give it a second to delete before proceeding + sleep 1 + done + + echo "Cleaning up VPCs..." + local vpc_arns + vpc_arns=$(echo "${resources}" | grep 'arn:aws:ec2:.*:vpc/' || echo "") + + for vpc_arn in ${vpc_arns}; do + local vpc_id + vpc_id=$(echo "${vpc_arn}" | awk -F'/' '{print $NF}') + echo "Deleting VPC: ${vpc_id}" + aws ec2 delete-vpc --vpc-id "${vpc_id}" --region "${AWS_REGION}" || echo "Failed to delete VPC ${vpc_id}" + done + + echo "Cleaning up SNS..." + local sns_topic_arns + sns_topic_arns=$(echo "${resources}" | grep 'arn:aws:sns:.*' || echo "") + + for sns_topic_arn in ${sns_topic_arns}; do + echo "Deleting SNS Topic: ${sns_topic_arn}" + aws sns delete-topic --topic-arn "${sns_topic_arn}" --region "${AWS_REGION}" || echo "Failed to delete SNS Topic ${sns_topic_arn}" + done + + echo "Cleaning up logs..." + local log_group_arns + log_group_arns=$(echo "${resources}" | grep 'arn:aws:logs:.*:log-group:' || echo "") + + for log_group_arn in ${log_group_arns}; do + local log_group_name + log_group_name=$(echo "${log_group_arn}" | awk -F'log-group:' '{print $NF}') + echo "Deleting Log Group: ${log_group_name}" + aws logs delete-log-group --log-group-name "${log_group_name}" --region "${AWS_REGION}" || echo "Failed to delete Log Group ${log_group_name}" + done + # Schedule KMS keys for deletion (minimum 7 days waiting period) echo "Scheduling KMS keys for deletion..." local kms_arns - kms_arns=$(echo "${resources}" | tr '\t' '\n' | grep 'arn:aws:kms:' || echo "") + kms_arns=$(echo "${resources}" | grep 'arn:aws:kms:' || echo "") for key_arn in ${kms_arns}; do local key_id key_id=$(echo "${key_arn}" | awk -F'/' '{print $NF}') + + local key_state + key_state=$(aws kms describe-key --key-id "${key_id}" --query 'KeyMetadata.KeyState' --output text) + if [[ "${key_state}" = "PendingDeletion" ]]; then + local key_delete_time + key_delete_time=$(aws kms describe-key --key-id "${key_id}" --query 'KeyMetadata.DeletionDate' --output text) + echo "KMS key already scheduled for deletion: ${key_id} at ${key_delete_time}" + continue + fi + echo "Scheduling KMS key for deletion: ${key_id}" aws kms schedule-key-deletion --key-id "${key_id}" --pending-window-in-days 7 --region "${AWS_REGION}" || echo "Failed to schedule deletion for key ${key_id}" done + echo "Cleaning up IAM..." + # Policies are returned via the Resource Groups Tagging API, Roles/Users are + # not, so get at things via the Policy + local iam_policy_arns + iam_policy_arns=$(echo "${resources}" | grep 'arn:aws:iam:.*:policy/' || echo "") + # Track the roles the project policies are attached to for later deletion + # without having to loop through _all_ roles in the account, may do this + # different in the future + iam_role_names=() + + for iam_policy_arn in ${iam_policy_arns}; do + local attached_role_names + attached_role_names=$(aws iam list-entities-for-policy --policy-arn "${iam_policy_arn}" --entity-filter Role --query 'PolicyRoles[*].RoleName' --output text | tr '\t' '\n') + + for role_name in ${attached_role_names}; do + iam_role_names+=("${role_name}") + echo "Detaching policy from IAM Role: ${role_name}" + aws iam detach-role-policy --policy-arn "${iam_policy_arn}" --role-name "${role_name}" || echo "Failed to detach IAM policy from role: ${iam_policy_arn} from ${role_name}" + done + + echo "Deleting IAM Policy: ${iam_policy_arn}" + aws iam delete-policy --policy-arn "${iam_policy_arn}" --region "${AWS_REGION}" || echo "Failed to delete IAM Policy ${iam_policy_arn}" + done + + unique_iam_role_names=$(printf "%s\n" "${iam_role_names[@]}" | sort -u) + for role_name in ${unique_iam_role_names}; do + # confirm the role is indeed for the project + role_project_tag=$(aws iam list-role-tags \ + --role-name "${role_name}" \ + --query "tags[?key=='project'].value" \ + --output text 2>/dev/null || echo "") + + if [[ "${role_project_tag}" == "${project}" ]]; then + echo "Deleting IAM Role: ${role_name}" + delete-iam-role "${role_name}" || echo "Failed to delete IAM Role ${role_name}" + fi + done + echo "Cleanup complete for project: ${project}" echo "" } diff --git a/template-only-bin/delete-iam-role b/template-only-bin/delete-iam-role new file mode 100755 index 00000000..3a23c001 --- /dev/null +++ b/template-only-bin/delete-iam-role @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +role_name=$1 + +attached_policy_arns=$(aws iam list-attached-role-policies --role-name "${role_name}" --query 'AttachedPolicies[*].PolicyArn' --output text) + +for attached_policy_arn in ${attached_policy_arns}; do + aws iam detach-role-policy --role-name "${role_name}" --policy-arn "${attached_policy_arn}" +done + +inline_policy_names=$(aws iam list-role-policies --role-name "${role_name}" --query 'PolicyNames[*]' --output text) + +for inline_policy_name in ${inline_policy_names}; do + aws iam delete-role-policy --role-name "${role_name}" --policy-name "${inline_policy_name}" +done + +aws iam delete-role --role-name "${role_name}" diff --git a/template-only-bin/empty-s3-bucket b/template-only-bin/empty-s3-bucket new file mode 100755 index 00000000..ad017a7f --- /dev/null +++ b/template-only-bin/empty-s3-bucket @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Derived from https://stackoverflow.com/a/61123579 +set -euo pipefail + +bucket_name=${1} +aws_region=${2:-us-east-1} + +aws s3 rm "s3://${bucket_name}" --recursive --region "${aws_region}" 2>/dev/null || echo "Bucket already empty or inaccessible" + +# shellcheck disable=SC2016 +num_objects=$(aws s3api list-object-versions --bucket "${bucket_name}" --region "${aws_region}" --query='length(Versions[*] || `[]` )' | awk '{ print $1 }') +echo "'${num_objects}' objects to remove" +if [[ "${num_objects}" != "0" ]]; then + start=$SECONDS + while [[ $num_objects -gt 0 ]] + do + # shellcheck disable=SC2016 + aws s3api delete-objects --bucket "${bucket_name}" --delete "$(aws s3api list-object-versions --bucket "${bucket_name}" --region "${aws_region}" --max-items 500 --query='{Objects: Versions[0:500].{Key:Key,VersionId:VersionId}}')" --query 'length(Deleted[*] || `[]` )' > /dev/null + num_objects=$((num_objects > 500 ? num_objects - 500 : 0)) + echo "Removed batch of Objects... Remaining : ${num_objects} ($(( SECONDS - start ))s)" + done +fi + +# shellcheck disable=SC2016 +num_objects=$(aws s3api list-object-versions --bucket "${bucket_name}" --region "${aws_region}" --query='length(DeleteMarkers[*] || `[]` )' | awk '{ print $1 }') +echo "'${num_objects}' markers to remove" +if [[ "$num_objects" != "0" ]]; then + start=$SECONDS + while [[ $num_objects -gt 0 ]] + do + # shellcheck disable=SC2016 + aws s3api delete-objects --bucket "${bucket_name}" --delete "$(aws s3api list-object-versions --bucket "${bucket_name}" --region "${aws_region}" --max-items 500 --query='{Objects: DeleteMarkers[0:500].{Key:Key,VersionId:VersionId}}')" --query 'length(Deleted[*] || `[]` )' > /dev/null + num_objects=$((num_objects > 500 ? num_objects - 500 : 0)) + echo "Removed batch of Markers... Remaining : ${num_objects} (took $(( SECONDS - start ))s)" + done +fi diff --git a/template-only-bin/list-iam-roles-with-tag b/template-only-bin/list-iam-roles-with-tag new file mode 100755 index 00000000..de88d6fc --- /dev/null +++ b/template-only-bin/list-iam-roles-with-tag @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +tag_key=$1 +tag_value=$2 + +role_names=$(aws iam list-roles --query 'Roles[*].RoleName' --output text) + +for role_name in ${role_names}; do + tags=$(aws iam list-role-tags --role-name "${role_name}" --query "Tags[?Key=='${tag_key}' && Value=='${tag_value}']" --output text) + + if [ -n "$tags" ]; then + echo "${role_name}" + fi +done