Cleanup Cloud Artifacts #149
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cleanup Cloud Artifacts | |
| on: | |
| schedule: | |
| - cron: "0 1 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| dry_run: | |
| type: boolean | |
| default: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run && 'true' || 'false' }} | |
| jobs: | |
| cleanup-aws: | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.ref != 'refs/heads/master' && 'cloud-test' || null }} | |
| concurrency: | |
| group: cloud-aws | |
| cancel-in-progress: false | |
| queue: max | |
| steps: | |
| - name: Configure AWS | |
| uses: aws-actions/configure-aws-credentials@cbe3b392738ccf3f987d68400dafcf4b0624a56c # v6.2.4 | |
| with: | |
| aws-region: us-west-2 | |
| role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME }} | |
| - name: Delete old and abandoned resources | |
| run: | | |
| set -euo pipefail | |
| delete_aws_images() { | |
| region="$1" | |
| description="$2" | |
| image_ids="$3" | |
| if [ -z "${image_ids}" ]; then | |
| echo "No ${description} to delete in ${region}." | |
| return | |
| fi | |
| while read -r image_id; do | |
| if [ "${DRY_RUN}" = "true" ]; then | |
| echo "Would delete ${description} in ${region}: ${image_id}" | |
| continue | |
| fi | |
| echo "Deleting ${description} in ${region}: ${image_id}" | |
| mapfile -t snapshot_ids < <(aws ec2 describe-images \ | |
| --region "${region}" \ | |
| --image-ids "${image_id}" \ | |
| --query 'Images[].BlockDeviceMappings[].Ebs.SnapshotId' \ | |
| --output text \ | |
| | tr '\t' '\n' \ | |
| | sed '/^$/d') | |
| aws ec2 deregister-image --region "${region}" --image-id "${image_id}" | |
| for snapshot_id in "${snapshot_ids[@]}"; do | |
| echo "Deleting AWS snapshot in ${region}: ${snapshot_id}" | |
| aws ec2 delete-snapshot --region "${region}" --snapshot-id "${snapshot_id}" | |
| done | |
| done <<< "${image_ids}" | |
| } | |
| delete_aws_images "us-west-2" "old AWS AMIs" "$(aws ec2 describe-images \ | |
| --region us-west-2 \ | |
| --owners self \ | |
| --filters "Name=tag:ManagedBy,Values=packer" "Name=tag:Name,Values=quic-perf-runner" \ | |
| --query 'sort_by(Images, &CreationDate)[:-1].ImageId' \ | |
| --output text \ | |
| | tr '\t' '\n' \ | |
| | sed '/^$/d')" | |
| for region in $(aws ec2 describe-regions --query 'Regions[].RegionName' --output text); do | |
| mapfile -t instance_ids < <(aws ec2 describe-instances \ | |
| --region "${region}" \ | |
| --filters "Name=tag:ManagedBy,Values=perf-dashboard" \ | |
| --output json \ | |
| | jq -r '.Reservations[].Instances[] | select(.State.Name != "terminated") | select(any(.Tags[]?; .Key == "RunId" and (.Value | startswith("quic-perf-aws-")))) | .InstanceId') | |
| for instance_id in "${instance_ids[@]}"; do | |
| if [ "${DRY_RUN}" = "true" ]; then | |
| echo "Would terminate AWS instance in ${region}: ${instance_id}" | |
| else | |
| echo "Terminating AWS instance in ${region}: ${instance_id}" | |
| fi | |
| done | |
| if [ "${DRY_RUN}" != "true" ] && [ "${#instance_ids[@]}" -gt 0 ]; then | |
| aws ec2 terminate-instances --region "${region}" --instance-ids "${instance_ids[@]}" | |
| aws ec2 wait instance-terminated --region "${region}" --instance-ids "${instance_ids[@]}" | |
| fi | |
| mapfile -t group_ids < <(aws ec2 describe-security-groups \ | |
| --region "${region}" \ | |
| --filters "Name=tag:ManagedBy,Values=perf-dashboard" \ | |
| --output json \ | |
| | jq -r '.SecurityGroups[] | select(any(.Tags[]?; .Key == "RunId" and (.Value | startswith("quic-perf-aws-")))) | .GroupId') | |
| for group_id in "${group_ids[@]}"; do | |
| if [ "${DRY_RUN}" = "true" ]; then | |
| echo "Would delete AWS security group in ${region}: ${group_id}" | |
| else | |
| echo "Deleting AWS security group in ${region}: ${group_id}" | |
| aws ec2 delete-security-group --region "${region}" --group-id "${group_id}" | |
| fi | |
| done | |
| delete_aws_images "${region}" "AWS benchmark AMI copies" "$(aws ec2 describe-images \ | |
| --region "${region}" \ | |
| --owners self \ | |
| --filters "Name=tag:ManagedBy,Values=perf-dashboard" "Name=tag:Name,Values=quic-perf-runner" \ | |
| --output json \ | |
| | jq -r '.Images[] | select(any(.Tags[]?; .Key == "RunId" and (.Value | startswith("quic-perf-aws-")))) | .ImageId')" | |
| done | |
| cleanup-gcp: | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.ref != 'refs/heads/master' && 'cloud-test' || null }} | |
| concurrency: | |
| group: cloud-gcp | |
| cancel-in-progress: false | |
| queue: max | |
| env: | |
| CLOUDSDK_CORE_PROJECT: ${{ vars.GCP_PROJECT_ID }} | |
| steps: | |
| - name: Authenticate to GCP | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 | |
| with: | |
| workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ vars.GCP_PACKER_SERVICE_ACCOUNT }} | |
| - name: Delete abandoned benchmark instances | |
| run: | | |
| set -euo pipefail | |
| instances="$(gcloud compute instances list \ | |
| --filter="labels.managed_by=perf-dashboard" \ | |
| --format=json \ | |
| | jq -r '.[] | select((.labels.run_id // "") | startswith("quic-perf-gcp-")) | [(.zone | split("/")[-1]), .name] | @tsv')" | |
| if [ -z "${instances}" ]; then | |
| echo "No abandoned GCP instances to delete." | |
| exit 0 | |
| fi | |
| while read -r zone instance_name; do | |
| if [ "${DRY_RUN}" = "true" ]; then | |
| echo "Would delete GCP instance in ${zone}: ${instance_name}" | |
| else | |
| echo "Deleting GCP instance in ${zone}: ${instance_name}" | |
| gcloud compute instances delete "${instance_name}" --zone "${zone}" --quiet | |
| fi | |
| done <<< "${instances}" | |
| - name: Keep only the newest image, delete the rest | |
| run: | | |
| set -euo pipefail | |
| image_names="$(gcloud compute images list \ | |
| --filter="family=quic-perf-runner" \ | |
| --sort-by="~creationTimestamp" \ | |
| --format="value(name)" \ | |
| | tail -n +2 \ | |
| | sed '/^$/d')" | |
| if [ -z "${image_names}" ]; then | |
| echo "No old GCP images to delete." | |
| exit 0 | |
| fi | |
| while read -r image_name; do | |
| if [ "${DRY_RUN}" = "true" ]; then | |
| echo "Would delete GCP image: ${image_name}" | |
| else | |
| echo "Deleting GCP image: ${image_name}" | |
| gcloud compute images delete "${image_name}" --quiet | |
| fi | |
| done <<< "${image_names}" | |
| cleanup-azure: | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.ref != 'refs/heads/master' && 'cloud-test' || null }} | |
| concurrency: | |
| group: cloud-azure | |
| cancel-in-progress: false | |
| queue: max | |
| steps: | |
| - name: Authenticate to Azure | |
| uses: azure/login@7ddb5af1ef8758cf1353cf3b42f940aee27ba21c # v3.0.2 | |
| with: | |
| client-id: ${{ vars.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ vars.AZURE_TENANT_ID }} | |
| subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} | |
| - name: Delete old, temporary, and abandoned resources | |
| env: | |
| AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} | |
| run: | | |
| set -euo pipefail | |
| delete_resources() { | |
| description="$1" | |
| resource_ids="$2" | |
| if [ -z "${resource_ids}" ]; then | |
| echo "No ${description} to delete." | |
| return | |
| fi | |
| while read -r resource_id; do | |
| if [ "${DRY_RUN}" = "true" ]; then | |
| echo "Would delete ${description}: ${resource_id}" | |
| else | |
| echo "Deleting ${description}: ${resource_id}" | |
| az resource delete --ids "${resource_id}" | |
| fi | |
| done <<< "${resource_ids}" | |
| } | |
| delete_resources "Azure temporary Packer resources" "$(az resource list \ | |
| --resource-group "${AZURE_RESOURCE_GROUP}" \ | |
| --query "[?tags.ManagedBy=='packer' && tags.PackerLifecycle=='temporary'].id" \ | |
| --output tsv)" | |
| delete_resources "old Azure images" "$(az image list \ | |
| --resource-group "${AZURE_RESOURCE_GROUP}" \ | |
| --query "sort_by([?tags.ManagedBy=='packer' && starts_with(name, 'quic-perf-runner-') && not_null(tags.PackerLifecycle, 'retained')!='temporary'], &name)[:-1].id" \ | |
| --output tsv)" | |
| vm_resources="$(az vm list \ | |
| --resource-group "${AZURE_RESOURCE_GROUP}" \ | |
| --query "[?tags.ManagedBy=='perf-dashboard' && starts_with(not_null(tags.RunId, ''), 'quic-perf-azure-')].[id,storageProfile.osDisk.managedDisk.id]" \ | |
| --output tsv)" | |
| delete_resources "Azure benchmark VMs" "$(printf '%s\n' "${vm_resources}" | cut -f1 | sed '/^$/d')" | |
| delete_resources "Azure benchmark VM disks" "$(printf '%s\n' "${vm_resources}" | cut -f2 | sed '/^$/d')" | |
| delete_resources "Azure benchmark orphan disks" "$(az disk list \ | |
| --resource-group "${AZURE_RESOURCE_GROUP}" \ | |
| --query "[?starts_with(not_null(tags.RunId, ''), 'quic-perf-azure-') || starts_with(name, 'quic-perf-azure-')].id" \ | |
| --output tsv)" | |
| for resource_type in \ | |
| Microsoft.Network/networkInterfaces \ | |
| Microsoft.Network/publicIPAddresses \ | |
| Microsoft.Network/networkSecurityGroups \ | |
| Microsoft.Network/virtualNetworks \ | |
| Microsoft.Compute/galleries/images/versions \ | |
| Microsoft.Compute/galleries/images \ | |
| Microsoft.Compute/galleries; do | |
| delete_resources "Azure ${resource_type}" "$(az resource list \ | |
| --resource-group "${AZURE_RESOURCE_GROUP}" \ | |
| --resource-type "${resource_type}" \ | |
| --query "[?tags.ManagedBy=='perf-dashboard' && starts_with(not_null(tags.RunId, ''), 'quic-perf-azure-')].id" \ | |
| --output tsv)" | |
| done |