orphan-sweep #1079
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Hourly sweeper for leaked runner VMs (Hetzner + AWS EC2) and offline | |
| # GHA runners. Companion to builds.yml; catches orphans from control-plane | |
| # crashes, cancelled workflows, and probe-step leaks. | |
| # | |
| # PS-11179 (2026-05-28): EC2 sweep added for the AWS fallback path. | |
| name: orphan-sweep | |
| on: | |
| schedule: | |
| - cron: '23 * * * *' # hourly, off the top of hour | |
| workflow_dispatch: | |
| inputs: | |
| dry_run: | |
| description: 'Dry-run: list what would be deleted, do nothing' | |
| type: boolean | |
| default: false | |
| max_age_hours: | |
| description: 'Orphan age threshold (hours); default 6h; lower at your own risk (build-arm64 has timeout-minutes: 240)' | |
| type: string | |
| default: '6' | |
| # Workflow-level permissions are empty. The sweep-ec2 job opts in to | |
| # id-token: write for OIDC -> AWS STS. Other jobs need nothing. | |
| permissions: {} | |
| concurrency: | |
| group: orphan-sweep | |
| cancel-in-progress: false # let prior sweep finish; serialize with next | |
| jobs: | |
| sweep: | |
| runs-on: ubuntu-latest | |
| permissions: {} | |
| env: | |
| HCLOUD_TOKEN: ${{ secrets.GHA_RUNNER_HCLOUD_TOKEN }} | |
| GH_TOKEN: ${{ secrets.GHA_RUNNER_PAT }} | |
| DRY_RUN: ${{ inputs.dry_run || 'false' }} | |
| MAX_AGE_HOURS: ${{ inputs.max_age_hours || '6' }} | |
| steps: | |
| - name: Sweep Hetzner orphans | |
| run: | | |
| set -euo pipefail | |
| : "${HCLOUD_TOKEN:?secret HCLOUD_TOKEN required}" | |
| NOW=$(date -u +%s) | |
| MAX_AGE_SECONDS=$(( MAX_AGE_HOURS * 3600 )) | |
| echo "Threshold: ${MAX_AGE_HOURS}h (${MAX_AGE_SECONDS}s); dry_run=${DRY_RUN}" | |
| # Page through all servers (max 50/page). Servers we own start with ps-arm64- or cap-probe-. | |
| PAGE=1 | |
| : > /tmp/candidates.tsv | |
| while :; do | |
| RESP=$(curl -fsS -H "Authorization: Bearer $HCLOUD_TOKEN" \ | |
| "https://api.hetzner.cloud/v1/servers?per_page=50&page=${PAGE}") | |
| echo "$RESP" \ | |
| | jq -r --arg now "$NOW" --arg age "$MAX_AGE_SECONDS" ' | |
| .servers[] | |
| | select(.name | test("^(ps-arm64-|cap-probe-)")) | |
| | select(($now|tonumber) - (.created | fromdateiso8601) > ($age|tonumber)) | |
| | [.id, .name, .created] | @tsv' \ | |
| >> /tmp/candidates.tsv | |
| LAST=$(echo "$RESP" | jq -r '.meta.pagination.last_page // 1') | |
| [ "$PAGE" -ge "$LAST" ] && break | |
| PAGE=$((PAGE+1)) | |
| done | |
| TOTAL=$(wc -l < /tmp/candidates.tsv) | |
| DELETED=0 | |
| FAILED=0 | |
| if [ "$TOTAL" -gt 0 ]; then | |
| echo "Candidates (>${MAX_AGE_HOURS}h old, matching prefix):" | |
| cat /tmp/candidates.tsv | |
| while IFS=$'\t' read -r id name created; do | |
| if [ "$DRY_RUN" = "true" ]; then | |
| echo "::notice::[dry-run] would delete $name (id=$id, created=$created)" | |
| else | |
| echo "::warning::Deleting orphan: $name (id=$id, created=$created)" | |
| if curl -fsS -X DELETE -H "Authorization: Bearer $HCLOUD_TOKEN" \ | |
| "https://api.hetzner.cloud/v1/servers/$id" >/dev/null; then | |
| DELETED=$((DELETED+1)) | |
| else | |
| echo "::error::Failed to delete server $id" | |
| FAILED=$((FAILED+1)) | |
| fi | |
| fi | |
| done < /tmp/candidates.tsv | |
| fi | |
| { | |
| echo "### Hetzner sweep" | |
| echo "" | |
| echo "| metric | value |" | |
| echo "|---|---|" | |
| echo "| candidates | ${TOTAL} |" | |
| echo "| deleted | ${DELETED} |" | |
| echo "| failed | ${FAILED} |" | |
| echo "| threshold | ${MAX_AGE_HOURS}h |" | |
| echo "| dry_run | ${DRY_RUN} |" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Sweep offline GHA runners | |
| if: env.GH_TOKEN != '' | |
| run: | | |
| set -euo pipefail | |
| UNREGISTERED=0 | |
| FAILED=0 | |
| # offline runners with our naming prefix | |
| mapfile -t LINES < <( | |
| curl -fsS -H "Authorization: Bearer $GH_TOKEN" \ | |
| "https://api.github.com/repos/${{ github.repository }}/actions/runners?per_page=100" \ | |
| | jq -r '.runners[] | select(.status=="offline") | select(.name | test("^ps-arm64-")) | "\(.id)\t\(.name)"' | |
| ) | |
| for line in "${LINES[@]}"; do | |
| [ -z "$line" ] && continue | |
| id="${line%%$'\t'*}" | |
| name="${line#*$'\t'}" | |
| if [ "$DRY_RUN" = "true" ]; then | |
| echo "::notice::[dry-run] would unregister offline runner: $name (id=$id)" | |
| else | |
| echo "::warning::Unregistering offline runner: $name (id=$id)" | |
| if curl -fsS -X DELETE -H "Authorization: Bearer $GH_TOKEN" \ | |
| "https://api.github.com/repos/${{ github.repository }}/actions/runners/$id" >/dev/null; then | |
| UNREGISTERED=$((UNREGISTERED+1)) | |
| else | |
| FAILED=$((FAILED+1)) | |
| fi | |
| fi | |
| done | |
| { | |
| echo "" | |
| echo "### GHA offline runner sweep" | |
| echo "" | |
| echo "| metric | value |" | |
| echo "|---|---|" | |
| echo "| unregistered | ${UNREGISTERED} |" | |
| echo "| failed | ${FAILED} |" | |
| echo "| candidates | ${#LINES[@]} |" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| # PS-11179: EC2 orphan sweep for the AWS fallback path. | |
| # Mirrors the Hetzner sweep cadence + threshold: hourly, default 6h age. | |
| # OIDC -> STS via the same AWS_ROLE_ARN used by builds.yml. Region pinned | |
| # to eu-central-1 (matches create-runner-aws). | |
| sweep-ec2: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| env: | |
| DRY_RUN: ${{ inputs.dry_run || 'false' }} | |
| MAX_AGE_HOURS: ${{ inputs.max_age_hours || '6' }} | |
| AWS_REGION: eu-central-1 | |
| steps: | |
| - name: Configure AWS credentials via OIDC | |
| uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| role-session-name: orphan-sweep-${{ github.run_id }}-${{ github.run_attempt }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Sweep EC2 orphans | |
| run: | | |
| set -euo pipefail | |
| NOW=$(date -u +%s) | |
| MAX_AGE_SECONDS=$(( MAX_AGE_HOURS * 3600 )) | |
| echo "Threshold: ${MAX_AGE_HOURS}h (${MAX_AGE_SECONDS}s); dry_run=${DRY_RUN}; region=${AWS_REGION}" | |
| # Find every instance we own that is still alive. We tag-key on | |
| # iit-billing-tag + github_repository so we never accidentally | |
| # reach into another workload's instances. | |
| # The --query string uses JMESPath; the backticks around `Name` | |
| # are JMESPath literal syntax, not shell command substitution, | |
| # so single-quoting is required (shellcheck SC2016 noise). | |
| # shellcheck disable=SC2016 | |
| aws ec2 describe-instances \ | |
| --region "$AWS_REGION" \ | |
| --filters \ | |
| "Name=tag:iit-billing-tag,Values=percona-server-gha-fallback" \ | |
| "Name=tag:github_repository,Values=${{ github.repository }}" \ | |
| "Name=instance-state-name,Values=pending,running,stopping,stopped" \ | |
| --query 'Reservations[].Instances[].[InstanceId,LaunchTime,State.Name,Tags[?Key==`Name`]|[0].Value]' \ | |
| --output text > /tmp/ec2-instances.tsv || true | |
| TOTAL=$(wc -l < /tmp/ec2-instances.tsv | tr -d ' ') | |
| DELETED=0 | |
| FAILED=0 | |
| SKIPPED=0 | |
| : > /tmp/ec2-candidates.tsv | |
| if [ "${TOTAL:-0}" -gt 0 ]; then | |
| while IFS=$'\t' read -r id launch state name; do | |
| [ -z "$id" ] && continue | |
| # LaunchTime is RFC3339 (e.g. 2026-05-28T12:34:56.000Z); strip | |
| # millis and trailing Z so `date -d` parses on Ubuntu 24.04. | |
| launch_clean=$(echo "$launch" | sed -E 's/\.[0-9]+Z?$/Z/') | |
| launch_epoch=$(date -u -d "$launch_clean" +%s 2>/dev/null || echo "") | |
| if [ -z "$launch_epoch" ]; then | |
| echo "::warning::Skipping $id; could not parse LaunchTime=$launch" | |
| SKIPPED=$((SKIPPED+1)) | |
| continue | |
| fi | |
| age=$(( NOW - launch_epoch )) | |
| if [ "$age" -lt "$MAX_AGE_SECONDS" ]; then | |
| continue | |
| fi | |
| echo "$id $name $state $launch $age" >> /tmp/ec2-candidates.tsv | |
| done < /tmp/ec2-instances.tsv | |
| fi | |
| CANDIDATES=$(wc -l < /tmp/ec2-candidates.tsv | tr -d ' ') | |
| if [ "${CANDIDATES:-0}" -gt 0 ]; then | |
| echo "Candidates (>${MAX_AGE_HOURS}h old, matching tags):" | |
| cat /tmp/ec2-candidates.tsv | |
| while IFS=$'\t' read -r id name state launch age; do | |
| if [ "$DRY_RUN" = "true" ]; then | |
| echo "::notice::[dry-run] would terminate $name ($id; state=$state; age=${age}s)" | |
| else | |
| echo "::warning::Terminating orphan EC2 instance: $name ($id; state=$state; age=${age}s)" | |
| if aws ec2 terminate-instances \ | |
| --region "$AWS_REGION" \ | |
| --instance-ids "$id" >/dev/null 2>&1; then | |
| DELETED=$((DELETED+1)) | |
| else | |
| echo "::error::Failed to terminate $id" | |
| FAILED=$((FAILED+1)) | |
| fi | |
| fi | |
| done < /tmp/ec2-candidates.tsv | |
| fi | |
| { | |
| echo "" | |
| echo "### EC2 sweep (${AWS_REGION})" | |
| echo "" | |
| echo "| metric | value |" | |
| echo "|---|---|" | |
| echo "| live instances | ${TOTAL} |" | |
| echo "| candidates | ${CANDIDATES} |" | |
| echo "| terminated | ${DELETED} |" | |
| echo "| failed | ${FAILED} |" | |
| echo "| skipped (parse) | ${SKIPPED} |" | |
| echo "| threshold | ${MAX_AGE_HOURS}h |" | |
| echo "| dry_run | ${DRY_RUN} |" | |
| } >> "$GITHUB_STEP_SUMMARY" |