Repository navigation
Reconcile Preview Environments #177
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Reconcile Preview Environments | |
| # Event-driven teardown genuinely misses cleanups: a queued destroy can be evicted from the shared | |
| # per-PR concurrency group by a newer run, a destroy can fail on a throttle, and a long-lived | |
| # labelled PR sits forever. This sweep is the backstop. | |
| on: | |
| schedule: | |
| - cron: "0 */6 * * *" | |
| workflow_dispatch: | |
| # One sweep at a time. What protects a live preview is the re-confirmation before each delete. | |
| concurrency: | |
| group: preview-reconcile | |
| cancel-in-progress: false | |
| env: | |
| AWS_REGION: eu-west-1 | |
| TERRAFORM_VERSION: 1.16.0 | |
| CDN_DIR: terraform/cdn | |
| CDN_BACKEND_CONFIG: ../environments/dev/backend.config | |
| jobs: | |
| reconcile: | |
| name: Reap orphaned previews | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| pull-requests: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 | |
| with: | |
| aws-region: ${{ env.AWS_REGION }} | |
| role-to-assume: ${{ secrets.AWS_ACC_ROLE }} | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 | |
| with: | |
| terraform_version: ${{ env.TERRAFORM_VERSION }} | |
| terraform_wrapper: false | |
| - name: Read the CDN outputs | |
| id: cdn | |
| working-directory: ${{ env.CDN_DIR }} | |
| run: | | |
| set -euo pipefail | |
| terraform init -input=false -backend-config="${CDN_BACKEND_CONFIG}" | |
| terraform workspace select dev | |
| { | |
| echo "bucket=$(terraform output -raw bucket_name)" | |
| echo "kvs_arn=$(terraform output -raw key_value_store_arn)" | |
| } >> "$GITHUB_OUTPUT" | |
| # The store is the source of truth: a key is the only thing that makes a preview host answer. | |
| - name: Sweep pr-* keys | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| REPO: ${{ github.repository }} | |
| BUCKET: ${{ steps.cdn.outputs.bucket }} | |
| KVS_ARN: ${{ steps.cdn.outputs.kvs_arn }} | |
| run: | | |
| # No `set -e`: one stuck preview must not starve the rest. | |
| set -uo pipefail | |
| list_previews() { | |
| aws cloudfront-keyvaluestore list-keys --kvs-arn "$KVS_ARN" --no-cli-pager \ | |
| --query 'Items[].Key' --output text | tr '\t' '\n' | grep -E '^pr-[0-9]+\.' || true | |
| } | |
| # Open and still labelled means keep; closed, merged or unlabelled is an orphan. | |
| is_orphan() { | |
| local pr="$1" json state labelled | |
| if ! json=$(gh api "repos/${REPO}/pulls/${pr}" 2>/dev/null); then | |
| return 2 | |
| fi | |
| state=$(jq -r '.state' <<<"$json") | |
| labelled=$(jq -r '[.labels[].name] | index("create-preview") != null' <<<"$json") | |
| [ "$state" = "open" ] && [ "$labelled" = "true" ] && return 1 | |
| return 0 | |
| } | |
| if ! hosts=$(list_previews); then | |
| echo "::error::Could not list the routing store." | |
| exit 1 | |
| fi | |
| if [ -z "$hosts" ]; then | |
| echo "No preview keys." | |
| exit 0 | |
| fi | |
| failed=0 | |
| for host in $hosts; do | |
| pr="${host%%.*}" | |
| pr="${pr#pr-}" | |
| is_orphan "$pr" | |
| case $? in | |
| 1) echo "${host}: PR #${pr} is open and labelled — keeping."; continue ;; | |
| 2) echo "::warning::Could not query PR #${pr}; leaving ${host} for the next sweep." ; continue ;; | |
| esac | |
| # Re-confirm against the store as it is now: a deploy may have rewritten this key, or a | |
| # teardown removed it, since the sweep started. | |
| if ! list_previews | grep -qx "$host"; then | |
| echo "${host}: already gone." | |
| continue | |
| fi | |
| echo "::group::Reaping ${host}" | |
| etag=$(aws cloudfront-keyvaluestore describe-key-value-store --kvs-arn "$KVS_ARN" \ | |
| --no-cli-pager --query ETag --output text) | |
| if aws cloudfront-keyvaluestore delete-key --kvs-arn "$KVS_ARN" --if-match "$etag" \ | |
| --key "$host" --no-cli-pager >/dev/null \ | |
| && aws s3 rm "s3://${BUCKET}/pr-${pr}/" --recursive --only-show-errors; then | |
| echo "${host}: reaped." | |
| else | |
| echo "::warning::Failed to reap ${host}; it will be retried next sweep." | |
| failed=1 | |
| fi | |
| echo "::endgroup::" | |
| done | |
| exit "$failed" |