From cfc8d2c40644f018aee30808fda6d2a2b9270221 Mon Sep 17 00:00:00 2001 From: Sean Thomas Date: Mon, 11 May 2026 16:35:52 -0400 Subject: [PATCH 1/8] Add PR environment workflows ported from AWS template Closes #7. Adds the workflow + script + docs scaffolding to spin up a temporary Container Apps environment for each open pull request, keep it in sync as new commits land, and tear it down on close/merge. Ported directly from navapbc/template-infra with Azure-specific adaptations. - Reusable callable workflows: pr-environment-checks.yml, pr-environment-destroy.yml - Jinja app wrappers gated on app_has_dev_env_setup, matching the existing ci-{{app_name}}-infra-service.yml.jinja pattern - scan-orphaned-environments.yml runs daily, fails on orphaned PR workspaces (p-) or stale Terratest workspaces (>24h old). Slack alerting on failure deferred to #50 - bin/{update,destroy}-pr-environment use ${env}.azurerm.tfbackend instead of s3.tfbackend; update-pr-environment polls /health for service-stable since Container Apps has no `aws ecs wait` analog - bin/{orphaned-pr,stale-test}-environments scan workspaces; bin/util.sh provides get_app_names + base62_decode helpers - docs/infra/pull-request-environments.md ported from AWS, trimmed for Azure (no Cognito section, links to existing workspace doc) --- ...app_name}}-pr-environment-checks.yml.jinja | 33 ++++++++ ...pp_name}}-pr-environment-destroy.yml.jinja | 30 +++++++ .github/workflows/pr-environment-checks.yml | 63 +++++++++++++++ .github/workflows/pr-environment-destroy.yml | 44 ++++++++++ .../workflows/scan-orphaned-environments.yml | 70 ++++++++++++++++ bin/destroy-pr-environment | 59 ++++++++++++++ bin/orphaned-pr-environments | 49 ++++++++++++ bin/stale-test-environments | 63 +++++++++++++++ bin/update-pr-environment | 80 +++++++++++++++++++ bin/util.sh | 25 ++++++ docs/infra/pull-request-environments.md | 66 +++++++++++++++ 11 files changed, 582 insertions(+) create mode 100644 .github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja create mode 100644 .github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja create mode 100644 .github/workflows/pr-environment-checks.yml create mode 100644 .github/workflows/pr-environment-destroy.yml create mode 100644 .github/workflows/scan-orphaned-environments.yml create mode 100755 bin/destroy-pr-environment create mode 100755 bin/orphaned-pr-environments create mode 100755 bin/stale-test-environments create mode 100755 bin/update-pr-environment create mode 100755 bin/util.sh create mode 100644 docs/infra/pull-request-environments.md diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja new file mode 100644 index 0000000..899ff19 --- /dev/null +++ b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja @@ -0,0 +1,33 @@ +name: CI {{ app_name }} PR Environment Checks +on: + workflow_dispatch: + inputs: + pr_number: + required: true + type: string + commit_hash: + required: true + type: string + {% if app_has_dev_env_setup %} + pull_request: + {% else %} + # !! Once you've set up the dev environment and are ready to enable PR + # environments, run: + # + # nava-platform infra update --answers-only --data app_has_dev_env_setup=true . + # + # to enable these lines. They are here as comments for context. + # + # pull_request: + {% endif %} + +jobs: + update: + name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise + uses: ./.github/workflows/pr-environment-checks.yml + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.state == 'open' + with: + app_name: "{{ app_name }}" + environment: "dev" + pr_number: ${{'{{'}} inputs.pr_number || github.event.number {{'}}'}} + commit_hash: ${{'{{'}} inputs.commit_hash || github.event.pull_request.head.sha {{'}}'}} diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja new file mode 100644 index 0000000..137c93e --- /dev/null +++ b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja @@ -0,0 +1,30 @@ +name: CI {{ app_name }} PR Environment Destroy +on: + workflow_dispatch: + inputs: + pr_number: + required: true + type: string + {% if app_has_dev_env_setup %} + pull_request_target: + types: [closed] + {% else %} + # !! Once you've set up the dev environment and are ready to enable PR + # environments, run: + # + # nava-platform infra update --answers-only --data app_has_dev_env_setup=true . + # + # to enable these lines. They are here as comments for context. + # + # pull_request: + # types: [closed] + {% endif %} + +jobs: + destroy: + name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise + uses: ./.github/workflows/pr-environment-destroy.yml + with: + app_name: "{{ app_name }}" + environment: "dev" + pr_number: ${{'{{'}} inputs.pr_number || github.event.number {{'}}'}} diff --git a/.github/workflows/pr-environment-checks.yml b/.github/workflows/pr-environment-checks.yml new file mode 100644 index 0000000..1771bf8 --- /dev/null +++ b/.github/workflows/pr-environment-checks.yml @@ -0,0 +1,63 @@ +name: PR Environment Update +run-name: Update PR Environment ${{ inputs.pr_number }} +on: + workflow_call: + inputs: + app_name: + required: true + type: string + environment: + required: true + type: string + pr_number: + required: true + type: string + commit_hash: + required: true + type: string + +concurrency: pr-environment-${{ inputs.app_name }}-${{ inputs.pr_number }} + +jobs: + build-and-publish: + name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise + uses: ./.github/workflows/build-and-publish.yml + with: + app_name: ${{ inputs.app_name }} + ref: ${{ inputs.commit_hash }} + + update: + name: Update environment + needs: [build-and-publish] + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + pull-requests: write # Needed to comment on PR + repository-projects: read # Workaround for GitHub CLI bug https://github.com/cli/cli/issues/6274 + + outputs: + service_endpoint: ${{ steps.update-environment.outputs.service_endpoint }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Terraform + uses: ./.github/actions/setup-terraform + + - name: Configure Azure credentials + uses: ./.github/actions/configure-azure-credentials + with: + app_name: ${{ inputs.app_name }} + environment: ${{ inputs.environment }} + + - name: Update environment + id: update-environment + run: | + ./bin/update-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" "${{ inputs.commit_hash }}" + service_endpoint=$(terraform -chdir="infra/${{ inputs.app_name }}/service" output -raw service_endpoint) + echo "service_endpoint=${service_endpoint}" + echo "service_endpoint=${service_endpoint}" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/pr-environment-destroy.yml b/.github/workflows/pr-environment-destroy.yml new file mode 100644 index 0000000..627e93a --- /dev/null +++ b/.github/workflows/pr-environment-destroy.yml @@ -0,0 +1,44 @@ +name: PR Environment Destroy +run-name: Destroy PR Environment ${{ inputs.pr_number }} +on: + workflow_call: + inputs: + app_name: + required: true + type: string + environment: + required: true + type: string + pr_number: + required: true + type: string + +concurrency: pr-environment-${{ inputs.app_name }}-${{ inputs.pr_number }} + +jobs: + destroy: + name: Destroy environment + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + pull-requests: write # Needed to comment on PR + repository-projects: read # Workaround for GitHub CLI bug https://github.com/cli/cli/issues/6274 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Terraform + uses: ./.github/actions/setup-terraform + + - name: Configure Azure credentials + uses: ./.github/actions/configure-azure-credentials + with: + app_name: ${{ inputs.app_name }} + environment: ${{ inputs.environment }} + + - name: Destroy environment + run: ./bin/destroy-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/scan-orphaned-environments.yml b/.github/workflows/scan-orphaned-environments.yml new file mode 100644 index 0000000..cc5a9ad --- /dev/null +++ b/.github/workflows/scan-orphaned-environments.yml @@ -0,0 +1,70 @@ +# This workflow scans for temporary environments that were not properly cleaned up +# This can happen if the PR environment destroy workflow failed or didn't run +# or if the temporary environments created by the infra service tests were not cleaned up +name: Scan orphaned environments + +on: + workflow_dispatch: + schedule: + # Run every day at 07:30 UTC (3:30am ET, 12:30am PT) after engineers are likely done with work + - cron: "30 7 * * *" + +jobs: + get-app-names: + name: Get app names + runs-on: ubuntu-latest + + outputs: + app_names: ${{ steps.get-app-names.outputs.app_names }} + + steps: + - uses: actions/checkout@v4 + + - name: Get app names + id: get-app-names + run: | + source bin/util.sh + app_names="$(get_app_names)" + # turn app_names into a json list using jq + app_names="$(echo "${app_names}" | jq -R -s -c 'split("\n")[:-1]')" + echo "App names retrieved: ${app_names}" + echo "app_names=${app_names}" >> "$GITHUB_OUTPUT" + shell: bash + + scan: + name: Scan + runs-on: ubuntu-latest + needs: get-app-names + + strategy: + fail-fast: false + matrix: + app_name: ${{ fromJson(needs.get-app-names.outputs.app_names) }} + scan_script: [orphaned-pr-environments, stale-test-environments] + + permissions: + contents: read + id-token: write + pull-requests: read + + steps: + - uses: actions/checkout@v4 + + - name: Set up Terraform + uses: ./.github/actions/setup-terraform + + - name: Configure Azure credentials + uses: ./.github/actions/configure-azure-credentials + with: + app_name: ${{ matrix.app_name }} + environment: dev + + - name: List PR workspaces + run: | + ./bin/${{ matrix.scan_script }} ${{ matrix.app_name }} + env: + GH_TOKEN: ${{ github.token }} + TF_IN_AUTOMATION: "true" + + # TODO(#50): Add a notify job that posts to Slack on failure. Until then, + # orphaned-environment alerts only show up in the Actions UI. diff --git a/bin/destroy-pr-environment b/bin/destroy-pr-environment new file mode 100755 index 0000000..1cdcac3 --- /dev/null +++ b/bin/destroy-pr-environment @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# Destroy the temporary environment that was created for the pull request. +# +# Positional parameters: +# app_name (required) – the name of subdirectory of /infra that holds the +# application's infrastructure code. +# environment - the name of the application environment (e.g. dev, staging, prod) +# pr_number - the pull request number in GitHub +# ----------------------------------------------------------------------------- +set -euo pipefail + +app_name="$1" +environment="$2" +pr_number="$3" + +workspace="p-${pr_number}" + +echo "::group::Initialize Terraform with backend for environment: ${environment}" +terraform -chdir="infra/${app_name}/service" init -backend-config="${environment}.azurerm.tfbackend" +echo "::endgroup::" + +echo "Select Terraform workspace: ${workspace}" +terraform -chdir="infra/${app_name}/service" workspace select "${workspace}" + +echo "::group::Destroy resources" +terraform -chdir="infra/${app_name}/service" destroy -var="environment_name=${environment}" -input=false -auto-approve +echo "::endgroup::" + +echo "Select default workspace" +terraform -chdir="infra/${app_name}/service" workspace select default + +echo "Delete workspace: ${workspace}" +terraform -chdir="infra/${app_name}/service" workspace delete "${workspace}" + +pr_info=$(cat < +## Preview environment for ${app_name} +♻️ Environment destroyed ♻️ + +EOF +) + +pr_body="$(gh pr view "${pr_number}" --json body | jq --raw-output .body)" + +# clean up older single-app section if present +if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*}" +fi + +if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*/$pr_info}" +else + pr_body="${pr_body}"$'\n\n'"${pr_info}" +fi + +echo "Update PR description with PR environment info" +echo "${pr_info}" +gh pr edit "${pr_number}" --body "${pr_body}" diff --git a/bin/orphaned-pr-environments b/bin/orphaned-pr-environments new file mode 100755 index 0000000..ce597eb --- /dev/null +++ b/bin/orphaned-pr-environments @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# This script checks for orphaned PR environments by listing all PR workspaces +# and checking if the associated PR is closed. If the PR is closed the +# resources in the workspace should have been destroyed and the workspace +# deleted, so existing workspaces for closed PRs are considered orphaned. +# ----------------------------------------------------------------------------- +set -euo pipefail + +GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY:-/dev/null} + +app_name="$1" + +echo "::group::Initialize Terraform" +echo terraform -chdir="infra/${app_name}/service" init -input=false -reconfigure -backend-config="dev.azurerm.tfbackend" +terraform -chdir="infra/${app_name}/service" init -input=false -reconfigure -backend-config="dev.azurerm.tfbackend" +echo "::endgroup::" + +echo "::group::List PRs with PR environments" +echo terraform -chdir="infra/${app_name}/service" workspace list +workspaces="$(terraform -chdir="infra/${app_name}/service" workspace list)" +# grep will exit with code `1` if there's no match, so ignore that for when +# there are no PR workspaces for the application +pr_nums="$(echo "${workspaces}" | { grep -o 'p-[0-9]\+$' || test $? = 1; } | sed 's/p-//')" +echo "PRs" +echo "${pr_nums}" +echo "::endgroup::" + +echo "::group::Check status of each PR" +closed_prs=() +for pr_num in $pr_nums; do + pr_status="$(gh pr view "$pr_num" --json state --jq ".state")" + echo "PR ${pr_num}: ${pr_status}" + + if [ "$pr_status" == "CLOSED" ] || [ "$pr_status" == "MERGED" ]; then + closed_prs+=("$pr_num") + fi +done +echo "::endgroup::" + +# if closed_prs is not empty exit with 1 otherwise exit with 0 +if [ ${#closed_prs[@]} -gt 0 ]; then + echo "🧹 Found orphaned PR environments for the following PRs: **${closed_prs[*]}**" + echo "🧹 Found orphaned PR environments for the following PRs: **${closed_prs[*]}**" >> "${GITHUB_STEP_SUMMARY}" + exit 1 +fi + +echo "✅ No orphaned PR environments" +echo "✅ No orphaned PR environments" >> "${GITHUB_STEP_SUMMARY}" diff --git a/bin/stale-test-environments b/bin/stale-test-environments new file mode 100755 index 0000000..9bce536 --- /dev/null +++ b/bin/stale-test-environments @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# This script checks for orphaned environments created by infra service tests +# by checking if the environment is older than a certain threshold. +# ----------------------------------------------------------------------------- +set -euo pipefail + +source bin/util.sh + +GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY:-/dev/null} + +# 24 hours in seconds +MAX_ENVIRONMENT_AGE_MILLIS=86400 + +app_name="$1" + +echo "::group::Initialize Terraform" +echo terraform -chdir="infra/${app_name}/service" init -input=false -reconfigure -backend-config="dev.azurerm.tfbackend" +terraform -chdir="infra/${app_name}/service" init -input=false -reconfigure -backend-config="dev.azurerm.tfbackend" +echo "::endgroup::" + +echo "::group::List test IDs of test environments" +echo terraform -chdir="infra/${app_name}/service" workspace list +workspaces="$(terraform -chdir="infra/${app_name}/service" workspace list)" +# grep will exit with code `1` if there's no match, so ignore that for when +# there are no PR workspaces for the application +test_ids="$(echo "${workspaces}" | { grep -o 't-[A-Za-z0-9]\+' || test $? = 1; } | sed 's/t-//')" +echo "Test IDs" +echo "${test_ids}" +echo "::endgroup::" + +echo "::group::Check age of each test environment" +stale_tests=() +for test_id in $test_ids; do + # Base 62 decode the test ID to get the timestamp + test_timestamp="$(base62_decode "${test_id}")" + current_timestamp="$(date +%s)" + age="$((current_timestamp - test_timestamp))" + + if [ "${age}" -gt "${MAX_ENVIRONMENT_AGE_MILLIS}" ]; then + echo "stale ID: ${test_id} age: $((age / 3600)) hours started: $(date -r "${test_timestamp}")" + echo "stale ID: ${test_id} age: $((age / 3600)) hours started: $(date -r "${test_timestamp}")" >> "${GITHUB_STEP_SUMMARY}" + stale_tests+=("${test_id}") + fi + + # If age is less than 0, the test ID is invalid + if [ "${age}" -lt 0 ]; then + echo "invalid ID: ${test_id}" + echo "invalid ID: ${test_id}" >> "${GITHUB_STEP_SUMMARY}" + stale_tests+=("${test_id}") + fi +done +echo "::endgroup::" + +# if stale_tests is not empty exit with 1 otherwise exit with 0 +if [ ${#stale_tests[@]} -gt 0 ]; then + echo "🧹 Found stale test environments for the following test IDs: **${stale_tests[*]}**" + echo "🧹 Found stale test environments for the following test IDs: **${stale_tests[*]}**" >> "${GITHUB_STEP_SUMMARY}" + exit 1 +fi + +echo "✅ No stale test environments" +echo "✅ No stale test environments" >> "${GITHUB_STEP_SUMMARY}" diff --git a/bin/update-pr-environment b/bin/update-pr-environment new file mode 100755 index 0000000..29dbce3 --- /dev/null +++ b/bin/update-pr-environment @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# Create or update a temporary environment that will exist while a pull request +# is open. +# +# Positional parameters: +# app_name (required) – the name of subdirectory of /infra that holds the +# application's infrastructure code. +# environment - the name of the application environment (e.g. dev, staging, prod) +# pr_number - the pull request number in GitHub +# image_tag - the commit hash to deploy for the temporary environment +# ----------------------------------------------------------------------------- +set -euo pipefail + +app_name="$1" +environment="$2" +pr_number="$3" +image_tag="$4" + +workspace="p-${pr_number}" + +echo "::group::Initialize Terraform with backend for environment: ${environment}" +terraform -chdir="infra/${app_name}/service" init -backend-config="${environment}.azurerm.tfbackend" +echo "::endgroup::" + +echo "Select or create Terraform workspace: ${workspace}" +terraform -chdir="infra/${app_name}/service" workspace select -or-create "${workspace}" + +echo "::group::Apply changes to environment using image tag: ${image_tag}" +terraform -chdir="infra/${app_name}/service" apply -input=false -auto-approve -var="environment_name=${environment}" -var="image_tag=${image_tag}" +echo "::endgroup::" + +service_endpoint="$(terraform -chdir="infra/${app_name}/service" output -raw service_endpoint)" + +echo "::group::Wait for ${service_endpoint} to become healthy" +# Azure Container Apps has no clean CLI equivalent of `aws ecs wait +# services-stable`. Poll the service's /health endpoint until it returns 200 +# (or give up after ~5 minutes). This matches what infra/test/infra_test.go +# does for Terratest. +attempt=0 +max_attempts=60 +sleep_seconds=5 +until curl --silent --show-error --fail --max-time 10 "${service_endpoint}/health" > /dev/null; do + attempt=$((attempt + 1)) + if [ "${attempt}" -ge "${max_attempts}" ]; then + echo "Service did not become healthy within $((max_attempts * sleep_seconds)) seconds" + exit 1 + fi + echo "Attempt ${attempt}/${max_attempts}: service not healthy yet, retrying in ${sleep_seconds}s..." + sleep "${sleep_seconds}" +done +echo "Service is healthy" +echo "::endgroup::" + +pr_info=$(cat < +## Preview environment for ${app_name} +- Service endpoint: ${service_endpoint} +- Deployed commit: ${image_tag} + +EOF +) + +pr_body="$(gh pr view "${pr_number}" --json body | jq --raw-output .body)" + +# clean up older single-app section if present +if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*}" +fi + +# update or add the environment info +if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*/$pr_info}" +else + pr_body="${pr_body}"$'\n\n'"${pr_info}" +fi + +echo "Update PR description with PR environment info" +echo "${pr_info}" +gh pr edit "${pr_number}" --body "${pr_body}" diff --git a/bin/util.sh b/bin/util.sh new file mode 100755 index 0000000..f4540a1 --- /dev/null +++ b/bin/util.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Utility functions + +# Retrieve the names of the applications in the repo by listing the directories in the "infra" directory +# and filtering out the directories that are not applications. +# Returns: A list of application names. +function get_app_names() { + find "infra" -maxdepth 1 -type d -not -name "infra" -not -name "accounts" -not -name "modules" -not -name "networks" -not -name "project-config" -not -name "test" -exec basename {} \; +} + +# Base 62 decode a string. +# Returns: String as base 10 number. +function base62_decode() { + local digits="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + local s=$1 + local n=0 + + for ((i=0;i<${#s};i++)); do + c=${s:$i:1} + pos=${digits%%"$c"*} + n=$((n*62 + ${#pos})) + done + + echo $n +} diff --git a/docs/infra/pull-request-environments.md b/docs/infra/pull-request-environments.md new file mode 100644 index 0000000..5ca94d1 --- /dev/null +++ b/docs/infra/pull-request-environments.md @@ -0,0 +1,66 @@ +# Pull request environments + +A temporary environment is created for each pull request that stays up while the pull request is open. The endpoint for the pull request and the deployed commit are added to the pull request description, and updated when the environment is updated. Use cases for the temporary pull request environment include: + +- Allow other delivery stakeholders—including product managers, designers, and business owners—to review changes before being merged and deployed +- Enable automated end-to-end tests on the pull request +- Enable automated accessibility checks on the pull request +- Facilitate workspace creation for developing and testing service layer infrastructure changes + +## Lifecycle of pull request environments + +A pull request environment is created when a pull request is opened or reopened, and destroyed when the pull request is merged or closed. When new commits are pushed up to the pull request, the pull request environment is updated. + +## Shared database of pull request environments + +Pull request environments share the same database as the dev environment. This has the following benefits: + +- Enables testers to leverage existing test user accounts and accumulated test data +- Reduces the need for manual data seeding or migration scripts +- Better simulates production-like conditions where the application must handle pre-existing data, potentially revealing edge cases and integration issues that might not be apparent with a fresh database +- Reduces environment provisioning time significantly since creating and destroying database clusters typically takes 20-40 minutes + +## Limitations of shared resources in pull request environments + +Because PR environments share certain resources (e.g., the database) with the dev environment rather than provisioning their own, there are inherent limitations on what can be tested in a PR environment: + +- **Configuration changes to shared resources cannot be tested in a PR environment.** Changes to the database layer will not take effect until the PR is merged and deployed to dev. +- **Multiple PR environments share the same resource instance.** Changes made by one PR environment (e.g., data written to the database) may be visible to other PR environments sharing the same resource. + +For more on how Terraform workspaces are used to isolate temporary environments, see [Develop and Test Infrastructure in Isolation Using Workspaces](./develop-and-test-infrastructure-in-isolation-using-workspaces.md). + +## Isolate database migrations into separate pull requests + +Database migrations are not reflected in PR environments. In particular, PR environments share the same database with the dev environment, so database migrations that exist in the pull request are not run on the database to avoid impacting the dev environment. + +Therefore, isolate database changes in their own pull request and merge that pull request first before opening pull requests with application changes that depend on those database changes. This practice has the following benefits: + +- Enables PR environments to continue to be fully functional and testable when there are application changes that depend on database changes +- Enables database changes to be tested and deployed in isolation, which helps ensure that existing deployed application code is backwards compatible with new database changes + +This guidance is not strict. It is still okay to combine database migrations and application changes in a single pull request. However, when doing so, note that the PR environment may not be fully functional if the application changes rely on database migrations. + +Note also that this guidance pertains to pull requests, not local development. It is still okay and encouraged to develop database and application changes together during local development. + +## Implementing pull request environments for each application + +Pull request environments are created by GitHub Actions workflows. There are two reusable callable workflows that manage pull request environments: + +- [pr-environment-checks.yml](/.github/workflows/pr-environment-checks.yml) - creates or updates a temporary environment in a separate Terraform workspace for a given application and pull request +- [pr-environment-destroy.yml](/.github/workflows/pr-environment-destroy.yml) - destroys a temporary environment and workspace for a given application and pull request + +Using these reusable workflows, configure PR environments for each application with application-specific workflows: + +- `ci--pr-environment-checks.yml` + - Based on [ci-{{app_name}}-pr-environment-checks.yml](/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) +- `ci--pr-environment-destroy.yml` + - Based on [ci-{{app_name}}-pr-environment-destroy.yml](/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja) + +## Orphaned environment cleanup + +A scheduled workflow ([scan-orphaned-environments.yml](/.github/workflows/scan-orphaned-environments.yml)) runs daily and checks for two kinds of leftover environments: + +- **Orphaned PR environments** — Terraform workspaces named `p-` whose associated pull request is closed or merged. These indicate that the PR environment destroy workflow failed or didn't run. +- **Stale test environments** — Terraform workspaces named `t-` left behind by Terratest runs that exceeded the maximum environment age (24 hours). + +When orphans are found the scan job fails. Currently failures only surface in the GitHub Actions UI; Slack alerting is tracked in [#50](https://github.com/navapbc/template-infra-azure/issues/50). From 8c57aca4c3966b33e48cf9a362608a9b9c4e6ba1 Mon Sep 17 00:00:00 2001 From: Sean Thomas Date: Thu, 14 May 2026 18:23:10 -0400 Subject: [PATCH 2/8] Address PR #51 review findings - bin/stale-test-environments: replace BSD-only `date -r TIMESTAMP` with GNU-portable `date -d @TIMESTAMP` so the script works on ubuntu-latest runners when stale workspaces are present - bin/stale-test-environments: rename MAX_ENVIRONMENT_AGE_MILLIS to MAX_ENVIRONMENT_AGE_SECONDS to match the actual unit - pr-environment-checks.yml: drop the unreachable service_endpoint job output (e2e job was intentionally not ported) - bin/update-pr-environment: add curl --location and correct the worst- case timeout in the failure message (was counting only sleep time) - ci-{{app_name}}-pr-environment-destroy.yml.jinja: align the disabled trigger placeholder with the active form (pull_request_target) - docs/infra/pull-request-environments.md: explain why destroy triggers on pull_request_target rather than pull_request See #54 for the cross-cutting storage-account-sharing issue surfaced by this review (filed against #37). --- .../ci-{{app_name}}-pr-environment-destroy.yml.jinja | 2 +- .github/workflows/pr-environment-checks.yml | 10 +--------- bin/stale-test-environments | 8 ++++---- bin/update-pr-environment | 11 ++++++----- docs/infra/pull-request-environments.md | 2 ++ 5 files changed, 14 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja index 137c93e..348965a 100644 --- a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja +++ b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja @@ -16,7 +16,7 @@ on: # # to enable these lines. They are here as comments for context. # - # pull_request: + # pull_request_target: # types: [closed] {% endif %} diff --git a/.github/workflows/pr-environment-checks.yml b/.github/workflows/pr-environment-checks.yml index 1771bf8..b8d220b 100644 --- a/.github/workflows/pr-environment-checks.yml +++ b/.github/workflows/pr-environment-checks.yml @@ -37,9 +37,6 @@ jobs: pull-requests: write # Needed to comment on PR repository-projects: read # Workaround for GitHub CLI bug https://github.com/cli/cli/issues/6274 - outputs: - service_endpoint: ${{ steps.update-environment.outputs.service_endpoint }} - steps: - uses: actions/checkout@v4 @@ -53,11 +50,6 @@ jobs: environment: ${{ inputs.environment }} - name: Update environment - id: update-environment - run: | - ./bin/update-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" "${{ inputs.commit_hash }}" - service_endpoint=$(terraform -chdir="infra/${{ inputs.app_name }}/service" output -raw service_endpoint) - echo "service_endpoint=${service_endpoint}" - echo "service_endpoint=${service_endpoint}" >> "$GITHUB_OUTPUT" + run: ./bin/update-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" "${{ inputs.commit_hash }}" env: GH_TOKEN: ${{ github.token }} diff --git a/bin/stale-test-environments b/bin/stale-test-environments index 9bce536..f70cfc0 100755 --- a/bin/stale-test-environments +++ b/bin/stale-test-environments @@ -10,7 +10,7 @@ source bin/util.sh GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY:-/dev/null} # 24 hours in seconds -MAX_ENVIRONMENT_AGE_MILLIS=86400 +MAX_ENVIRONMENT_AGE_SECONDS=86400 app_name="$1" @@ -37,9 +37,9 @@ for test_id in $test_ids; do current_timestamp="$(date +%s)" age="$((current_timestamp - test_timestamp))" - if [ "${age}" -gt "${MAX_ENVIRONMENT_AGE_MILLIS}" ]; then - echo "stale ID: ${test_id} age: $((age / 3600)) hours started: $(date -r "${test_timestamp}")" - echo "stale ID: ${test_id} age: $((age / 3600)) hours started: $(date -r "${test_timestamp}")" >> "${GITHUB_STEP_SUMMARY}" + if [ "${age}" -gt "${MAX_ENVIRONMENT_AGE_SECONDS}" ]; then + echo "stale ID: ${test_id} age: $((age / 3600)) hours started: $(date -d "@${test_timestamp}")" + echo "stale ID: ${test_id} age: $((age / 3600)) hours started: $(date -d "@${test_timestamp}")" >> "${GITHUB_STEP_SUMMARY}" stale_tests+=("${test_id}") fi diff --git a/bin/update-pr-environment b/bin/update-pr-environment index 29dbce3..1f7f881 100755 --- a/bin/update-pr-environment +++ b/bin/update-pr-environment @@ -34,16 +34,17 @@ service_endpoint="$(terraform -chdir="infra/${app_name}/service" output -raw ser echo "::group::Wait for ${service_endpoint} to become healthy" # Azure Container Apps has no clean CLI equivalent of `aws ecs wait -# services-stable`. Poll the service's /health endpoint until it returns 200 -# (or give up after ~5 minutes). This matches what infra/test/infra_test.go -# does for Terratest. +# services-stable`. Poll the service's /health endpoint until it returns 2xx. +# This matches what infra/test/infra_test.go does for Terratest. attempt=0 max_attempts=60 sleep_seconds=5 -until curl --silent --show-error --fail --max-time 10 "${service_endpoint}/health" > /dev/null; do +curl_max_time=10 +until curl --silent --show-error --fail --location --max-time "${curl_max_time}" "${service_endpoint}/health" > /dev/null; do attempt=$((attempt + 1)) if [ "${attempt}" -ge "${max_attempts}" ]; then - echo "Service did not become healthy within $((max_attempts * sleep_seconds)) seconds" + max_wait=$((max_attempts * (curl_max_time + sleep_seconds))) + echo "Service did not become healthy after ${max_attempts} attempts (~${max_wait}s worst case)" exit 1 fi echo "Attempt ${attempt}/${max_attempts}: service not healthy yet, retrying in ${sleep_seconds}s..." diff --git a/docs/infra/pull-request-environments.md b/docs/infra/pull-request-environments.md index 5ca94d1..72e80d7 100644 --- a/docs/infra/pull-request-environments.md +++ b/docs/infra/pull-request-environments.md @@ -56,6 +56,8 @@ Using these reusable workflows, configure PR environments for each application w - `ci--pr-environment-destroy.yml` - Based on [ci-{{app_name}}-pr-environment-destroy.yml](/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja) +Note that the destroy wrapper triggers on `pull_request_target: [closed]` rather than `pull_request: [closed]`. `pull_request_target` runs the workflow file from the base branch (with access to repo secrets, including Azure credentials), even when a pull request originates from a fork. `pull_request` from a fork does not get secrets, so the destroy step would not be able to authenticate to Azure. Because the workflow file used is the base branch's, a PR author cannot inject malicious code into the destroy step. + ## Orphaned environment cleanup A scheduled workflow ([scan-orphaned-environments.yml](/.github/workflows/scan-orphaned-environments.yml)) runs daily and checks for two kinds of leftover environments: From f28e97ee950b24392bedee1f4a81bf3862376bea Mon Sep 17 00:00:00 2001 From: Sean Thomas Date: Mon, 29 Jun 2026 15:39:07 -0400 Subject: [PATCH 3/8] Improve PR environment scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix health check timeout: reduce from 15min to 5min (20 attempts × 15s) - Add error context to health check loop for better debugging - Add app_name validation to all scripts to prevent confusing errors - Add GNU date portability comment in stale-test-environments - Add input validation to base62_decode function for empty strings and invalid characters --- bin/destroy-pr-environment | 6 ++++++ bin/orphaned-pr-environments | 6 ++++++ bin/stale-test-environments | 7 +++++++ bin/update-pr-environment | 13 +++++++++++-- bin/util.sh | 13 +++++++++++++ 5 files changed, 43 insertions(+), 2 deletions(-) diff --git a/bin/destroy-pr-environment b/bin/destroy-pr-environment index 1cdcac3..c930edc 100755 --- a/bin/destroy-pr-environment +++ b/bin/destroy-pr-environment @@ -14,6 +14,12 @@ app_name="$1" environment="$2" pr_number="$3" +# Validate app_name +if [ ! -d "infra/${app_name}/service" ]; then + echo "Error: App '${app_name}' not found in infra/" + exit 1 +fi + workspace="p-${pr_number}" echo "::group::Initialize Terraform with backend for environment: ${environment}" diff --git a/bin/orphaned-pr-environments b/bin/orphaned-pr-environments index ce597eb..88324fb 100755 --- a/bin/orphaned-pr-environments +++ b/bin/orphaned-pr-environments @@ -11,6 +11,12 @@ GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY:-/dev/null} app_name="$1" +# Validate app_name +if [ ! -d "infra/${app_name}/service" ]; then + echo "Error: App '${app_name}' not found in infra/" + exit 1 +fi + echo "::group::Initialize Terraform" echo terraform -chdir="infra/${app_name}/service" init -input=false -reconfigure -backend-config="dev.azurerm.tfbackend" terraform -chdir="infra/${app_name}/service" init -input=false -reconfigure -backend-config="dev.azurerm.tfbackend" diff --git a/bin/stale-test-environments b/bin/stale-test-environments index f70cfc0..019d86c 100755 --- a/bin/stale-test-environments +++ b/bin/stale-test-environments @@ -14,6 +14,12 @@ MAX_ENVIRONMENT_AGE_SECONDS=86400 app_name="$1" +# Validate app_name +if [ ! -d "infra/${app_name}/service" ]; then + echo "Error: App '${app_name}' not found in infra/" + exit 1 +fi + echo "::group::Initialize Terraform" echo terraform -chdir="infra/${app_name}/service" init -input=false -reconfigure -backend-config="dev.azurerm.tfbackend" terraform -chdir="infra/${app_name}/service" init -input=false -reconfigure -backend-config="dev.azurerm.tfbackend" @@ -38,6 +44,7 @@ for test_id in $test_ids; do age="$((current_timestamp - test_timestamp))" if [ "${age}" -gt "${MAX_ENVIRONMENT_AGE_SECONDS}" ]; then + # Note: date -d is GNU-specific; this script is designed for Linux environments (GitHub Actions) echo "stale ID: ${test_id} age: $((age / 3600)) hours started: $(date -d "@${test_timestamp}")" echo "stale ID: ${test_id} age: $((age / 3600)) hours started: $(date -d "@${test_timestamp}")" >> "${GITHUB_STEP_SUMMARY}" stale_tests+=("${test_id}") diff --git a/bin/update-pr-environment b/bin/update-pr-environment index 1f7f881..a6fbddb 100755 --- a/bin/update-pr-environment +++ b/bin/update-pr-environment @@ -17,6 +17,12 @@ environment="$2" pr_number="$3" image_tag="$4" +# Validate app_name +if [ ! -d "infra/${app_name}/service" ]; then + echo "Error: App '${app_name}' not found in infra/" + exit 1 +fi + workspace="p-${pr_number}" echo "::group::Initialize Terraform with backend for environment: ${environment}" @@ -37,17 +43,20 @@ echo "::group::Wait for ${service_endpoint} to become healthy" # services-stable`. Poll the service's /health endpoint until it returns 2xx. # This matches what infra/test/infra_test.go does for Terratest. attempt=0 -max_attempts=60 +max_attempts=20 sleep_seconds=5 curl_max_time=10 -until curl --silent --show-error --fail --location --max-time "${curl_max_time}" "${service_endpoint}/health" > /dev/null; do +# Max wait: 20 attempts × (10s curl timeout + 5s sleep) = 300s = 5 minutes +until response=$(curl --silent --show-error --fail --location --max-time "${curl_max_time}" "${service_endpoint}/health" 2>&1); do attempt=$((attempt + 1)) if [ "${attempt}" -ge "${max_attempts}" ]; then max_wait=$((max_attempts * (curl_max_time + sleep_seconds))) echo "Service did not become healthy after ${max_attempts} attempts (~${max_wait}s worst case)" + echo "Last error: ${response}" exit 1 fi echo "Attempt ${attempt}/${max_attempts}: service not healthy yet, retrying in ${sleep_seconds}s..." + echo " Error: ${response}" sleep "${sleep_seconds}" done echo "Service is healthy" diff --git a/bin/util.sh b/bin/util.sh index f4540a1..788ceae 100755 --- a/bin/util.sh +++ b/bin/util.sh @@ -15,9 +15,22 @@ function base62_decode() { local s=$1 local n=0 + # Handle empty string + if [ -z "$s" ]; then + echo "0" + return + fi + for ((i=0;i<${#s};i++)); do c=${s:$i:1} pos=${digits%%"$c"*} + + # Check if character is valid (if pos equals digits, character wasn't found) + if [ "$pos" = "$digits" ]; then + echo "0" + return + fi + n=$((n*62 + ${#pos})) done From ba6611dccb92d536cb239679dd8bc37345bae154 Mon Sep 17 00:00:00 2001 From: Sean Thomas Date: Tue, 30 Jun 2026 09:33:35 -0400 Subject: [PATCH 4/8] Add resilience improvements for PR environments - Add retry logic for terraform apply/destroy to handle transient Azure API errors - Add retry logic for PR body updates to handle race conditions when multiple workflows run - Add workspace existence checks in destroy script to handle edge cases - Add safety checks to prevent deleting default workspace --- bin/destroy-pr-environment | 96 ++++++++++++++++++++++++++++++++------ bin/update-pr-environment | 81 ++++++++++++++++++++++++++------ 2 files changed, 148 insertions(+), 29 deletions(-) diff --git a/bin/destroy-pr-environment b/bin/destroy-pr-environment index c930edc..f52f983 100755 --- a/bin/destroy-pr-environment +++ b/bin/destroy-pr-environment @@ -22,15 +22,57 @@ fi workspace="p-${pr_number}" +# Safety check: never allow deleting the default workspace +if [ "$workspace" = "default" ] || [ -z "$workspace" ]; then + echo "Error: Refusing to delete default or empty workspace" + exit 1 +fi + echo "::group::Initialize Terraform with backend for environment: ${environment}" terraform -chdir="infra/${app_name}/service" init -backend-config="${environment}.azurerm.tfbackend" echo "::endgroup::" +echo "Check if Terraform workspace exists: ${workspace}" +# List workspaces and check if our workspace exists +if terraform -chdir="infra/${app_name}/service" workspace list | grep -q " ${workspace}$"; then + echo "Workspace ${workspace} exists, proceeding with destroy" +else + echo "Workspace ${workspace} does not exist - nothing to destroy" + echo "This can happen if the workspace was already cleaned up or never created" + exit 0 +fi + echo "Select Terraform workspace: ${workspace}" terraform -chdir="infra/${app_name}/service" workspace select "${workspace}" echo "::group::Destroy resources" -terraform -chdir="infra/${app_name}/service" destroy -var="environment_name=${environment}" -input=false -auto-approve +# Retry terraform destroy to handle transient Azure API errors +max_tf_attempts=3 +tf_attempt=0 +tf_success=false + +while [ $tf_attempt -lt $max_tf_attempts ] && [ "$tf_success" = "false" ]; do + tf_attempt=$((tf_attempt + 1)) + + if [ $tf_attempt -gt 1 ]; then + echo "Retry attempt ${tf_attempt}/${max_tf_attempts} after transient failure..." + fi + + if terraform -chdir="infra/${app_name}/service" destroy -var="environment_name=${environment}" -input=false -auto-approve; then + tf_success=true + echo "Terraform destroy succeeded" + else + if [ $tf_attempt -lt $max_tf_attempts ]; then + echo "Terraform destroy failed, waiting 30s before retry..." + sleep 30 + fi + fi +done + +if [ "$tf_success" = "false" ]; then + echo "Error: Terraform destroy failed after ${max_tf_attempts} attempts" + exit 1 +fi echo "::endgroup::" echo "Select default workspace" @@ -47,19 +89,45 @@ pr_info=$(cat <"*""* ]]; then - pr_body="${pr_body//*}" -fi +# Use a retry loop to handle race conditions when multiple workflows update the PR body +max_attempts=5 +attempt=0 +updated=false -if [[ $pr_body == *""*""* ]]; then - pr_body="${pr_body//*/$pr_info}" -else - pr_body="${pr_body}"$'\n\n'"${pr_info}" -fi +while [ $attempt -lt $max_attempts ] && [ "$updated" = "false" ]; do + attempt=$((attempt + 1)) -echo "Update PR description with PR environment info" -echo "${pr_info}" -gh pr edit "${pr_number}" --body "${pr_body}" + # Read current PR body + pr_body="$(gh pr view "${pr_number}" --json body | jq --raw-output .body)" + + # clean up older single-app section if present + if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*}" + fi + + if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*/$pr_info}" + else + pr_body="${pr_body}"$'\n\n'"${pr_info}" + fi + + # Try to update the PR body + if gh pr edit "${pr_number}" --body "${pr_body}"; then + updated=true + echo "Successfully updated PR description" + else + if [ $attempt -lt $max_attempts ]; then + echo "Failed to update PR description (attempt ${attempt}/${max_attempts}), retrying in 2s..." + sleep 2 + fi + fi +done + +if [ "$updated" = "false" ]; then + echo "Warning: Failed to update PR description after ${max_attempts} attempts" + # Don't fail the workflow just because we couldn't update the PR body + echo "Continuing despite PR update failure..." +fi diff --git a/bin/update-pr-environment b/bin/update-pr-environment index a6fbddb..4176840 100755 --- a/bin/update-pr-environment +++ b/bin/update-pr-environment @@ -33,7 +33,33 @@ echo "Select or create Terraform workspace: ${workspace}" terraform -chdir="infra/${app_name}/service" workspace select -or-create "${workspace}" echo "::group::Apply changes to environment using image tag: ${image_tag}" -terraform -chdir="infra/${app_name}/service" apply -input=false -auto-approve -var="environment_name=${environment}" -var="image_tag=${image_tag}" +# Retry terraform apply to handle transient Azure API errors +max_tf_attempts=3 +tf_attempt=0 +tf_success=false + +while [ $tf_attempt -lt $max_tf_attempts ] && [ "$tf_success" = "false" ]; do + tf_attempt=$((tf_attempt + 1)) + + if [ $tf_attempt -gt 1 ]; then + echo "Retry attempt ${tf_attempt}/${max_tf_attempts} after transient failure..." + fi + + if terraform -chdir="infra/${app_name}/service" apply -input=false -auto-approve -var="environment_name=${environment}" -var="image_tag=${image_tag}"; then + tf_success=true + echo "Terraform apply succeeded" + else + if [ $tf_attempt -lt $max_tf_attempts ]; then + echo "Terraform apply failed, waiting 30s before retry..." + sleep 30 + fi + fi +done + +if [ "$tf_success" = "false" ]; then + echo "Error: Terraform apply failed after ${max_tf_attempts} attempts" + exit 1 +fi echo "::endgroup::" service_endpoint="$(terraform -chdir="infra/${app_name}/service" output -raw service_endpoint)" @@ -71,20 +97,45 @@ pr_info=$(cat <"*""* ]]; then - pr_body="${pr_body//*}" -fi +# Use a retry loop to handle race conditions when multiple workflows update the PR body +max_attempts=5 +attempt=0 +updated=false -# update or add the environment info -if [[ $pr_body == *""*""* ]]; then - pr_body="${pr_body//*/$pr_info}" -else - pr_body="${pr_body}"$'\n\n'"${pr_info}" -fi +while [ $attempt -lt $max_attempts ] && [ "$updated" = "false" ]; do + attempt=$((attempt + 1)) -echo "Update PR description with PR environment info" -echo "${pr_info}" -gh pr edit "${pr_number}" --body "${pr_body}" + # Read current PR body + pr_body="$(gh pr view "${pr_number}" --json body | jq --raw-output .body)" + + # clean up older single-app section if present + if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*}" + fi + + # update or add the environment info + if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*/$pr_info}" + else + pr_body="${pr_body}"$'\n\n'"${pr_info}" + fi + + # Try to update the PR body + if gh pr edit "${pr_number}" --body "${pr_body}"; then + updated=true + echo "Successfully updated PR description" + else + if [ $attempt -lt $max_attempts ]; then + echo "Failed to update PR description (attempt ${attempt}/${max_attempts}), retrying in 2s..." + sleep 2 + fi + fi +done + +if [ "$updated" = "false" ]; then + echo "Warning: Failed to update PR description after ${max_attempts} attempts" + exit 1 +fi From bf681a200fef1bbcbf09c1bc760a4e6de82c0a30 Mon Sep 17 00:00:00 2001 From: Sean Thomas Date: Tue, 30 Jun 2026 10:10:51 -0400 Subject: [PATCH 5/8] Fix workspace existence check for currently selected workspace The grep pattern was looking for workspaces prefixed with two spaces, but terraform workspace list shows the currently selected workspace with an asterisk prefix instead. This would cause the existence check to fail if trying to destroy the currently selected workspace. Changed pattern from ' ${workspace}$' to '^[* ] +${workspace}$' to match both selected (with *) and non-selected (with spaces) workspaces. --- bin/destroy-pr-environment | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/destroy-pr-environment b/bin/destroy-pr-environment index f52f983..e7c186f 100755 --- a/bin/destroy-pr-environment +++ b/bin/destroy-pr-environment @@ -34,7 +34,8 @@ echo "::endgroup::" echo "Check if Terraform workspace exists: ${workspace}" # List workspaces and check if our workspace exists -if terraform -chdir="infra/${app_name}/service" workspace list | grep -q " ${workspace}$"; then +# Note: workspace list shows current workspace with "* " prefix, others with " " prefix +if terraform -chdir="infra/${app_name}/service" workspace list | grep -qE "^[* ] +${workspace}$"; then echo "Workspace ${workspace} exists, proceeding with destroy" else echo "Workspace ${workspace} does not exist - nothing to destroy" From 7323fecb67a83e273e6d1c11d25d3d4c09bf6254 Mon Sep 17 00:00:00 2001 From: Sean Thomas Date: Wed, 22 Jul 2026 16:41:51 -0400 Subject: [PATCH 6/8] Wait for PR environment health via Azure CLI in a shared script Extract the PR-environment health wait into bin/wait-for-pr-environment-healthy so the polling loop can be shared with the AWS template, with the cloud-specific health check isolated to one function. Replace the /health endpoint poll with an Azure-native check: require the Container App's provisioningState to be Succeeded and its active revision to be running and healthy. Add a service_name output so the script gets the exact Container App name from Terraform instead of reconstructing it. --- bin/update-pr-environment | 28 ++------- bin/wait-for-pr-environment-healthy | 89 +++++++++++++++++++++++++++ infra/{{app_name}}/service/outputs.tf | 5 ++ 3 files changed, 100 insertions(+), 22 deletions(-) create mode 100755 bin/wait-for-pr-environment-healthy diff --git a/bin/update-pr-environment b/bin/update-pr-environment index 4176840..a11bf0d 100755 --- a/bin/update-pr-environment +++ b/bin/update-pr-environment @@ -63,29 +63,13 @@ fi echo "::endgroup::" service_endpoint="$(terraform -chdir="infra/${app_name}/service" output -raw service_endpoint)" +service_resource_group="$(terraform -chdir="infra/${app_name}/service" output -raw service_resource_group)" +service_name="$(terraform -chdir="infra/${app_name}/service" output -raw service_name)" -echo "::group::Wait for ${service_endpoint} to become healthy" -# Azure Container Apps has no clean CLI equivalent of `aws ecs wait -# services-stable`. Poll the service's /health endpoint until it returns 2xx. -# This matches what infra/test/infra_test.go does for Terratest. -attempt=0 -max_attempts=20 -sleep_seconds=5 -curl_max_time=10 -# Max wait: 20 attempts × (10s curl timeout + 5s sleep) = 300s = 5 minutes -until response=$(curl --silent --show-error --fail --location --max-time "${curl_max_time}" "${service_endpoint}/health" 2>&1); do - attempt=$((attempt + 1)) - if [ "${attempt}" -ge "${max_attempts}" ]; then - max_wait=$((max_attempts * (curl_max_time + sleep_seconds))) - echo "Service did not become healthy after ${max_attempts} attempts (~${max_wait}s worst case)" - echo "Last error: ${response}" - exit 1 - fi - echo "Attempt ${attempt}/${max_attempts}: service not healthy yet, retrying in ${sleep_seconds}s..." - echo " Error: ${response}" - sleep "${sleep_seconds}" -done -echo "Service is healthy" +echo "::group::Wait for ${service_name} to become healthy" +# The wait logic lives in a separate script so it can be shared with the AWS +# template — only the "service health" check inside it needs to differ. +./bin/wait-for-pr-environment-healthy "${service_resource_group}" "${service_name}" echo "::endgroup::" pr_info=$(cat <&1) || { + echo "az containerapp show failed: ${provisioning_state}" + return 1 + } + + if [ "${provisioning_state}" != "Succeeded" ]; then + echo "provisioningState is '${provisioning_state}', want 'Succeeded'" + return 1 + fi + + # Check that the latest revision is active and healthy/running. + local revision_status + revision_status=$(az containerapp revision list \ + --resource-group "${resource_group}" \ + --name "${container_app_name}" \ + --query "[?properties.active].{health:properties.healthState,running:properties.runningState} | [0]" \ + --output json 2>&1) || { + echo "az containerapp revision list failed: ${revision_status}" + return 1 + } + + local health_state running_state + health_state=$(echo "${revision_status}" | jq --raw-output '.health // "Unknown"') + running_state=$(echo "${revision_status}" | jq --raw-output '.running // "Unknown"') + + # healthState is "Healthy" once probes pass (or "None" when no probes are + # configured); runningState is "Running" once the revision is serving. + if { [ "${health_state}" = "Healthy" ] || [ "${health_state}" = "None" ]; } \ + && [ "${running_state}" = "Running" ]; then + return 0 + fi + + echo "active revision not ready yet (healthState='${health_state}', runningState='${running_state}')" + return 1 +} + +# Max wait: 20 attempts × 15s sleep = 300s = 5 minutes +until response=$(is_healthy); do + attempt=$((attempt + 1)) + if [ "${attempt}" -ge "${max_attempts}" ]; then + max_wait=$((max_attempts * sleep_seconds)) + echo "Service did not become healthy after ${max_attempts} attempts (~${max_wait}s worst case)" + echo "Last status: ${response}" + exit 1 + fi + echo "Attempt ${attempt}/${max_attempts}: service not healthy yet, retrying in ${sleep_seconds}s..." + echo " Status: ${response}" + sleep "${sleep_seconds}" +done +echo "Service is healthy" diff --git a/infra/{{app_name}}/service/outputs.tf b/infra/{{app_name}}/service/outputs.tf index 7544cba..219ff4f 100644 --- a/infra/{{app_name}}/service/outputs.tf +++ b/infra/{{app_name}}/service/outputs.tf @@ -10,6 +10,11 @@ output "service_resource_group" { value = local.resource_group_name } +output "service_name" { + description = "The name of the Container App running the service." + value = local.service_name +} + output "service_job_name" { value = module.service.service_job_name } From 24cc71c161ea9e56fdf2d4a0e419a15343021383 Mon Sep 17 00:00:00 2001 From: Sean Thomas Date: Thu, 23 Jul 2026 16:10:54 -0400 Subject: [PATCH 7/8] Address review comments on PR environment scripts - Remove legacy single-app PR comment cleanup code, which is irrelevant to the Azure template since it has no historic single-app PR environment comments to migrate. - Correct the PR-body retry-loop comment in update-pr-environment and destroy-pr-environment: the retry handles transient GitHub API errors, not concurrent-update races. Add a TODO referencing https://github.com/navapbc/template-infra/issues/982 for the actual fix. - Expand the TODO(#50) reference in scan-orphaned-environments.yml to the full issue URL. --- .github/workflows/scan-orphaned-environments.yml | 2 +- bin/destroy-pr-environment | 6 +++++- bin/update-pr-environment | 11 +++++------ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/scan-orphaned-environments.yml b/.github/workflows/scan-orphaned-environments.yml index cc5a9ad..0917cbd 100644 --- a/.github/workflows/scan-orphaned-environments.yml +++ b/.github/workflows/scan-orphaned-environments.yml @@ -66,5 +66,5 @@ jobs: GH_TOKEN: ${{ github.token }} TF_IN_AUTOMATION: "true" - # TODO(#50): Add a notify job that posts to Slack on failure. Until then, + # TODO(https://github.com/navapbc/template-infra-azure/issues/50): Add a notify job that posts to Slack on failure. Until then, # orphaned-environment alerts only show up in the Actions UI. diff --git a/bin/destroy-pr-environment b/bin/destroy-pr-environment index e7c186f..1071fd8 100755 --- a/bin/destroy-pr-environment +++ b/bin/destroy-pr-environment @@ -93,7 +93,11 @@ EOF echo "Update PR description with PR environment info" echo "${pr_info}" -# Use a retry loop to handle race conditions when multiple workflows update the PR body +# Retry to handle transient GitHub API errors when updating the PR body. This +# does NOT protect against races between concurrent workflows updating the same +# PR body; that requires a locking/read-modify-write fix tracked in +# https://github.com/navapbc/template-infra/issues/982 +# TODO(https://github.com/navapbc/template-infra/issues/982): handle concurrent PR body updates max_attempts=5 attempt=0 updated=false diff --git a/bin/update-pr-environment b/bin/update-pr-environment index a11bf0d..24262cb 100755 --- a/bin/update-pr-environment +++ b/bin/update-pr-environment @@ -84,7 +84,11 @@ EOF echo "Update PR description with PR environment info" echo "${pr_info}" -# Use a retry loop to handle race conditions when multiple workflows update the PR body +# Retry to handle transient GitHub API errors when updating the PR body. This +# does NOT protect against races between concurrent workflows updating the same +# PR body; that requires a locking/read-modify-write fix tracked in +# https://github.com/navapbc/template-infra/issues/982 +# TODO(https://github.com/navapbc/template-infra/issues/982): handle concurrent PR body updates max_attempts=5 attempt=0 updated=false @@ -95,11 +99,6 @@ while [ $attempt -lt $max_attempts ] && [ "$updated" = "false" ]; do # Read current PR body pr_body="$(gh pr view "${pr_number}" --json body | jq --raw-output .body)" - # clean up older single-app section if present - if [[ $pr_body == *""*""* ]]; then - pr_body="${pr_body//*}" - fi - # update or add the environment info if [[ $pr_body == *""*""* ]]; then pr_body="${pr_body//*/$pr_info}" From 1181fcbff3562252b6ece201299465bc26d62e30 Mon Sep 17 00:00:00 2001 From: Sean Thomas Date: Thu, 30 Jul 2026 18:01:26 -0400 Subject: [PATCH 8/8] Remove legacy single-app cleanup from destroy-pr-environment Removes the same legacy single-app PR comment cleanup block that was already removed from update-pr-environment. It is irrelevant to the Azure template (no historic single-app PR environment comments to migrate) and left the two scripts inconsistent. --- bin/destroy-pr-environment | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bin/destroy-pr-environment b/bin/destroy-pr-environment index 1071fd8..d08b2e5 100755 --- a/bin/destroy-pr-environment +++ b/bin/destroy-pr-environment @@ -108,11 +108,6 @@ while [ $attempt -lt $max_attempts ] && [ "$updated" = "false" ]; do # Read current PR body pr_body="$(gh pr view "${pr_number}" --json body | jq --raw-output .body)" - # clean up older single-app section if present - if [[ $pr_body == *""*""* ]]; then - pr_body="${pr_body//*}" - fi - if [[ $pr_body == *""*""* ]]; then pr_body="${pr_body//*/$pr_info}" else