diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja new file mode 100644 index 0000000..899ff19 --- /dev/null +++ b/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja @@ -0,0 +1,33 @@ +name: CI {{ app_name }} PR Environment Checks +on: + workflow_dispatch: + inputs: + pr_number: + required: true + type: string + commit_hash: + required: true + type: string + {% if app_has_dev_env_setup %} + pull_request: + {% else %} + # !! Once you've set up the dev environment and are ready to enable PR + # environments, run: + # + # nava-platform infra update --answers-only --data app_has_dev_env_setup=true . + # + # to enable these lines. They are here as comments for context. + # + # pull_request: + {% endif %} + +jobs: + update: + name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise + uses: ./.github/workflows/pr-environment-checks.yml + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.state == 'open' + with: + app_name: "{{ app_name }}" + environment: "dev" + pr_number: ${{'{{'}} inputs.pr_number || github.event.number {{'}}'}} + commit_hash: ${{'{{'}} inputs.commit_hash || github.event.pull_request.head.sha {{'}}'}} diff --git a/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja new file mode 100644 index 0000000..348965a --- /dev/null +++ b/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja @@ -0,0 +1,30 @@ +name: CI {{ app_name }} PR Environment Destroy +on: + workflow_dispatch: + inputs: + pr_number: + required: true + type: string + {% if app_has_dev_env_setup %} + pull_request_target: + types: [closed] + {% else %} + # !! Once you've set up the dev environment and are ready to enable PR + # environments, run: + # + # nava-platform infra update --answers-only --data app_has_dev_env_setup=true . + # + # to enable these lines. They are here as comments for context. + # + # pull_request_target: + # types: [closed] + {% endif %} + +jobs: + destroy: + name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise + uses: ./.github/workflows/pr-environment-destroy.yml + with: + app_name: "{{ app_name }}" + environment: "dev" + pr_number: ${{'{{'}} inputs.pr_number || github.event.number {{'}}'}} diff --git a/.github/workflows/pr-environment-checks.yml b/.github/workflows/pr-environment-checks.yml new file mode 100644 index 0000000..b8d220b --- /dev/null +++ b/.github/workflows/pr-environment-checks.yml @@ -0,0 +1,55 @@ +name: PR Environment Update +run-name: Update PR Environment ${{ inputs.pr_number }} +on: + workflow_call: + inputs: + app_name: + required: true + type: string + environment: + required: true + type: string + pr_number: + required: true + type: string + commit_hash: + required: true + type: string + +concurrency: pr-environment-${{ inputs.app_name }}-${{ inputs.pr_number }} + +jobs: + build-and-publish: + name: " " # GitHub UI is noisy when calling reusable workflows, so use whitespace for name to reduce noise + uses: ./.github/workflows/build-and-publish.yml + with: + app_name: ${{ inputs.app_name }} + ref: ${{ inputs.commit_hash }} + + update: + name: Update environment + needs: [build-and-publish] + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + pull-requests: write # Needed to comment on PR + repository-projects: read # Workaround for GitHub CLI bug https://github.com/cli/cli/issues/6274 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Terraform + uses: ./.github/actions/setup-terraform + + - name: Configure Azure credentials + uses: ./.github/actions/configure-azure-credentials + with: + app_name: ${{ inputs.app_name }} + environment: ${{ inputs.environment }} + + - name: Update environment + run: ./bin/update-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" "${{ inputs.commit_hash }}" + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/pr-environment-destroy.yml b/.github/workflows/pr-environment-destroy.yml new file mode 100644 index 0000000..627e93a --- /dev/null +++ b/.github/workflows/pr-environment-destroy.yml @@ -0,0 +1,44 @@ +name: PR Environment Destroy +run-name: Destroy PR Environment ${{ inputs.pr_number }} +on: + workflow_call: + inputs: + app_name: + required: true + type: string + environment: + required: true + type: string + pr_number: + required: true + type: string + +concurrency: pr-environment-${{ inputs.app_name }}-${{ inputs.pr_number }} + +jobs: + destroy: + name: Destroy environment + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + pull-requests: write # Needed to comment on PR + repository-projects: read # Workaround for GitHub CLI bug https://github.com/cli/cli/issues/6274 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Terraform + uses: ./.github/actions/setup-terraform + + - name: Configure Azure credentials + uses: ./.github/actions/configure-azure-credentials + with: + app_name: ${{ inputs.app_name }} + environment: ${{ inputs.environment }} + + - name: Destroy environment + run: ./bin/destroy-pr-environment "${{ inputs.app_name }}" "${{ inputs.environment }}" "${{ inputs.pr_number }}" + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/scan-orphaned-environments.yml b/.github/workflows/scan-orphaned-environments.yml new file mode 100644 index 0000000..0917cbd --- /dev/null +++ b/.github/workflows/scan-orphaned-environments.yml @@ -0,0 +1,70 @@ +# This workflow scans for temporary environments that were not properly cleaned up +# This can happen if the PR environment destroy workflow failed or didn't run +# or if the temporary environments created by the infra service tests were not cleaned up +name: Scan orphaned environments + +on: + workflow_dispatch: + schedule: + # Run every day at 07:30 UTC (3:30am ET, 12:30am PT) after engineers are likely done with work + - cron: "30 7 * * *" + +jobs: + get-app-names: + name: Get app names + runs-on: ubuntu-latest + + outputs: + app_names: ${{ steps.get-app-names.outputs.app_names }} + + steps: + - uses: actions/checkout@v4 + + - name: Get app names + id: get-app-names + run: | + source bin/util.sh + app_names="$(get_app_names)" + # turn app_names into a json list using jq + app_names="$(echo "${app_names}" | jq -R -s -c 'split("\n")[:-1]')" + echo "App names retrieved: ${app_names}" + echo "app_names=${app_names}" >> "$GITHUB_OUTPUT" + shell: bash + + scan: + name: Scan + runs-on: ubuntu-latest + needs: get-app-names + + strategy: + fail-fast: false + matrix: + app_name: ${{ fromJson(needs.get-app-names.outputs.app_names) }} + scan_script: [orphaned-pr-environments, stale-test-environments] + + permissions: + contents: read + id-token: write + pull-requests: read + + steps: + - uses: actions/checkout@v4 + + - name: Set up Terraform + uses: ./.github/actions/setup-terraform + + - name: Configure Azure credentials + uses: ./.github/actions/configure-azure-credentials + with: + app_name: ${{ matrix.app_name }} + environment: dev + + - name: List PR workspaces + run: | + ./bin/${{ matrix.scan_script }} ${{ matrix.app_name }} + env: + GH_TOKEN: ${{ github.token }} + TF_IN_AUTOMATION: "true" + + # TODO(https://github.com/navapbc/template-infra-azure/issues/50): Add a notify job that posts to Slack on failure. Until then, + # orphaned-environment alerts only show up in the Actions UI. diff --git a/bin/destroy-pr-environment b/bin/destroy-pr-environment new file mode 100755 index 0000000..d08b2e5 --- /dev/null +++ b/bin/destroy-pr-environment @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# Destroy the temporary environment that was created for the pull request. +# +# Positional parameters: +# app_name (required) – the name of subdirectory of /infra that holds the +# application's infrastructure code. +# environment - the name of the application environment (e.g. dev, staging, prod) +# pr_number - the pull request number in GitHub +# ----------------------------------------------------------------------------- +set -euo pipefail + +app_name="$1" +environment="$2" +pr_number="$3" + +# Validate app_name +if [ ! -d "infra/${app_name}/service" ]; then + echo "Error: App '${app_name}' not found in infra/" + exit 1 +fi + +workspace="p-${pr_number}" + +# Safety check: never allow deleting the default workspace +if [ "$workspace" = "default" ] || [ -z "$workspace" ]; then + echo "Error: Refusing to delete default or empty workspace" + exit 1 +fi + +echo "::group::Initialize Terraform with backend for environment: ${environment}" +terraform -chdir="infra/${app_name}/service" init -backend-config="${environment}.azurerm.tfbackend" +echo "::endgroup::" + +echo "Check if Terraform workspace exists: ${workspace}" +# List workspaces and check if our workspace exists +# Note: workspace list shows current workspace with "* " prefix, others with " " prefix +if terraform -chdir="infra/${app_name}/service" workspace list | grep -qE "^[* ] +${workspace}$"; then + echo "Workspace ${workspace} exists, proceeding with destroy" +else + echo "Workspace ${workspace} does not exist - nothing to destroy" + echo "This can happen if the workspace was already cleaned up or never created" + exit 0 +fi + +echo "Select Terraform workspace: ${workspace}" +terraform -chdir="infra/${app_name}/service" workspace select "${workspace}" + +echo "::group::Destroy resources" +# Retry terraform destroy to handle transient Azure API errors +max_tf_attempts=3 +tf_attempt=0 +tf_success=false + +while [ $tf_attempt -lt $max_tf_attempts ] && [ "$tf_success" = "false" ]; do + tf_attempt=$((tf_attempt + 1)) + + if [ $tf_attempt -gt 1 ]; then + echo "Retry attempt ${tf_attempt}/${max_tf_attempts} after transient failure..." + fi + + if terraform -chdir="infra/${app_name}/service" destroy -var="environment_name=${environment}" -input=false -auto-approve; then + tf_success=true + echo "Terraform destroy succeeded" + else + if [ $tf_attempt -lt $max_tf_attempts ]; then + echo "Terraform destroy failed, waiting 30s before retry..." + sleep 30 + fi + fi +done + +if [ "$tf_success" = "false" ]; then + echo "Error: Terraform destroy failed after ${max_tf_attempts} attempts" + exit 1 +fi +echo "::endgroup::" + +echo "Select default workspace" +terraform -chdir="infra/${app_name}/service" workspace select default + +echo "Delete workspace: ${workspace}" +terraform -chdir="infra/${app_name}/service" workspace delete "${workspace}" + +pr_info=$(cat < +## Preview environment for ${app_name} +♻️ Environment destroyed ♻️ + +EOF +) + +echo "Update PR description with PR environment info" +echo "${pr_info}" + +# Retry to handle transient GitHub API errors when updating the PR body. This +# does NOT protect against races between concurrent workflows updating the same +# PR body; that requires a locking/read-modify-write fix tracked in +# https://github.com/navapbc/template-infra/issues/982 +# TODO(https://github.com/navapbc/template-infra/issues/982): handle concurrent PR body updates +max_attempts=5 +attempt=0 +updated=false + +while [ $attempt -lt $max_attempts ] && [ "$updated" = "false" ]; do + attempt=$((attempt + 1)) + + # Read current PR body + pr_body="$(gh pr view "${pr_number}" --json body | jq --raw-output .body)" + + if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*/$pr_info}" + else + pr_body="${pr_body}"$'\n\n'"${pr_info}" + fi + + # Try to update the PR body + if gh pr edit "${pr_number}" --body "${pr_body}"; then + updated=true + echo "Successfully updated PR description" + else + if [ $attempt -lt $max_attempts ]; then + echo "Failed to update PR description (attempt ${attempt}/${max_attempts}), retrying in 2s..." + sleep 2 + fi + fi +done + +if [ "$updated" = "false" ]; then + echo "Warning: Failed to update PR description after ${max_attempts} attempts" + # Don't fail the workflow just because we couldn't update the PR body + echo "Continuing despite PR update failure..." +fi diff --git a/bin/orphaned-pr-environments b/bin/orphaned-pr-environments new file mode 100755 index 0000000..88324fb --- /dev/null +++ b/bin/orphaned-pr-environments @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# This script checks for orphaned PR environments by listing all PR workspaces +# and checking if the associated PR is closed. If the PR is closed the +# resources in the workspace should have been destroyed and the workspace +# deleted, so existing workspaces for closed PRs are considered orphaned. +# ----------------------------------------------------------------------------- +set -euo pipefail + +GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY:-/dev/null} + +app_name="$1" + +# Validate app_name +if [ ! -d "infra/${app_name}/service" ]; then + echo "Error: App '${app_name}' not found in infra/" + exit 1 +fi + +echo "::group::Initialize Terraform" +echo terraform -chdir="infra/${app_name}/service" init -input=false -reconfigure -backend-config="dev.azurerm.tfbackend" +terraform -chdir="infra/${app_name}/service" init -input=false -reconfigure -backend-config="dev.azurerm.tfbackend" +echo "::endgroup::" + +echo "::group::List PRs with PR environments" +echo terraform -chdir="infra/${app_name}/service" workspace list +workspaces="$(terraform -chdir="infra/${app_name}/service" workspace list)" +# grep will exit with code `1` if there's no match, so ignore that for when +# there are no PR workspaces for the application +pr_nums="$(echo "${workspaces}" | { grep -o 'p-[0-9]\+$' || test $? = 1; } | sed 's/p-//')" +echo "PRs" +echo "${pr_nums}" +echo "::endgroup::" + +echo "::group::Check status of each PR" +closed_prs=() +for pr_num in $pr_nums; do + pr_status="$(gh pr view "$pr_num" --json state --jq ".state")" + echo "PR ${pr_num}: ${pr_status}" + + if [ "$pr_status" == "CLOSED" ] || [ "$pr_status" == "MERGED" ]; then + closed_prs+=("$pr_num") + fi +done +echo "::endgroup::" + +# if closed_prs is not empty exit with 1 otherwise exit with 0 +if [ ${#closed_prs[@]} -gt 0 ]; then + echo "🧹 Found orphaned PR environments for the following PRs: **${closed_prs[*]}**" + echo "🧹 Found orphaned PR environments for the following PRs: **${closed_prs[*]}**" >> "${GITHUB_STEP_SUMMARY}" + exit 1 +fi + +echo "✅ No orphaned PR environments" +echo "✅ No orphaned PR environments" >> "${GITHUB_STEP_SUMMARY}" diff --git a/bin/stale-test-environments b/bin/stale-test-environments new file mode 100755 index 0000000..019d86c --- /dev/null +++ b/bin/stale-test-environments @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# This script checks for orphaned environments created by infra service tests +# by checking if the environment is older than a certain threshold. +# ----------------------------------------------------------------------------- +set -euo pipefail + +source bin/util.sh + +GITHUB_STEP_SUMMARY=${GITHUB_STEP_SUMMARY:-/dev/null} + +# 24 hours in seconds +MAX_ENVIRONMENT_AGE_SECONDS=86400 + +app_name="$1" + +# Validate app_name +if [ ! -d "infra/${app_name}/service" ]; then + echo "Error: App '${app_name}' not found in infra/" + exit 1 +fi + +echo "::group::Initialize Terraform" +echo terraform -chdir="infra/${app_name}/service" init -input=false -reconfigure -backend-config="dev.azurerm.tfbackend" +terraform -chdir="infra/${app_name}/service" init -input=false -reconfigure -backend-config="dev.azurerm.tfbackend" +echo "::endgroup::" + +echo "::group::List test IDs of test environments" +echo terraform -chdir="infra/${app_name}/service" workspace list +workspaces="$(terraform -chdir="infra/${app_name}/service" workspace list)" +# grep will exit with code `1` if there's no match, so ignore that for when +# there are no PR workspaces for the application +test_ids="$(echo "${workspaces}" | { grep -o 't-[A-Za-z0-9]\+' || test $? = 1; } | sed 's/t-//')" +echo "Test IDs" +echo "${test_ids}" +echo "::endgroup::" + +echo "::group::Check age of each test environment" +stale_tests=() +for test_id in $test_ids; do + # Base 62 decode the test ID to get the timestamp + test_timestamp="$(base62_decode "${test_id}")" + current_timestamp="$(date +%s)" + age="$((current_timestamp - test_timestamp))" + + if [ "${age}" -gt "${MAX_ENVIRONMENT_AGE_SECONDS}" ]; then + # Note: date -d is GNU-specific; this script is designed for Linux environments (GitHub Actions) + echo "stale ID: ${test_id} age: $((age / 3600)) hours started: $(date -d "@${test_timestamp}")" + echo "stale ID: ${test_id} age: $((age / 3600)) hours started: $(date -d "@${test_timestamp}")" >> "${GITHUB_STEP_SUMMARY}" + stale_tests+=("${test_id}") + fi + + # If age is less than 0, the test ID is invalid + if [ "${age}" -lt 0 ]; then + echo "invalid ID: ${test_id}" + echo "invalid ID: ${test_id}" >> "${GITHUB_STEP_SUMMARY}" + stale_tests+=("${test_id}") + fi +done +echo "::endgroup::" + +# if stale_tests is not empty exit with 1 otherwise exit with 0 +if [ ${#stale_tests[@]} -gt 0 ]; then + echo "🧹 Found stale test environments for the following test IDs: **${stale_tests[*]}**" + echo "🧹 Found stale test environments for the following test IDs: **${stale_tests[*]}**" >> "${GITHUB_STEP_SUMMARY}" + exit 1 +fi + +echo "✅ No stale test environments" +echo "✅ No stale test environments" >> "${GITHUB_STEP_SUMMARY}" diff --git a/bin/update-pr-environment b/bin/update-pr-environment new file mode 100755 index 0000000..24262cb --- /dev/null +++ b/bin/update-pr-environment @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# Create or update a temporary environment that will exist while a pull request +# is open. +# +# Positional parameters: +# app_name (required) – the name of subdirectory of /infra that holds the +# application's infrastructure code. +# environment - the name of the application environment (e.g. dev, staging, prod) +# pr_number - the pull request number in GitHub +# image_tag - the commit hash to deploy for the temporary environment +# ----------------------------------------------------------------------------- +set -euo pipefail + +app_name="$1" +environment="$2" +pr_number="$3" +image_tag="$4" + +# Validate app_name +if [ ! -d "infra/${app_name}/service" ]; then + echo "Error: App '${app_name}' not found in infra/" + exit 1 +fi + +workspace="p-${pr_number}" + +echo "::group::Initialize Terraform with backend for environment: ${environment}" +terraform -chdir="infra/${app_name}/service" init -backend-config="${environment}.azurerm.tfbackend" +echo "::endgroup::" + +echo "Select or create Terraform workspace: ${workspace}" +terraform -chdir="infra/${app_name}/service" workspace select -or-create "${workspace}" + +echo "::group::Apply changes to environment using image tag: ${image_tag}" +# Retry terraform apply to handle transient Azure API errors +max_tf_attempts=3 +tf_attempt=0 +tf_success=false + +while [ $tf_attempt -lt $max_tf_attempts ] && [ "$tf_success" = "false" ]; do + tf_attempt=$((tf_attempt + 1)) + + if [ $tf_attempt -gt 1 ]; then + echo "Retry attempt ${tf_attempt}/${max_tf_attempts} after transient failure..." + fi + + if terraform -chdir="infra/${app_name}/service" apply -input=false -auto-approve -var="environment_name=${environment}" -var="image_tag=${image_tag}"; then + tf_success=true + echo "Terraform apply succeeded" + else + if [ $tf_attempt -lt $max_tf_attempts ]; then + echo "Terraform apply failed, waiting 30s before retry..." + sleep 30 + fi + fi +done + +if [ "$tf_success" = "false" ]; then + echo "Error: Terraform apply failed after ${max_tf_attempts} attempts" + exit 1 +fi +echo "::endgroup::" + +service_endpoint="$(terraform -chdir="infra/${app_name}/service" output -raw service_endpoint)" +service_resource_group="$(terraform -chdir="infra/${app_name}/service" output -raw service_resource_group)" +service_name="$(terraform -chdir="infra/${app_name}/service" output -raw service_name)" + +echo "::group::Wait for ${service_name} to become healthy" +# The wait logic lives in a separate script so it can be shared with the AWS +# template — only the "service health" check inside it needs to differ. +./bin/wait-for-pr-environment-healthy "${service_resource_group}" "${service_name}" +echo "::endgroup::" + +pr_info=$(cat < +## Preview environment for ${app_name} +- Service endpoint: ${service_endpoint} +- Deployed commit: ${image_tag} + +EOF +) + +echo "Update PR description with PR environment info" +echo "${pr_info}" + +# Retry to handle transient GitHub API errors when updating the PR body. This +# does NOT protect against races between concurrent workflows updating the same +# PR body; that requires a locking/read-modify-write fix tracked in +# https://github.com/navapbc/template-infra/issues/982 +# TODO(https://github.com/navapbc/template-infra/issues/982): handle concurrent PR body updates +max_attempts=5 +attempt=0 +updated=false + +while [ $attempt -lt $max_attempts ] && [ "$updated" = "false" ]; do + attempt=$((attempt + 1)) + + # Read current PR body + pr_body="$(gh pr view "${pr_number}" --json body | jq --raw-output .body)" + + # update or add the environment info + if [[ $pr_body == *""*""* ]]; then + pr_body="${pr_body//*/$pr_info}" + else + pr_body="${pr_body}"$'\n\n'"${pr_info}" + fi + + # Try to update the PR body + if gh pr edit "${pr_number}" --body "${pr_body}"; then + updated=true + echo "Successfully updated PR description" + else + if [ $attempt -lt $max_attempts ]; then + echo "Failed to update PR description (attempt ${attempt}/${max_attempts}), retrying in 2s..." + sleep 2 + fi + fi +done + +if [ "$updated" = "false" ]; then + echo "Warning: Failed to update PR description after ${max_attempts} attempts" + exit 1 +fi diff --git a/bin/util.sh b/bin/util.sh new file mode 100755 index 0000000..788ceae --- /dev/null +++ b/bin/util.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Utility functions + +# Retrieve the names of the applications in the repo by listing the directories in the "infra" directory +# and filtering out the directories that are not applications. +# Returns: A list of application names. +function get_app_names() { + find "infra" -maxdepth 1 -type d -not -name "infra" -not -name "accounts" -not -name "modules" -not -name "networks" -not -name "project-config" -not -name "test" -exec basename {} \; +} + +# Base 62 decode a string. +# Returns: String as base 10 number. +function base62_decode() { + local digits="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + local s=$1 + local n=0 + + # Handle empty string + if [ -z "$s" ]; then + echo "0" + return + fi + + for ((i=0;i<${#s};i++)); do + c=${s:$i:1} + pos=${digits%%"$c"*} + + # Check if character is valid (if pos equals digits, character wasn't found) + if [ "$pos" = "$digits" ]; then + echo "0" + return + fi + + n=$((n*62 + ${#pos})) + done + + echo $n +} diff --git a/bin/wait-for-pr-environment-healthy b/bin/wait-for-pr-environment-healthy new file mode 100755 index 0000000..df44fb1 --- /dev/null +++ b/bin/wait-for-pr-environment-healthy @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# Wait for a PR environment's service to become healthy. +# +# This is broken out into its own script so the "wait" logic can be shared with +# the AWS template. The only cloud-specific piece is how "service health" is +# determined, which is isolated to the is_healthy() function below — projects +# (or the AWS template) can swap that one function out while reusing the polling +# loop. +# +# Positional parameters: +# resource_group (required) – resource group of the Container App +# container_app_name (required) – name of the Container App +# ----------------------------------------------------------------------------- +set -euo pipefail + +resource_group="$1" +container_app_name="$2" + +attempt=0 +max_attempts=20 +sleep_seconds=15 + +# Determine whether the service is healthy. Prints any diagnostic output on +# failure (captured by the caller and echoed on the final attempt). +# +# Azure Container Apps has no clean CLI equivalent of `aws ecs wait +# services-stable`, so we consider the service healthy when: +# 1. the Container App's provisioningState is "Succeeded", and +# 2. its latest revision is both Active and in a Healthy/Running state. +# +# To share this script with the AWS template, only this function needs to +# change (or be symlinked in) — the polling loop below stays the same. +is_healthy() { + local provisioning_state + provisioning_state=$(az containerapp show \ + --resource-group "${resource_group}" \ + --name "${container_app_name}" \ + --query "properties.provisioningState" \ + --output tsv 2>&1) || { + echo "az containerapp show failed: ${provisioning_state}" + return 1 + } + + if [ "${provisioning_state}" != "Succeeded" ]; then + echo "provisioningState is '${provisioning_state}', want 'Succeeded'" + return 1 + fi + + # Check that the latest revision is active and healthy/running. + local revision_status + revision_status=$(az containerapp revision list \ + --resource-group "${resource_group}" \ + --name "${container_app_name}" \ + --query "[?properties.active].{health:properties.healthState,running:properties.runningState} | [0]" \ + --output json 2>&1) || { + echo "az containerapp revision list failed: ${revision_status}" + return 1 + } + + local health_state running_state + health_state=$(echo "${revision_status}" | jq --raw-output '.health // "Unknown"') + running_state=$(echo "${revision_status}" | jq --raw-output '.running // "Unknown"') + + # healthState is "Healthy" once probes pass (or "None" when no probes are + # configured); runningState is "Running" once the revision is serving. + if { [ "${health_state}" = "Healthy" ] || [ "${health_state}" = "None" ]; } \ + && [ "${running_state}" = "Running" ]; then + return 0 + fi + + echo "active revision not ready yet (healthState='${health_state}', runningState='${running_state}')" + return 1 +} + +# Max wait: 20 attempts × 15s sleep = 300s = 5 minutes +until response=$(is_healthy); do + attempt=$((attempt + 1)) + if [ "${attempt}" -ge "${max_attempts}" ]; then + max_wait=$((max_attempts * sleep_seconds)) + echo "Service did not become healthy after ${max_attempts} attempts (~${max_wait}s worst case)" + echo "Last status: ${response}" + exit 1 + fi + echo "Attempt ${attempt}/${max_attempts}: service not healthy yet, retrying in ${sleep_seconds}s..." + echo " Status: ${response}" + sleep "${sleep_seconds}" +done +echo "Service is healthy" diff --git a/docs/infra/pull-request-environments.md b/docs/infra/pull-request-environments.md new file mode 100644 index 0000000..72e80d7 --- /dev/null +++ b/docs/infra/pull-request-environments.md @@ -0,0 +1,68 @@ +# Pull request environments + +A temporary environment is created for each pull request that stays up while the pull request is open. The endpoint for the pull request and the deployed commit are added to the pull request description, and updated when the environment is updated. Use cases for the temporary pull request environment include: + +- Allow other delivery stakeholders—including product managers, designers, and business owners—to review changes before being merged and deployed +- Enable automated end-to-end tests on the pull request +- Enable automated accessibility checks on the pull request +- Facilitate workspace creation for developing and testing service layer infrastructure changes + +## Lifecycle of pull request environments + +A pull request environment is created when a pull request is opened or reopened, and destroyed when the pull request is merged or closed. When new commits are pushed up to the pull request, the pull request environment is updated. + +## Shared database of pull request environments + +Pull request environments share the same database as the dev environment. This has the following benefits: + +- Enables testers to leverage existing test user accounts and accumulated test data +- Reduces the need for manual data seeding or migration scripts +- Better simulates production-like conditions where the application must handle pre-existing data, potentially revealing edge cases and integration issues that might not be apparent with a fresh database +- Reduces environment provisioning time significantly since creating and destroying database clusters typically takes 20-40 minutes + +## Limitations of shared resources in pull request environments + +Because PR environments share certain resources (e.g., the database) with the dev environment rather than provisioning their own, there are inherent limitations on what can be tested in a PR environment: + +- **Configuration changes to shared resources cannot be tested in a PR environment.** Changes to the database layer will not take effect until the PR is merged and deployed to dev. +- **Multiple PR environments share the same resource instance.** Changes made by one PR environment (e.g., data written to the database) may be visible to other PR environments sharing the same resource. + +For more on how Terraform workspaces are used to isolate temporary environments, see [Develop and Test Infrastructure in Isolation Using Workspaces](./develop-and-test-infrastructure-in-isolation-using-workspaces.md). + +## Isolate database migrations into separate pull requests + +Database migrations are not reflected in PR environments. In particular, PR environments share the same database with the dev environment, so database migrations that exist in the pull request are not run on the database to avoid impacting the dev environment. + +Therefore, isolate database changes in their own pull request and merge that pull request first before opening pull requests with application changes that depend on those database changes. This practice has the following benefits: + +- Enables PR environments to continue to be fully functional and testable when there are application changes that depend on database changes +- Enables database changes to be tested and deployed in isolation, which helps ensure that existing deployed application code is backwards compatible with new database changes + +This guidance is not strict. It is still okay to combine database migrations and application changes in a single pull request. However, when doing so, note that the PR environment may not be fully functional if the application changes rely on database migrations. + +Note also that this guidance pertains to pull requests, not local development. It is still okay and encouraged to develop database and application changes together during local development. + +## Implementing pull request environments for each application + +Pull request environments are created by GitHub Actions workflows. There are two reusable callable workflows that manage pull request environments: + +- [pr-environment-checks.yml](/.github/workflows/pr-environment-checks.yml) - creates or updates a temporary environment in a separate Terraform workspace for a given application and pull request +- [pr-environment-destroy.yml](/.github/workflows/pr-environment-destroy.yml) - destroys a temporary environment and workspace for a given application and pull request + +Using these reusable workflows, configure PR environments for each application with application-specific workflows: + +- `ci--pr-environment-checks.yml` + - Based on [ci-{{app_name}}-pr-environment-checks.yml](/.github/workflows/ci-{{app_name}}-pr-environment-checks.yml.jinja) +- `ci--pr-environment-destroy.yml` + - Based on [ci-{{app_name}}-pr-environment-destroy.yml](/.github/workflows/ci-{{app_name}}-pr-environment-destroy.yml.jinja) + +Note that the destroy wrapper triggers on `pull_request_target: [closed]` rather than `pull_request: [closed]`. `pull_request_target` runs the workflow file from the base branch (with access to repo secrets, including Azure credentials), even when a pull request originates from a fork. `pull_request` from a fork does not get secrets, so the destroy step would not be able to authenticate to Azure. Because the workflow file used is the base branch's, a PR author cannot inject malicious code into the destroy step. + +## Orphaned environment cleanup + +A scheduled workflow ([scan-orphaned-environments.yml](/.github/workflows/scan-orphaned-environments.yml)) runs daily and checks for two kinds of leftover environments: + +- **Orphaned PR environments** — Terraform workspaces named `p-` whose associated pull request is closed or merged. These indicate that the PR environment destroy workflow failed or didn't run. +- **Stale test environments** — Terraform workspaces named `t-` left behind by Terratest runs that exceeded the maximum environment age (24 hours). + +When orphans are found the scan job fails. Currently failures only surface in the GitHub Actions UI; Slack alerting is tracked in [#50](https://github.com/navapbc/template-infra-azure/issues/50). diff --git a/infra/{{app_name}}/service/outputs.tf b/infra/{{app_name}}/service/outputs.tf index 7544cba..219ff4f 100644 --- a/infra/{{app_name}}/service/outputs.tf +++ b/infra/{{app_name}}/service/outputs.tf @@ -10,6 +10,11 @@ output "service_resource_group" { value = local.resource_group_name } +output "service_name" { + description = "The name of the Container App running the service." + value = local.service_name +} + output "service_job_name" { value = module.service.service_job_name }