deploy-azd.yml

name: deploy-azd-core (reusable)

on:
  workflow_call:
    inputs:
      environment:
        description: Deployment environment name (for azd -e)
        required: true
        type: string
      githubEnvironment:
        description: GitHub Actions environment name used for approvals and protection rules
        required: false
        type: string
        default: branch
      location:
        description: Azure location
        required: true
        type: string
      projectName:
        description: Project prefix used by naming convention
        required: false
        type: string
        default: holidaypeakhub405
      imageTag:
        description: Image tag to deploy
        required: false
        type: string
        default: latest
      sourceSha:
        description: Explicit tested source commit SHA to checkout across the workflow
        required: false
        type: string
        default: ''
      sourceRef:
        description: Explicit tested source ref to checkout when sourceSha is not provided
        required: false
        type: string
        default: ''
      deployStatic:
        description: Provision static web app resources
        required: false
        type: boolean
        default: true
      uiOnly:
        description: Deploy only the UI using SWA token (skips provision and Azure login jobs)
        required: false
        type: boolean
        default: false
      apiBaseUrl:
        description: Optional API base URL for UI build override
        required: false
        type: string
        default: ''
      deployChangedOnly:
        description: Legacy flag (changed-app-only deployment is always enforced)
        required: false
        type: boolean
        default: true
      forceApimSync:
        description: Force APIM sync and smoke checks even when no changed services are detected
        required: false
        type: boolean
        default: false
      autoAllowAcrRunnerIp:
        description: Temporarily allow this GitHub runner egress IP in ACR firewall during deploy and remove it afterward
        required: false
        type: boolean
        default: true
      skipApiSmokeChecks:
        description: Skip API smoke checks for urgent UI-only deployments
        required: false
        type: boolean
        default: false
      serviceFilter:
        description: Comma-separated list of agent service names to deploy (empty = all changed). Use to scope deploys to a subset of services.
        required: false
        type: string
        default: ''
      skipProvision:
        description: Skip azd provision and reuse the current azd environment plus existing infrastructure for an emergency redeploy
        required: false
        type: boolean
        default: false
      skipPromptGates:
        description: Skip the prompt-sync CI gates (image prompt verification and Foundry instructions verification). Use only for urgent deploys where the gates are known to be unrelated to the change.
        required: false
        type: boolean
        default: false
      skipPostgresPreflight:
        description: Skip the PostgreSQL password preflight probe run before the CRUD rollout. When true, the gate is bypassed and a warning is logged.
        required: false
        type: boolean
        default: false
      deployFoundrySurfaces:
        description: Plan or apply Foundry Hosted/Custom surface registration after tested agent images are available.
        required: false
        type: boolean
        default: false
      foundrySurfaceMode:
        description: Foundry surface registration mode. Use plan for dry-run artifacts; use apply to create/update Hosted Agent versions.
        required: false
        type: string
        default: plan
    secrets:
      AZURE_CLIENT_ID:
        required: true
      AZURE_TENANT_ID:
        required: true
      AZURE_SUBSCRIPTION_ID:
        required: true

permissions:
  id-token: write
  contents: write

env:
  FOUNDRY_RUNTIME_STRICT_ENFORCEMENT: "true"
  FOUNDRY_RUNTIME_AUTO_ENSURE_ON_STARTUP: "true"
  DEPLOY_SOURCE_SHA: ${{ inputs.sourceSha != '' && inputs.sourceSha || github.sha }}
  DEPLOY_SOURCE_REF: ${{ inputs.sourceRef != '' && inputs.sourceRef || github.ref }}
  DEPLOY_SOURCE_CHECKOUT_REF: ${{ inputs.sourceSha != '' && inputs.sourceSha || (inputs.sourceRef != '' && inputs.sourceRef || github.sha) }}
  REPOSITORY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
  FLUX_SOURCE_NAME: holiday-peak-gitops
  FLUX_SOURCE_NAMESPACE: flux-system

concurrency:
  group: deploy-azd-${{ inputs.environment }}
  cancel-in-progress: false

jobs:
  detect-changes:
    if: ${{ !inputs.uiOnly }}
    runs-on: ubuntu-latest
    outputs:
      crud_changed: ${{ steps.detect.outputs.crud_changed }}
      ui_changed: ${{ steps.detect.outputs.ui_changed }}
      agents_changed: ${{ steps.detect.outputs.agents_changed }}
      changed_agents_matrix: ${{ steps.detect.outputs.changed_agents_matrix }}
      changed_agent_services_csv: ${{ steps.detect.outputs.changed_agent_services_csv }}
      changed_aks_services_csv: ${{ steps.detect.outputs.changed_aks_services_csv }}
      changed_aks_matrix: ${{ steps.detect.outputs.changed_aks_matrix }}
      lib_changed: ${{ steps.detect.outputs.lib_changed }}
      infra_changed: ${{ steps.detect.outputs.infra_changed }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ env.DEPLOY_SOURCE_CHECKOUT_REF }}

      - name: Detect changed services
        id: detect
        shell: bash
        run: |
          set -euo pipefail

          DEFAULT_BRANCH="${{ github.event.repository.default_branch }}"
          TARGET_SHA="${DEPLOY_SOURCE_SHA}"
          echo "Detected deploy source SHA: ${TARGET_SHA}"
          echo "Detected deploy source ref: ${DEPLOY_SOURCE_REF}"
          git fetch origin "$DEFAULT_BRANCH" --depth=1 || true

          CHANGED_FILES=""
          DIFF_RANGE=""
          DIFF_STRATEGY=""

          if [ "${{ github.event_name }}" = "push" ] && [ -n "${{ github.event.before }}" ] && [ "${{ github.event.before }}" != "0000000000000000000000000000000000000000" ]; then
            if git cat-file -e "${{ github.event.before }}^{commit}" 2>/dev/null; then
              PUSH_BASE="${{ github.event.before }}"
              MERGE_BASE=$(git merge-base "$PUSH_BASE" "$TARGET_SHA" || true)
              if [ -n "$MERGE_BASE" ]; then
                DIFF_RANGE="$PUSH_BASE...$TARGET_SHA"
                DIFF_STRATEGY="push-before-three-dot"
              else
                DIFF_RANGE="$PUSH_BASE..$TARGET_SHA"
                DIFF_STRATEGY="push-before-two-dot-fallback"
                echo "::warning::No common ancestor between push before SHA and current SHA; using two-dot fallback diff range '$DIFF_RANGE'."
              fi
            else
              echo "::warning::Push before SHA '${{ github.event.before }}' is not available in local clone; falling back to default branch baseline."
            fi
          fi

          if [ -z "$DIFF_RANGE" ] && [ -n "${{ inputs.sourceSha }}" ] && [ -n "$TARGET_SHA" ] && git cat-file -e "${TARGET_SHA}^{commit}" 2>/dev/null; then
            if git rev-parse --verify "${TARGET_SHA}^1" >/dev/null 2>&1; then
              DIFF_RANGE="${TARGET_SHA}^1...${TARGET_SHA}"
              DIFF_STRATEGY="source-sha-first-parent-three-dot"
            fi
          fi

          if [ -z "$DIFF_RANGE" ]; then
            DEFAULT_REF="origin/$DEFAULT_BRANCH"
            if git rev-parse --verify "$DEFAULT_REF" >/dev/null 2>&1; then
              MERGE_BASE=$(git merge-base "$DEFAULT_REF" HEAD || true)
              if [ -n "$MERGE_BASE" ]; then
                DIFF_RANGE="$MERGE_BASE...HEAD"
                DIFF_STRATEGY="default-branch-merge-base-three-dot"
              else
                DIFF_RANGE="$DEFAULT_REF..HEAD"
                DIFF_STRATEGY="default-branch-two-dot-fallback"
                echo "::warning::No common ancestor with '$DEFAULT_REF'; using conservative fallback diff range '$DIFF_RANGE'."
              fi
            else
              DIFF_STRATEGY="full-tree-fallback"
              echo "::warning::Default branch reference '$DEFAULT_REF' is unavailable; treating tracked files as changed."
            fi
          fi

          if [ "$DIFF_STRATEGY" = "full-tree-fallback" ]; then
            CHANGED_FILES=$(git ls-files)
          else
            CHANGED_FILES=$(git diff --name-only "$DIFF_RANGE" || true)
          fi

          CHANGED_COUNT=$(printf '%s\n' "$CHANGED_FILES" | sed '/^$/d' | wc -l | xargs)
          echo "Changed-services detection strategy: $DIFF_STRATEGY"
          if [ -n "$DIFF_RANGE" ]; then
            echo "Changed-services diff range: $DIFF_RANGE"
          fi
          echo "Changed-services file count: $CHANGED_COUNT"

          mapfile -t AGENT_SERVICES < <(python3 - <<'PY'
          import re

          with open('azure.yaml', encoding='utf-8') as f:
            lines = f.readlines()

          in_services = False
          current_service = None
          current_host = None
          services = []

          for raw in lines:
            line = raw.rstrip('\n')
            if not in_services:
              if re.match(r'^services:\s*$', line):
                in_services = True
              continue

            if re.match(r'^[^\s]', line):
              break

            service_match = re.match(r'^  ([a-z0-9\-]+):\s*$', line)
            if service_match:
              if current_service and current_host == 'aks' and current_service != 'crud-service':
                services.append(current_service)
              current_service = service_match.group(1)
              current_host = None
              continue

            host_match = re.match(r'^    host:\s*(\S+)\s*$', line)
            if host_match:
              current_host = host_match.group(1)

          if current_service and current_host == 'aks' and current_service != 'crud-service':
            services.append(current_service)

          for service in services:
            print(service)
          PY
          )

          CRUD_CHANGED=false
          if echo "$CHANGED_FILES" | grep -Eq '^apps/crud-service/'; then
            CRUD_CHANGED=true
          fi

          UI_CHANGED=false
          if echo "$CHANGED_FILES" | grep -Eq '^apps/ui/'; then
            UI_CHANGED=true
          fi

          LIB_CHANGED=false
          if echo "$CHANGED_FILES" | grep -Eq '^lib/'; then
            LIB_CHANGED=true
          fi

          INFRA_CHANGED=false
          if echo "$CHANGED_FILES" | grep -Eq '^(\.infra/|azure\.yaml|\.kubernetes/)'; then
            INFRA_CHANGED=true
          fi

          AGENTS_CHANGED=false
          MATRIX='[]'
          CHANGED_AGENTS=()

          for service in "${AGENT_SERVICES[@]}"; do
            if echo "$CHANGED_FILES" | grep -Eq "^apps/$service/"; then
              CHANGED_AGENTS+=("$service")
            fi
          done

          # If lib/ changed, all agent services need rebuild
          if [ "$LIB_CHANGED" = true ]; then
            CHANGED_AGENTS=("${AGENT_SERVICES[@]}")
            CRUD_CHANGED=true
          fi

          # Apply service filter when provided (scoped deployments)
          SERVICE_FILTER="${{ inputs.serviceFilter }}"
          if [ -n "$SERVICE_FILTER" ]; then
            IFS=',' read -ra FILTER_LIST <<< "$SERVICE_FILTER"
            FILTERED_AGENTS=()
            for filter_svc in "${FILTER_LIST[@]}"; do
              trimmed=$(echo "$filter_svc" | xargs)
              for agent in "${AGENT_SERVICES[@]}"; do
                if [ "$agent" = "$trimmed" ]; then
                  FILTERED_AGENTS+=("$agent")
                fi
              done
            done
            CHANGED_AGENTS=("${FILTERED_AGENTS[@]}")
            # When a filter is active, do not auto-include CRUD unless it is in the filter
            CRUD_IN_FILTER=false
            for filter_svc in "${FILTER_LIST[@]}"; do
              trimmed=$(echo "$filter_svc" | xargs)
              if [ "$trimmed" = "crud-service" ]; then
                CRUD_IN_FILTER=true
              fi
            done
            if [ "$CRUD_IN_FILTER" = false ]; then
              CRUD_CHANGED=false
            else
              CRUD_CHANGED=true
            fi
            UI_CHANGED=false
            echo "Service filter applied: ${CHANGED_AGENTS[*]:-none}"
          fi

          if [ ${#CHANGED_AGENTS[@]} -gt 0 ]; then
            AGENTS_CHANGED=true
            MATRIX=$(printf '%s\n' "${CHANGED_AGENTS[@]}" | jq -Rcs 'split("\n")[:-1] | map(select(length > 0)) | map({service: .})')
          fi

          CHANGED_AGENT_SERVICES_CSV=""
          if [ ${#CHANGED_AGENTS[@]} -gt 0 ]; then
            CHANGED_AGENT_SERVICES_CSV=$(IFS=,; echo "${CHANGED_AGENTS[*]}")
          fi

          CHANGED_AKS_SERVICES_CSV="$CHANGED_AGENT_SERVICES_CSV"
          if [ "$CRUD_CHANGED" = true ]; then
            if [ -n "$CHANGED_AKS_SERVICES_CSV" ]; then
              CHANGED_AKS_SERVICES_CSV="crud-service,$CHANGED_AKS_SERVICES_CSV"
            else
              CHANGED_AKS_SERVICES_CSV="crud-service"
            fi
          fi

          CHANGED_AKS_MATRIX='[]'
          if [ -n "$CHANGED_AKS_SERVICES_CSV" ]; then
            CHANGED_AKS_MATRIX=$(printf '%s\n' "$CHANGED_AKS_SERVICES_CSV" | tr ',' '\n' | jq -Rcs 'split("\n")[:-1] | map(select(length > 0)) | map({service: .})')
          fi

          echo "crud_changed=$CRUD_CHANGED" >> "$GITHUB_OUTPUT"
          echo "ui_changed=$UI_CHANGED" >> "$GITHUB_OUTPUT"
          echo "agents_changed=$AGENTS_CHANGED" >> "$GITHUB_OUTPUT"
          printf 'changed_agents_matrix=%s\n' "$MATRIX" >> "$GITHUB_OUTPUT"
          echo "changed_agent_services_csv=$CHANGED_AGENT_SERVICES_CSV" >> "$GITHUB_OUTPUT"
          echo "changed_aks_services_csv=$CHANGED_AKS_SERVICES_CSV" >> "$GITHUB_OUTPUT"
          printf 'changed_aks_matrix=%s\n' "$CHANGED_AKS_MATRIX" >> "$GITHUB_OUTPUT"
          echo "lib_changed=$LIB_CHANGED" >> "$GITHUB_OUTPUT"
          echo "infra_changed=$INFRA_CHANGED" >> "$GITHUB_OUTPUT"

  provision:
    if: ${{ !inputs.uiOnly }}
    needs: detect-changes
    runs-on: ubuntu-latest
    environment: ${{ inputs.githubEnvironment }}
    outputs:
      AI_SERVICES_NAME: ${{ steps.outputs.outputs.AI_SERVICES_NAME }}
      AI_SEARCH_NAME: ${{ steps.outputs.outputs.AI_SEARCH_NAME }}
      AI_SEARCH_ENDPOINT: ${{ steps.outputs.outputs.AI_SEARCH_ENDPOINT }}
      AI_SEARCH_INDEX: ${{ steps.outputs.outputs.AI_SEARCH_INDEX }}
      AI_SEARCH_VECTOR_INDEX: ${{ steps.outputs.outputs.AI_SEARCH_VECTOR_INDEX }}
      AI_SEARCH_INDEXER_NAME: ${{ steps.outputs.outputs.AI_SEARCH_INDEXER_NAME }}
      EMBEDDING_DEPLOYMENT_NAME: ${{ steps.outputs.outputs.EMBEDDING_DEPLOYMENT_NAME }}
      AI_SEARCH_AUTH_MODE: ${{ steps.outputs.outputs.AI_SEARCH_AUTH_MODE }}
      PROJECT_ENDPOINT: ${{ steps.outputs.outputs.PROJECT_ENDPOINT }}
      PROJECT_NAME: ${{ steps.outputs.outputs.PROJECT_NAME }}
      APIM_GATEWAY_URL: ${{ steps.outputs.outputs.APIM_GATEWAY_URL }}
      COSMOS_ACCOUNT_URI: ${{ steps.outputs.outputs.COSMOS_ACCOUNT_URI }}
      COSMOS_DATABASE: ${{ steps.outputs.outputs.COSMOS_DATABASE }}
      COSMOS_CONTAINER: ${{ steps.outputs.outputs.COSMOS_CONTAINER }}
      KEY_VAULT_URI: ${{ steps.outputs.outputs.KEY_VAULT_URI }}
      REDIS_HOST: ${{ steps.outputs.outputs.REDIS_HOST }}
      EVENT_HUB_NAMESPACE: ${{ steps.outputs.outputs.EVENT_HUB_NAMESPACE }}
      APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ steps.outputs.outputs.APPLICATIONINSIGHTS_CONNECTION_STRING }}
      BLOB_ACCOUNT_URL: ${{ steps.outputs.outputs.BLOB_ACCOUNT_URL }}
      BLOB_CONTAINER: ${{ steps.outputs.outputs.BLOB_CONTAINER }}
      POSTGRES_HOST: ${{ steps.outputs.outputs.POSTGRES_HOST }}
      POSTGRES_USER: ${{ steps.outputs.outputs.POSTGRES_USER }}
      POSTGRES_ADMIN_USER: ${{ steps.outputs.outputs.POSTGRES_ADMIN_USER }}
      POSTGRES_AUTH_MODE: ${{ steps.outputs.outputs.POSTGRES_AUTH_MODE }}
      POSTGRES_DATABASE: ${{ steps.outputs.outputs.POSTGRES_DATABASE }}
      AGC_SUPPORT_ENABLED: ${{ steps.outputs.outputs.AGC_SUPPORT_ENABLED }}
      AGC_GATEWAY_CLASS: ${{ steps.outputs.outputs.AGC_GATEWAY_CLASS }}
      AGC_SUBNET_ID: ${{ steps.outputs.outputs.AGC_SUBNET_ID }}
      AGC_FRONTEND_HOSTNAME: ${{ steps.outputs.outputs.AGC_FRONTEND_HOSTNAME }}
      AGC_FRONTEND_REFERENCE: ${{ steps.outputs.outputs.AGC_FRONTEND_REFERENCE }}
      AGC_CONTROLLER_DEPLOYMENT_MODE: ${{ steps.outputs.outputs.AGC_CONTROLLER_DEPLOYMENT_MODE }}
      AGC_CONTROLLER_IDENTITY_NAME: ${{ steps.outputs.outputs.AGC_CONTROLLER_IDENTITY_NAME }}
      AGC_CONTROLLER_IDENTITY_CLIENT_ID: ${{ steps.outputs.outputs.AGC_CONTROLLER_IDENTITY_CLIENT_ID }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      CHANGED_AGENT_SERVICES: ${{ needs.detect-changes.outputs.changed_agent_services_csv }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ env.DEPLOY_SOURCE_CHECKOUT_REF }}

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup azd
        uses: Azure/setup-azd@v2

      - name: Authenticate azd (OIDC)
        run: |
          azd auth login \
            --client-id "${AZURE_CLIENT_ID}" \
            --tenant-id "${AZURE_TENANT_ID}" \
            --federated-credential-provider github \
            --no-prompt

      - name: Configure azd environment context
        run: |
          azd env set AZURE_SUBSCRIPTION_ID "${AZURE_SUBSCRIPTION_ID}" -e "${{ inputs.environment }}"
          azd env set AZURE_LOCATION "${{ inputs.location }}" -e "${{ inputs.environment }}"
          azd env set AZURE_ENV_NAME "${{ inputs.environment }}" -e "${{ inputs.environment }}"
          azd env set AZURE_RESOURCE_GROUP "${{ inputs.projectName }}-${{ inputs.environment }}-rg" -e "${{ inputs.environment }}"
          azd env set resourceGroupName "${{ inputs.projectName }}-${{ inputs.environment }}-rg" -e "${{ inputs.environment }}"
          azd env set AZURE_AKS_CLUSTER_NAME "${{ inputs.projectName }}-${{ inputs.environment }}-aks" -e "${{ inputs.environment }}"
          azd env set AKS_CLUSTER_NAME "${{ inputs.projectName }}-${{ inputs.environment }}-aks" -e "${{ inputs.environment }}"
          azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT "${{ inputs.projectName }}${{ inputs.environment }}acr.azurecr.io" -e "${{ inputs.environment }}"
          azd env set deployShared true -e "${{ inputs.environment }}"
          azd env set deployStatic "${{ inputs.deployStatic }}" -e "${{ inputs.environment }}"
          azd env set environment "${{ inputs.environment }}" -e "${{ inputs.environment }}"
          azd env set location "${{ inputs.location }}" -e "${{ inputs.environment }}"
          azd env set projectName "${{ inputs.projectName }}" -e "${{ inputs.environment }}"
          azd env set IMAGE_PREFIX "ghcr.io/${{ github.repository_owner }}" -e "${{ inputs.environment }}"
          azd env set IMAGE_TAG "${{ inputs.imageTag }}" -e "${{ inputs.environment }}"
          azd env set K8S_NAMESPACE holiday-peak -e "${{ inputs.environment }}"
          azd env set K8S_CRUD_NAMESPACE holiday-peak-crud -e "${{ inputs.environment }}"
          azd env set K8S_AGENTS_NAMESPACE holiday-peak-agents -e "${{ inputs.environment }}"
          azd env set KEDA_ENABLED true -e "${{ inputs.environment }}"
          # PostgreSQL auth: use password for dev until Entra AD admin is configured
          # in Bicep (administrators param on AVM PostgreSQL module). Enabling
          # activeDirectoryAuth without an admin has no effect — see ADR-017 follow-up.
          azd env set POSTGRES_AUTH_MODE password -e "${{ inputs.environment }}"

      - name: Install kubelogin
        run: |
          for attempt in 1 2 3 4 5; do
            if az aks install-cli --kubelogin-version latest; then
              exit 0
            fi
            echo "kubelogin install failed on attempt ${attempt}; retrying..."
            sleep 15
          done
          echo "kubelogin install failed after retries"
          exit 1

      - name: Ensure hook scripts executable
        run: chmod +x ./.infra/azd/hooks/*.sh

      - name: Ensure AKS cluster is running
        run: |
          AKS_RG="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          AKS_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-aks"

          if az aks show --resource-group "$AKS_RG" --name "$AKS_NAME" >/dev/null 2>&1; then
            POWER_STATE=$(az aks show --resource-group "$AKS_RG" --name "$AKS_NAME" --query "powerState.code" -o tsv 2>/dev/null || true)
            if [ "$POWER_STATE" = "Stopped" ]; then
              echo "AKS cluster is stopped. Starting $AKS_NAME..."
              az aks start --resource-group "$AKS_RG" --name "$AKS_NAME"
              for _ in $(seq 1 40); do
                CURRENT_STATE=$(az aks show --resource-group "$AKS_RG" --name "$AKS_NAME" --query "powerState.code" -o tsv 2>/dev/null || true)
                if [ "$CURRENT_STATE" = "Running" ]; then
                  echo "AKS cluster is running."
                  break
                fi
                sleep 30
              done

              FINAL_STATE=$(az aks show --resource-group "$AKS_RG" --name "$AKS_NAME" --query "powerState.code" -o tsv 2>/dev/null || true)
              if [ "$FINAL_STATE" != "Running" ]; then
                echo "AKS cluster failed to reach Running state (current: ${FINAL_STATE:-unknown})." >&2
                exit 1
              fi
            fi
          else
            echo "AKS cluster does not exist yet. Continuing."
          fi

      - name: Ensure deploy principal has AKS RBAC Cluster Admin
        shell: bash
        run: |
          set -euo pipefail

          AKS_RG="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          DEPLOY_SP_OBJECT_ID=$(az ad sp show --id "${AZURE_CLIENT_ID}" --query id -o tsv)
          RG_ID=$(az group show --name "$AKS_RG" --query id -o tsv)

          EXISTING=$(az role assignment list \
            --scope "$RG_ID" \
            --assignee-object-id "$DEPLOY_SP_OBJECT_ID" \
            --query "[?roleDefinitionName=='Azure Kubernetes Service RBAC Cluster Admin'] | length(@)" -o tsv)

          if [ "$EXISTING" = "0" ]; then
            echo "Assigning Azure Kubernetes Service RBAC Cluster Admin to deploy principal at RG scope $AKS_RG."
            az role assignment create \
              --assignee-object-id "$DEPLOY_SP_OBJECT_ID" \
              --assignee-principal-type ServicePrincipal \
              --role "Azure Kubernetes Service RBAC Cluster Admin" \
              --scope "$RG_ID"
          else
            echo "Deploy principal already has Azure Kubernetes Service RBAC Cluster Admin at RG scope $AKS_RG."
          fi

          VERIFIED=$(az role assignment list \
            --scope "$RG_ID" \
            --assignee-object-id "$DEPLOY_SP_OBJECT_ID" \
            --query "[?roleDefinitionName=='Azure Kubernetes Service RBAC Cluster Admin'] | length(@)" -o tsv)

          if [ "$VERIFIED" = "0" ]; then
            echo "Failed to verify Azure Kubernetes Service RBAC Cluster Admin assignment for deploy principal at scope $AKS_RG."
            exit 1
          fi

      - name: Preflight drift remediation (non-prod)
        if: ${{ !inputs.skipProvision && inputs.environment != 'prod' && inputs.environment != 'production' }}
        shell: bash
        run: |
          set -euo pipefail

          RG_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          KEY_VAULT_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-kv"
          POSTGRES_SERVER_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-postgres"
          DESIRED_LOCATION="$(echo "${{ inputs.location }}" | tr '[:upper:]' '[:lower:]')"

          echo "Running non-prod preflight drift remediation for environment '${{ inputs.environment }}'."

          # Remediate soft-deleted Key Vault name conflicts that can block azd provision.
          if az keyvault show --name "$KEY_VAULT_NAME" --resource-group "$RG_NAME" >/dev/null 2>&1; then
            CURRENT_KV_LOCATION=$(az keyvault show \
              --name "$KEY_VAULT_NAME" \
              --resource-group "$RG_NAME" \
              --query location -o tsv | tr '[:upper:]' '[:lower:]')

            if [ "$CURRENT_KV_LOCATION" != "$DESIRED_LOCATION" ]; then
              LOCATION_SUFFIX=$(echo "$DESIRED_LOCATION" | tr -cd '[:alnum:]' | cut -c1-4)
              KEY_VAULT_OVERRIDE=$(echo "${{ inputs.projectName }}-${{ inputs.environment }}-kv-${LOCATION_SUFFIX}" | tr '[:upper:]' '[:lower:]' | cut -c1-24)
              KEY_VAULT_OVERRIDE="${KEY_VAULT_OVERRIDE%-}"

              echo "Key Vault location mismatch detected for $KEY_VAULT_NAME ($CURRENT_KV_LOCATION vs $DESIRED_LOCATION)."
              echo "Setting keyVaultNameOverride to $KEY_VAULT_OVERRIDE for this deployment."
              azd env set keyVaultNameOverride "$KEY_VAULT_OVERRIDE" -e "${{ inputs.environment }}"
            else
              echo "Key Vault $KEY_VAULT_NAME already exists in $RG_NAME with matching location $CURRENT_KV_LOCATION."
            fi
          else
            DELETED_KV_LOCATION=$(az keyvault list-deleted \
              --query "[?name=='$KEY_VAULT_NAME'] | [0].properties.location" \
              -o tsv)

            if [ -n "$DELETED_KV_LOCATION" ]; then
              echo "Recovering soft-deleted Key Vault $KEY_VAULT_NAME."
              if az keyvault recover --name "$KEY_VAULT_NAME" >/dev/null 2>&1; then
                echo "Recovery initiated for $KEY_VAULT_NAME."
              else
                echo "Recovery failed for $KEY_VAULT_NAME. Attempting purge fallback from $DELETED_KV_LOCATION."
                az keyvault purge --name "$KEY_VAULT_NAME" --location "$DELETED_KV_LOCATION"
              fi

              for _ in $(seq 1 20); do
                if az keyvault show --name "$KEY_VAULT_NAME" --resource-group "$RG_NAME" >/dev/null 2>&1; then
                  echo "Key Vault $KEY_VAULT_NAME is now available in $RG_NAME."
                  break
                fi
                sleep 10
              done
            else
              echo "No soft-deleted Key Vault conflict found for $KEY_VAULT_NAME."
            fi
          fi

          # Start stopped PostgreSQL Flexible Server so provisioning can reconcile state.
          if az postgres flexible-server show --resource-group "$RG_NAME" --name "$POSTGRES_SERVER_NAME" >/dev/null 2>&1; then
            POSTGRES_STATE=$(az postgres flexible-server show \
              --resource-group "$RG_NAME" \
              --name "$POSTGRES_SERVER_NAME" \
              --query state -o tsv)

            if [ "$POSTGRES_STATE" = "Stopped" ]; then
              echo "PostgreSQL server $POSTGRES_SERVER_NAME is stopped. Starting..."
              az postgres flexible-server start --resource-group "$RG_NAME" --name "$POSTGRES_SERVER_NAME"

              POSTGRES_READY=false
              for _ in $(seq 1 40); do
                CURRENT_STATE=$(az postgres flexible-server show \
                  --resource-group "$RG_NAME" \
                  --name "$POSTGRES_SERVER_NAME" \
                  --query state -o tsv)
                if [ "$CURRENT_STATE" = "Ready" ]; then
                  echo "PostgreSQL server is ready."
                  POSTGRES_READY=true
                  break
                fi
                sleep 15
              done

              if [ "$POSTGRES_READY" != "true" ]; then
                echo "PostgreSQL server $POSTGRES_SERVER_NAME did not reach Ready state in time."
                exit 1
              fi
            else
              echo "PostgreSQL server $POSTGRES_SERVER_NAME state is $POSTGRES_STATE. No start needed."
            fi
          else
            echo "PostgreSQL server $POSTGRES_SERVER_NAME does not exist yet. Continuing."
          fi

      - name: Validate truth Event Hubs IaC declarations
        if: ${{ !inputs.skipProvision }}
        shell: bash
        run: |
          set -euo pipefail

          python3 <<'PY'
          import re
          from pathlib import Path

          bicep_path = Path('.infra/modules/shared-infrastructure/shared-infrastructure.bicep')
          bicep = bicep_path.read_text(encoding='utf-8')

          expected_event_hub_parents = {
            'searchEnrichmentJobsEventHub': 'platformJobsNamespaceResource',
            'ingestJobsEventHub': 'platformJobsNamespaceResource',
            'enrichmentJobsEventHub': 'platformJobsNamespaceResource',
            'completenessJobsEventHub': 'platformJobsNamespaceResource',
            'exportJobsEventHub': 'platformJobsNamespaceResource',
            'hitlJobsEventHub': 'platformJobsNamespaceResource',
          }
          expected_consumer_group_parents = {
            'ingestionGroupConsumerGroup': 'ingestJobsEventHub',
            'enrichmentEngineConsumerGroup': 'enrichmentJobsEventHub',
            'completenessEngineConsumerGroup': 'completenessJobsEventHub',
            'exportEngineConsumerGroup': 'exportJobsEventHub',
            'hitlServiceConsumerGroup': 'hitlJobsEventHub',
            'searchEnrichmentAgentConsumerGroup': 'searchEnrichmentJobsEventHub',
            'searchEnrichmentConsumerGroup': 'searchEnrichmentJobsEventHub',
          }

          failures = []
          for resource_name, parent_name in expected_event_hub_parents.items():
            pattern = rf"resource\s+{resource_name}\s+'Microsoft\.EventHub/namespaces/eventhubs@2024-01-01'\s+existing\s*=\s*\{{\s*parent:\s*{parent_name}\s*"
            if not re.search(pattern, bicep, flags=re.MULTILINE):
              failures.append(f'{resource_name} -> {parent_name}')

          for resource_name, parent_name in expected_consumer_group_parents.items():
            pattern = rf"resource\s+{resource_name}\s+'Microsoft\.EventHub/namespaces/eventhubs/consumergroups@2024-01-01'\s*=\s*\{{\s*parent:\s*{parent_name}\s*"
            if not re.search(pattern, bicep, flags=re.MULTILINE):
              failures.append(f'{resource_name} -> {parent_name}')

          if failures:
            raise SystemExit(
              'Platform jobs Event Hubs namespace ownership drift detected in '
              f'{bicep_path}: ' + ', '.join(failures)
            )

          print('Platform jobs Event Hubs namespace ownership matches the dual-namespace contract.')
          PY

      - name: Validate app dependency manifest coverage
        if: ${{ !inputs.skipProvision }}
        shell: bash
        run: |
          set -euo pipefail

          python3 .infra/validate_app_dependency_manifest.py \
            --azure-config azure.yaml \
            --manifest .infra/app-dependency-manifest.yaml

      - name: Validate Event Hubs IaC declarations
        if: ${{ !inputs.skipProvision }}
        shell: bash
        run: |
          set -euo pipefail

          BICEP_FILE=".infra/modules/shared-infrastructure/shared-infrastructure.bicep"
          MISSING=0

          for hub in order-events product-events return-events inventory-events shipment-events payment-events user-events ingest-jobs enrichment-jobs completeness-jobs export-jobs hitl-jobs search-enrichment-jobs; do
            if ! grep -q "'$hub'" "$BICEP_FILE"; then
              echo "Missing required Event Hub declaration in $BICEP_FILE: $hub" >&2
              MISSING=1
            fi
          done

          for group in ingestion-group enrichment-engine export-engine hitl-service search-enrichment-consumer search-enrichment-agent catalog-search-group enrichment-group acp-transform-group normalization-group assortment-group validation-group completeness-engine support-group returns-group; do
            if ! grep -q "'$group'" "$BICEP_FILE"; then
              echo "Missing required consumer group declaration in $BICEP_FILE: $group" >&2
              MISSING=1
            fi
          done

          if [ "$MISSING" -ne 0 ]; then
            if [ "${{ inputs.environment }}" = "prod" ] || [ "${{ inputs.environment }}" = "production" ]; then
              echo "Event Hub IaC validation failed in production environment." >&2
              exit 1
            fi
            echo "Proceeding despite Event Hub IaC declaration warnings in non-production environment." >&2
          fi

          echo "Event Hub and consumer group declarations are present in IaC."

      - name: Validate skip-provision mode
        if: ${{ inputs.skipProvision }}
        shell: bash
        run: |
          set -euo pipefail

          INFRA_CHANGED="${{ needs.detect-changes.outputs.infra_changed }}"

          if [ "$INFRA_CHANGED" = "true" ]; then
            echo "::warning::skipProvision is active but infrastructure files changed. Provisioning will be skipped. If you need infra reconciliation, rerun with skipProvision=false."
          fi

          echo "skipProvision=true: reusing the current azd environment values and existing infrastructure for this deploy."

      - name: Provision infrastructure
        if: ${{ !inputs.skipProvision }}
        shell: bash
        run: |
          set -euo pipefail

          PROVISION_LOG="$(mktemp)"

          if azd provision --no-prompt -e "${{ inputs.environment }}" 2>&1 | tee "$PROVISION_LOG"; then
            exit 0
          fi

          if [ "${{ inputs.environment }}" != "prod" ] && [ "${{ inputs.environment }}" != "production" ] \
            && grep -q "RoleAssignmentExists" "$PROVISION_LOG" \
            && ! grep -q "BadRequest:" "$PROVISION_LOG"; then
            echo "Non-prod provision hit an idempotent RoleAssignmentExists conflict." >&2
            echo "Infrastructure was deployed but azd did not save outputs. Running azd env refresh to populate them." >&2
            azd env refresh -e '${{ inputs.environment }}' --no-prompt
            echo "azd env refresh completed after RoleAssignmentExists fallback."
            exit 0
          fi

          echo "Provision infrastructure failed with non-ignorable errors." >&2
          exit 1

      - name: Ensure deploy principal has AcrPush on ACR
        shell: bash
        run: |
          set -euo pipefail

          RG_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          ACR_NAME=$(echo "${{ inputs.projectName }}${{ inputs.environment }}acr" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]')
          DEPLOY_SP_OBJECT_ID=$(az ad sp show --id "${AZURE_CLIENT_ID}" --query id -o tsv)
          ACR_ID=$(az acr show --resource-group "$RG_NAME" --name "$ACR_NAME" --query id -o tsv)

          EXISTING=$(az role assignment list \
            --scope "$ACR_ID" \
            --assignee-object-id "$DEPLOY_SP_OBJECT_ID" \
            --query "[?roleDefinitionName=='AcrPush'] | length(@)" -o tsv)

          if [ "$EXISTING" = "0" ]; then
            echo "Assigning AcrPush to deploy principal on ACR $ACR_NAME."
            az role assignment create \
              --assignee-object-id "$DEPLOY_SP_OBJECT_ID" \
              --assignee-principal-type ServicePrincipal \
              --role "AcrPush" \
              --scope "$ACR_ID"
          else
            echo "Deploy principal already has AcrPush on ACR $ACR_NAME."
          fi

          VERIFIED=$(az role assignment list \
            --scope "$ACR_ID" \
            --assignee-object-id "$DEPLOY_SP_OBJECT_ID" \
            --query "[?roleDefinitionName=='AcrPush'] | length(@)" -o tsv)

          if [ "$VERIFIED" = "0" ]; then
            echo "Failed to verify AcrPush assignment for deploy principal on scope $ACR_ID."
            exit 1
          fi

      - name: Ensure AKS identity can read Key Vault secrets
        run: |
          AKS_RG="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          AKS_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-aks"
          KEY_VAULT_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-kv"

          KUBELET_OBJECT_ID=$(az aks show \
            --resource-group "$AKS_RG" \
            --name "$AKS_NAME" \
            --query "identityProfile.kubeletidentity.objectId" -o tsv)

          KEY_VAULT_ID=$(az keyvault show \
            --resource-group "$AKS_RG" \
            --name "$KEY_VAULT_NAME" \
            --query id -o tsv)

          EXISTING=$(az role assignment list \
            --assignee-object-id "$KUBELET_OBJECT_ID" \
            --scope "$KEY_VAULT_ID" \
            --query "[?roleDefinitionName=='Key Vault Secrets User'] | length(@)" -o tsv)

          if [ "$EXISTING" = "0" ]; then
            az role assignment create \
              --assignee-object-id "$KUBELET_OBJECT_ID" \
              --assignee-principal-type ServicePrincipal \
              --role "Key Vault Secrets User" \
              --scope "$KEY_VAULT_ID"
          fi

      - name: Refresh azd environment from deployed infrastructure
        if: ${{ inputs.skipProvision }}
        shell: bash
        run: |
          set -euo pipefail
          echo "skipProvision=true: refreshing azd environment outputs from the last successful deployment."
          azd env refresh -e '${{ inputs.environment }}' --no-prompt
          echo "azd env refresh completed."

      # ── Recover outputs when ARM deployment state is Failed ──────────
      # azd env refresh silently returns no values when the ARM deployment
      # is in a Failed state (e.g. RoleAssignmentExists conflicts cause
      # the deployment to be marked Failed even though resources deployed).
      # This step validates critical outputs and queries Azure directly
      # for any that are missing.
      - name: Validate and recover provisioned outputs
        shell: bash
        run: |
          set -euo pipefail
          ENV="${{ inputs.environment }}"
          RG="${{ inputs.projectName }}-${ENV}-rg"

          # Install ALB CLI extension for AGC recovery queries
          az extension add --name alb --only-show-errors 2>/dev/null || true

          # Load current azd env values
          eval "$(azd env get-values -e "$ENV" | sed 's/^/export /')"

          RECOVERED=0
          AGC_RECOVERED=0

          set_if_empty() {
            local k="$1" v="$2"
            azd env set "$k" "$v" -e "$ENV"
            export "${k}=${v}"
            echo "  ✓ Recovered ${k}"
            RECOVERED=$((RECOVERED + 1))
          }

          # ── PostgreSQL ─────────────────────────────────────────────
          if [ -z "${POSTGRES_HOST:-}" ]; then
            V=$(az postgres flexible-server list -g "$RG" \
              --query "[0].fullyQualifiedDomainName" -o tsv 2>/dev/null || true)
            [ -n "${V:-}" ] && set_if_empty POSTGRES_HOST "$V"
          fi

          if [ -z "${POSTGRES_ADMIN_USER:-}" ]; then
            V=$(az postgres flexible-server list -g "$RG" \
              --query "[0].administratorLogin" -o tsv 2>/dev/null || true)
            [ -n "${V:-}" ] && set_if_empty POSTGRES_ADMIN_USER "$V"
          fi

          if [ -z "${POSTGRES_DATABASE:-}" ]; then
            set_if_empty POSTGRES_DATABASE "holiday_peak_crud"
          fi

          if [ -z "${POSTGRES_AUTH_MODE:-}" ]; then
            set_if_empty POSTGRES_AUTH_MODE "password"
          fi

          if [ -z "${POSTGRES_USER:-}" ]; then
            if [ "${POSTGRES_AUTH_MODE:-password}" = "password" ]; then
              set_if_empty POSTGRES_USER "${POSTGRES_ADMIN_USER:-crud_admin}"
            else
              set_if_empty POSTGRES_USER "${{ inputs.projectName }}-${ENV}-crud-identity"
            fi
          fi

          # ── Cosmos DB ──────────────────────────────────────────────
          if [ -z "${COSMOS_ACCOUNT_URI:-}" ]; then
            V=$(az cosmosdb list -g "$RG" \
              --query "[0].documentEndpoint" -o tsv 2>/dev/null || true)
            [ -n "${V:-}" ] && set_if_empty COSMOS_ACCOUNT_URI "$V"
          fi

          if [ -z "${COSMOS_DATABASE:-}" ]; then
            set_if_empty COSMOS_DATABASE "holiday-peak-db"
          fi

          # ── Key Vault ──────────────────────────────────────────────
          if [ -z "${KEY_VAULT_URI:-}" ]; then
            V=$(az keyvault list -g "$RG" \
              --query "[0].properties.vaultUri" -o tsv 2>/dev/null || true)
            [ -n "${V:-}" ] && set_if_empty KEY_VAULT_URI "$V"
          fi

          # ── Redis ──────────────────────────────────────────────────
          if [ -z "${REDIS_HOST:-}" ]; then
            V=$(az redis list -g "$RG" \
              --query "[0].name" -o tsv 2>/dev/null || true)
            [ -n "${V:-}" ] && set_if_empty REDIS_HOST "$V"
          fi

          # ── Event Hubs (main namespace, not platform-jobs) ─────────
          if [ -z "${EVENT_HUB_NAMESPACE:-}" ]; then
            V=$(az eventhubs namespace list -g "$RG" \
              --query "[?ends_with(name, '-eventhub')].name | [0]" -o tsv 2>/dev/null || true)
            [ -n "${V:-}" ] && set_if_empty EVENT_HUB_NAMESPACE "$V"
          fi

          # ── Application Insights ───────────────────────────────────
          if [ -z "${APPLICATIONINSIGHTS_CONNECTION_STRING:-}" ]; then
            V=$(az monitor app-insights component list -g "$RG" \
              --query "[0].connectionString" -o tsv 2>/dev/null || true)
            [ -n "${V:-}" ] && set_if_empty APPLICATIONINSIGHTS_CONNECTION_STRING "$V"
          fi

          # ── Storage ────────────────────────────────────────────────
          if [ -z "${BLOB_ACCOUNT_URL:-}" ]; then
            V=$(az storage account list -g "$RG" \
              --query "[0].name" -o tsv 2>/dev/null || true)
            if [ -n "${V:-}" ]; then
              set_if_empty BLOB_ACCOUNT_URL "https://${V}.blob.core.windows.net"
            fi
          fi

          # ── AI Search ──────────────────────────────────────────────
          if [ -z "${AI_SEARCH_NAME:-}" ]; then
            V=$(az search service list -g "$RG" \
              --query "[0].name" -o tsv 2>/dev/null || true)
            [ -n "${V:-}" ] && set_if_empty AI_SEARCH_NAME "$V"
          fi

          if [ -z "${AI_SEARCH_ENDPOINT:-}" ] && [ -n "${AI_SEARCH_NAME:-}" ]; then
            set_if_empty AI_SEARCH_ENDPOINT "https://${AI_SEARCH_NAME}.search.windows.net"
          fi

          if [ -z "${AI_SEARCH_INDEX:-}" ]; then
            set_if_empty AI_SEARCH_INDEX "catalog-products"
          fi

          if [ -z "${AI_SEARCH_VECTOR_INDEX:-}" ]; then
            set_if_empty AI_SEARCH_VECTOR_INDEX "product_search_index"
          fi

          if [ -z "${AI_SEARCH_INDEXER_NAME:-}" ]; then
            set_if_empty AI_SEARCH_INDEXER_NAME "search-enriched-products-indexer"
          fi

          if [ -z "${EMBEDDING_DEPLOYMENT_NAME:-}" ]; then
            set_if_empty EMBEDDING_DEPLOYMENT_NAME "text-embedding-3-large"
          fi

          if [ -z "${AI_SEARCH_AUTH_MODE:-}" ]; then
            set_if_empty AI_SEARCH_AUTH_MODE "managed_identity"
          fi

          # ── AI Services ────────────────────────────────────────────
          if [ -z "${AI_SERVICES_NAME:-}" ]; then
            V=$(az cognitiveservices account list -g "$RG" \
              --query "[?kind=='AIServices'] | [0].name" -o tsv 2>/dev/null || true)
            [ -n "${V:-}" ] && set_if_empty AI_SERVICES_NAME "$V"
          fi

          # ── AI Project ─────────────────────────────────────────────
          if [ -z "${PROJECT_NAME:-}" ]; then
            V=$(az resource list -g "$RG" \
              --resource-type "Microsoft.MachineLearningServices/workspaces" \
              --query "[?kind=='Project'] | [0].name" -o tsv 2>/dev/null || true)
            [ -n "${V:-}" ] && set_if_empty PROJECT_NAME "$V"
          fi

          if [ -z "${PROJECT_ENDPOINT:-}" ] && [ -n "${AI_SERVICES_NAME:-}" ] && [ -n "${PROJECT_NAME:-}" ]; then
            set_if_empty PROJECT_ENDPOINT \
              "https://${AI_SERVICES_NAME}.services.ai.azure.com/api/projects/${PROJECT_NAME}"
          fi

          # ── Application Gateway for Containers (AGC) ──────────────
          # AGC outputs are critical for validate-agc-readiness, sync-apim,
          # and sync-apic jobs. Recover from Azure CLI when azd env refresh
          # returns empty values due to ARM deployment state being Failed.
          VNET_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-vnet"

          # Detect AGC support by checking for the agc subnet
          if [ -z "${AGC_SUPPORT_ENABLED:-}" ] || [ "${AGC_SUPPORT_ENABLED}" != "true" ]; then
            AGC_SUBNET_EXISTS=$(az network vnet subnet show \
              -g "$RG" --vnet-name "$VNET_NAME" -n agc \
              --query "name" -o tsv 2>/dev/null || true)
            if [ -n "${AGC_SUBNET_EXISTS:-}" ]; then
              set_if_empty AGC_SUPPORT_ENABLED "true"
              AGC_RECOVERED=$((AGC_RECOVERED + 1))
            fi
          fi

          if [ "${AGC_SUPPORT_ENABLED:-}" = "true" ]; then
            # Deterministic keys
            if [ -z "${AGC_GATEWAY_CLASS:-}" ]; then
              set_if_empty AGC_GATEWAY_CLASS "azure-alb-external"
              AGC_RECOVERED=$((AGC_RECOVERED + 1))
            fi

            if [ -z "${AGC_FRONTEND_REFERENCE:-}" ]; then
              set_if_empty AGC_FRONTEND_REFERENCE "gateway-class:azure-alb-external"
              AGC_RECOVERED=$((AGC_RECOVERED + 1))
            fi

            if [ -z "${AGC_CONTROLLER_DEPLOYMENT_MODE:-}" ]; then
              set_if_empty AGC_CONTROLLER_DEPLOYMENT_MODE "helm-workload-identity"
              AGC_RECOVERED=$((AGC_RECOVERED + 1))
            fi

            # Subnet ID via CLI query
            if [ -z "${AGC_SUBNET_ID:-}" ]; then
              V=$(az network vnet subnet show \
                -g "$RG" --vnet-name "$VNET_NAME" -n agc \
                --query "id" -o tsv 2>/dev/null || true)
              [ -n "${V:-}" ] && { set_if_empty AGC_SUBNET_ID "$V"; AGC_RECOVERED=$((AGC_RECOVERED + 1)); }
            fi

            # Controller identity (naming convention: {project}-{env}-agc-controller)
            AGC_IDENTITY_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-agc-controller"
            if [ -z "${AGC_CONTROLLER_IDENTITY_NAME:-}" ]; then
              V=$(az identity show -g "$RG" -n "$AGC_IDENTITY_NAME" \
                --query "name" -o tsv 2>/dev/null || true)
              [ -n "${V:-}" ] && { set_if_empty AGC_CONTROLLER_IDENTITY_NAME "$V"; AGC_RECOVERED=$((AGC_RECOVERED + 1)); }
            fi

            if [ -z "${AGC_CONTROLLER_IDENTITY_CLIENT_ID:-}" ]; then
              V=$(az identity show -g "$RG" -n "$AGC_IDENTITY_NAME" \
                --query "clientId" -o tsv 2>/dev/null || true)
              [ -n "${V:-}" ] && { set_if_empty AGC_CONTROLLER_IDENTITY_CLIENT_ID "$V"; AGC_RECOVERED=$((AGC_RECOVERED + 1)); }
            fi

            # Frontend hostname via ALB CLI. Prefer the environment RG, then fall back
            # to the AKS node RG where ALB controller-managed resources may live.
            if [ -z "${AGC_FRONTEND_HOSTNAME:-}" ]; then
              AKS_CLUSTER_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-aks"
              AKS_NODE_RESOURCE_GROUP=$(az aks show -g "$RG" -n "$AKS_CLUSTER_NAME" \
                --query "nodeResourceGroup" -o tsv 2>/dev/null || true)
              declare -a ALB_RESOURCE_GROUPS=("$RG")
              if [ -n "${AKS_NODE_RESOURCE_GROUP:-}" ] && [ "$AKS_NODE_RESOURCE_GROUP" != "$RG" ]; then
                ALB_RESOURCE_GROUPS+=("$AKS_NODE_RESOURCE_GROUP")
              fi

              ALB_FOUND=false
              for ALB_RESOURCE_GROUP in "${ALB_RESOURCE_GROUPS[@]}"; do
                ALB_NAME=$(az network alb list -g "$ALB_RESOURCE_GROUP" \
                  --query "[0].name" -o tsv 2>/dev/null || true)
                if [ -z "${ALB_NAME:-}" ]; then
                  continue
                fi

                ALB_FOUND=true
                V=$(az network alb frontend list -g "$ALB_RESOURCE_GROUP" --alb-name "$ALB_NAME" \
                  --query "[0].fqdn" -o tsv 2>/dev/null || true)
                if [ -n "${V:-}" ]; then
                  set_if_empty AGC_FRONTEND_HOSTNAME "$V"
                  AGC_RECOVERED=$((AGC_RECOVERED + 1))
                  echo "Recovered AGC_FRONTEND_HOSTNAME from ALB '$ALB_NAME' in resource group '$ALB_RESOURCE_GROUP'."
                  break
                fi

                echo "::notice::AGC frontend on ALB '$ALB_NAME' in $ALB_RESOURCE_GROUP is not populated yet. Non-fatal; validate-agc-readiness will gate downstream jobs."
              done

              if [ -z "${AGC_FRONTEND_HOSTNAME:-}" ] && [ "$ALB_FOUND" = false ]; then
                echo "::notice::No ALB resource found in $RG${AKS_NODE_RESOURCE_GROUP:+ or $AKS_NODE_RESOURCE_GROUP} — AGC_FRONTEND_HOSTNAME cannot be recovered. Non-fatal; validate-agc-readiness will gate downstream jobs."
              fi
            fi

            if [ "$AGC_RECOVERED" -gt 0 ]; then
              echo "::warning::Recovered $AGC_RECOVERED AGC output(s) via direct Azure CLI queries."
            fi
          fi

          # ── Summary ────────────────────────────────────────────────
          if [ "$RECOVERED" -gt 0 ]; then
            echo "::warning::Recovered $RECOVERED output(s) via direct Azure queries. ARM deployment state may be Failed — investigate RoleAssignment conflicts in shared-infrastructure."
          else
            echo "All provisioned outputs present — no recovery needed."
          fi

      - name: Export provisioned outputs
        id: outputs
        run: |
          eval "$(azd env get-values -e '${{ inputs.environment }}' | sed 's/^/export /')"
          # Fallback: read individual keys directly from azd env when eval drops them
          for key in BLOB_ACCOUNT_URL BLOB_CONTAINER COSMOS_CONTAINER PROJECT_ENDPOINT PROJECT_NAME APPLICATIONINSIGHTS_CONNECTION_STRING AGC_SUPPORT_ENABLED AGC_GATEWAY_CLASS AGC_SUBNET_ID AGC_FRONTEND_HOSTNAME AGC_FRONTEND_REFERENCE AGC_CONTROLLER_DEPLOYMENT_MODE AGC_CONTROLLER_IDENTITY_NAME AGC_CONTROLLER_IDENTITY_CLIENT_ID; do
            eval "cur=\${$key:-}"
            if [ -z "$cur" ]; then
              if val=$(azd env get-value "$key" -e '${{ inputs.environment }}' 2>/dev/null); then
                [ -n "$val" ] && export "$key=$val"
              fi
            fi
          done
          APIM_GATEWAY_URL_VALUE="${APIM_GATEWAY_URL:-${apimGatewayUrl:-}}"
          if [ "$APIM_GATEWAY_URL_VALUE" = "null" ]; then
            APIM_GATEWAY_URL_VALUE=""
          fi
          {
            echo "AI_SERVICES_NAME=${AI_SERVICES_NAME}"
            echo "AI_SEARCH_NAME=${AI_SEARCH_NAME}"
            echo "AI_SEARCH_ENDPOINT=${AI_SEARCH_ENDPOINT}"
            echo "AI_SEARCH_INDEX=${AI_SEARCH_INDEX}"
            echo "AI_SEARCH_VECTOR_INDEX=${AI_SEARCH_VECTOR_INDEX}"
            echo "AI_SEARCH_INDEXER_NAME=${AI_SEARCH_INDEXER_NAME}"
            echo "EMBEDDING_DEPLOYMENT_NAME=${EMBEDDING_DEPLOYMENT_NAME}"
            echo "AI_SEARCH_AUTH_MODE=${AI_SEARCH_AUTH_MODE}"
            echo "PROJECT_ENDPOINT=${PROJECT_ENDPOINT}"
            echo "PROJECT_NAME=${PROJECT_NAME}"
            echo "APIM_GATEWAY_URL=${APIM_GATEWAY_URL_VALUE}"
            echo "COSMOS_ACCOUNT_URI=${COSMOS_ACCOUNT_URI}"
            echo "COSMOS_DATABASE=${COSMOS_DATABASE}"
            echo "KEY_VAULT_URI=${KEY_VAULT_URI}"
            echo "REDIS_HOST=${REDIS_HOST}"
            echo "EVENT_HUB_NAMESPACE=${EVENT_HUB_NAMESPACE}"
            echo "APPLICATIONINSIGHTS_CONNECTION_STRING=${APPLICATIONINSIGHTS_CONNECTION_STRING}"
            echo "POSTGRES_HOST=${POSTGRES_HOST}"
            echo "POSTGRES_USER=${POSTGRES_USER}"
            echo "POSTGRES_ADMIN_USER=${POSTGRES_ADMIN_USER}"
            echo "POSTGRES_AUTH_MODE=${POSTGRES_AUTH_MODE}"
            echo "POSTGRES_DATABASE=${POSTGRES_DATABASE}"
            echo "AGC_SUPPORT_ENABLED=${AGC_SUPPORT_ENABLED}"
            echo "AGC_GATEWAY_CLASS=${AGC_GATEWAY_CLASS}"
            echo "AGC_SUBNET_ID=${AGC_SUBNET_ID}"
            echo "AGC_FRONTEND_HOSTNAME=${AGC_FRONTEND_HOSTNAME}"
            echo "AGC_FRONTEND_REFERENCE=${AGC_FRONTEND_REFERENCE}"
            echo "AGC_CONTROLLER_DEPLOYMENT_MODE=${AGC_CONTROLLER_DEPLOYMENT_MODE}"
            echo "AGC_CONTROLLER_IDENTITY_NAME=${AGC_CONTROLLER_IDENTITY_NAME}"
            echo "AGC_CONTROLLER_IDENTITY_CLIENT_ID=${AGC_CONTROLLER_IDENTITY_CLIENT_ID}"
          } >> "$GITHUB_OUTPUT"

  prepare-acr-build-access:
    runs-on: ubuntu-latest
    if: ${{ !inputs.uiOnly && needs.detect-changes.outputs.changed_aks_services_csv != '' }}
    needs:
      - provision
      - detect-changes
    environment: ${{ inputs.githubEnvironment }}
    outputs:
      acr_name: ${{ steps.registry.outputs.acr_name }}
      rg_name: ${{ steps.registry.outputs.rg_name }}
      login_server: ${{ steps.registry.outputs.login_server }}
      public_network_access_before: ${{ steps.access.outputs.public_network_access_before }}
      default_action_before: ${{ steps.access.outputs.default_action_before }}
      export_policy_before: ${{ steps.access.outputs.export_policy_before }}
      public_access_temporarily_enabled: ${{ steps.access.outputs.public_access_temporarily_enabled }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Resolve ACR registry
        id: registry
        shell: bash
        run: |
          set -euo pipefail

          RG_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          ACR_NAME=$(echo "${{ inputs.projectName }}${{ inputs.environment }}acr" | tr -cd '[:alnum:]' | tr '[:upper:]' '[:lower:]')

          if ! az acr show --resource-group "$RG_NAME" --name "$ACR_NAME" >/dev/null 2>&1; then
            echo "ACR registry '$ACR_NAME' was not found in resource group '$RG_NAME'." >&2
            exit 1
          fi

          ACR_LOGIN_SERVER=$(az acr show \
            --resource-group "$RG_NAME" \
            --name "$ACR_NAME" \
            --query "loginServer" -o tsv)

          echo "acr_name=${ACR_NAME}" >> "$GITHUB_OUTPUT"
          echo "rg_name=${RG_NAME}" >> "$GITHUB_OUTPUT"
          echo "login_server=${ACR_LOGIN_SERVER}" >> "$GITHUB_OUTPUT"

      - name: Prepare temporary ACR public access for tested-image builds
        id: access
        shell: bash
        run: |
          set -euo pipefail

          ACR_NAME="${{ steps.registry.outputs.acr_name }}"
          RG_NAME="${{ steps.registry.outputs.rg_name }}"
          RESTORE_ON_ERROR=false

          cleanup() {
            if [ "$RESTORE_ON_ERROR" != "true" ]; then
              return
            fi

            if [ -n "${DEFAULT_ACTION_BEFORE:-}" ]; then
              az acr update \
                --name "$ACR_NAME" \
                --resource-group "$RG_NAME" \
                --default-action "$DEFAULT_ACTION_BEFORE" >/dev/null || true
            fi

            if [ "${PUBLIC_NETWORK_ACCESS_BEFORE:-}" = "Disabled" ]; then
              az acr update \
                --name "$ACR_NAME" \
                --resource-group "$RG_NAME" \
                --public-network-enabled false >/dev/null || true
            fi

            if [ "${EXPORT_POLICY_BEFORE:-}" = "disabled" ]; then
              az resource update \
                --resource-group "$RG_NAME" \
                --name "$ACR_NAME" \
                --resource-type "Microsoft.ContainerRegistry/registries" \
                --set "properties.policies.exportPolicy.status=disabled" >/dev/null || true
            fi
          }
          trap cleanup EXIT

          PUBLIC_NETWORK_ACCESS_BEFORE=$(az acr show \
            --resource-group "$RG_NAME" \
            --name "$ACR_NAME" \
            --query "publicNetworkAccess" -o tsv)
          DEFAULT_ACTION_BEFORE=$(az acr show \
            --resource-group "$RG_NAME" \
            --name "$ACR_NAME" \
            --query "networkRuleSet.defaultAction" -o tsv)
          EXPORT_POLICY_BEFORE=$(az acr show \
            --resource-group "$RG_NAME" \
            --name "$ACR_NAME" \
            --query "policies.exportPolicy.status" -o tsv)

          if [ -z "$DEFAULT_ACTION_BEFORE" ] || [ "$DEFAULT_ACTION_BEFORE" = "null" ]; then
            DEFAULT_ACTION_BEFORE="Allow"
          fi
          if [ -z "$EXPORT_POLICY_BEFORE" ] || [ "$EXPORT_POLICY_BEFORE" = "null" ]; then
            EXPORT_POLICY_BEFORE="enabled"
          fi

          echo "public_network_access_before=${PUBLIC_NETWORK_ACCESS_BEFORE}" >> "$GITHUB_OUTPUT"
          echo "default_action_before=${DEFAULT_ACTION_BEFORE}" >> "$GITHUB_OUTPUT"
          echo "export_policy_before=${EXPORT_POLICY_BEFORE}" >> "$GITHUB_OUTPUT"
          echo "public_access_temporarily_enabled=false" >> "$GITHUB_OUTPUT"

          if [ "${{ inputs.autoAllowAcrRunnerIp }}" != "true" ] || [ "$PUBLIC_NETWORK_ACCESS_BEFORE" != "Disabled" ]; then
            exit 0
          fi

          RESTORE_ON_ERROR=true

          # Export policy must be enabled before publicNetworkAccess can be turned on
          if [ "$EXPORT_POLICY_BEFORE" = "disabled" ]; then
            az resource update \
              --resource-group "$RG_NAME" \
              --name "$ACR_NAME" \
              --resource-type "Microsoft.ContainerRegistry/registries" \
              --set "properties.policies.exportPolicy.status=enabled" >/dev/null
          fi

          az acr update \
            --name "$ACR_NAME" \
            --resource-group "$RG_NAME" \
            --public-network-enabled true \
            --default-action Allow >/dev/null

          CURRENT_PUBLIC_NETWORK_ACCESS=""
          CURRENT_DEFAULT_ACTION=""
          for attempt in 1 2 3 4 5; do
            CURRENT_PUBLIC_NETWORK_ACCESS=$(az acr show \
              --resource-group "$RG_NAME" \
              --name "$ACR_NAME" \
              --query "publicNetworkAccess" -o tsv)
            CURRENT_DEFAULT_ACTION=$(az acr show \
              --resource-group "$RG_NAME" \
              --name "$ACR_NAME" \
              --query "networkRuleSet.defaultAction" -o tsv)

            if [ "$CURRENT_PUBLIC_NETWORK_ACCESS" = "Enabled" ] && [ "$CURRENT_DEFAULT_ACTION" = "Allow" ]; then
              break
            fi

            sleep 10
          done

          if [ "$CURRENT_PUBLIC_NETWORK_ACCESS" != "Enabled" ] || [ "$CURRENT_DEFAULT_ACTION" != "Allow" ]; then
            echo "Failed to temporarily enable least-open ACR public access on ${ACR_NAME} for GitHub-hosted tested-image builds." >&2
            exit 1
          fi

          # Wait for ACR data-plane to serve OAuth token exchange after state change
          DATA_PLANE_READY=false
          for dp_attempt in 1 2 3 4 5 6; do
            if az acr login --name "$ACR_NAME" 2>/dev/null; then
              DATA_PLANE_READY=true
              break
            fi
            sleep 15
          done

          if [ "$DATA_PLANE_READY" != "true" ]; then
            echo "ACR data-plane not ready after enabling public access on ${ACR_NAME}. Token exchange unavailable." >&2
            exit 1
          fi

          echo "public_access_temporarily_enabled=true" >> "$GITHUB_OUTPUT"
          RESTORE_ON_ERROR=false

  build-aks-images:
    runs-on: ubuntu-latest
    if: ${{ !inputs.uiOnly && needs.detect-changes.outputs.changed_aks_services_csv != '' }}
    needs:
      - provision
      - detect-changes
      - prepare-acr-build-access
    environment: ${{ inputs.githubEnvironment }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.detect-changes.outputs.changed_aks_matrix) }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      CHANGED_AGENT_SERVICES: ${{ needs.detect-changes.outputs.changed_agent_services_csv }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ env.DEPLOY_SOURCE_CHECKOUT_REF }}

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Preflight ACR data-plane accessibility
        id: acr_preflight_build
        shell: bash
        run: |
          set -euo pipefail

          ACR_NAME="${{ needs.prepare-acr-build-access.outputs.acr_name }}"
          RG_NAME="${{ needs.prepare-acr-build-access.outputs.rg_name }}"
          ACR_LOGIN_SERVER="${{ needs.prepare-acr-build-access.outputs.login_server }}"

          echo "runner_ip=" >> "$GITHUB_OUTPUT"
          echo "runner_ip_allowlist_added=false" >> "$GITHUB_OUTPUT"

          if ! STATUS_CODE=$(curl -sS -o /tmp/acr-v2-probe-build.txt -w "%{http_code}" --max-time 25 "https://${ACR_LOGIN_SERVER}/v2/"); then
            STATUS_CODE="000"
          fi

          if [ "$STATUS_CODE" = "200" ] || [ "$STATUS_CODE" = "401" ]; then
            echo "runner_ip=" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          RUNNER_IP=""
          if ! RUNNER_IP=$(curl -sS --max-time 10 https://api.ipify.org); then
            RUNNER_IP=""
          fi

          if [ -n "$RUNNER_IP" ] && [ "${{ inputs.autoAllowAcrRunnerIp }}" = "true" ] && [ "$STATUS_CODE" = "403" ]; then
            EXISTING_RULE_COUNT=$(az acr network-rule list \
              --name "$ACR_NAME" \
              --resource-group "$RG_NAME" \
              --query "ipRules[].value" -o tsv 2>/dev/null | grep -Ec "^${RUNNER_IP}(/32)?$" || true)
            EXISTING_RULE_COUNT="${EXISTING_RULE_COUNT:-0}"

            if [ "$EXISTING_RULE_COUNT" = "0" ]; then
              az acr network-rule add \
                --name "$ACR_NAME" \
                --resource-group "$RG_NAME" \
                --ip-address "$RUNNER_IP" >/dev/null
              echo "runner_ip_allowlist_added=true" >> "$GITHUB_OUTPUT"
            fi

            echo "runner_ip=${RUNNER_IP}" >> "$GITHUB_OUTPUT"
            sleep 15
            if ! RETRY_CODE=$(curl -sS -o /tmp/acr-v2-probe-build-retry.txt -w "%{http_code}" --max-time 25 "https://${ACR_LOGIN_SERVER}/v2/"); then
              RETRY_CODE="000"
            fi
            if [ "$RETRY_CODE" = "200" ] || [ "$RETRY_CODE" = "401" ]; then
              exit 0
            fi
          fi

          echo "ACR preflight failed for tested-image build path (HTTP $STATUS_CODE)." >&2
          exit 1

      - name: Login to ACR
        shell: bash
        run: |
          set -euo pipefail

          ACR_NAME="${{ needs.prepare-acr-build-access.outputs.acr_name }}"
          ACR_SCOPE=$(az acr show \
            --resource-group "${{ needs.prepare-acr-build-access.outputs.rg_name }}" \
            --name "$ACR_NAME" \
            --query id -o tsv)
          DEPLOY_SP_OBJECT_ID=$(az ad sp show --id "${AZURE_CLIENT_ID}" --query id -o tsv)
          RETRY_DELAYS=(15 30 60 90)
          LOGIN_SUCCESS=false

          for attempt in 1 2 3 4 5; do
            if az acr login --name "$ACR_NAME"; then
              LOGIN_SUCCESS=true
              break
            fi

            ACR_PUSH_ASSIGNMENTS=$(az role assignment list \
              --scope "$ACR_SCOPE" \
              --assignee-object-id "$DEPLOY_SP_OBJECT_ID" \
              --query "[?roleDefinitionName=='AcrPush'] | length(@)" -o tsv 2>/dev/null || true)
            ACR_PUSH_ASSIGNMENTS="${ACR_PUSH_ASSIGNMENTS:-0}"

            if [ "$attempt" -lt 5 ]; then
              delay=${RETRY_DELAYS[$((attempt-1))]}
              echo "::warning::az acr login failed on attempt ${attempt}/5 for ${ACR_NAME}. Visible AcrPush assignments for deploy principal ${DEPLOY_SP_OBJECT_ID} at ${ACR_SCOPE}: ${ACR_PUSH_ASSIGNMENTS}. This usually indicates RBAC propagation delay between ARM and the ACR data plane. Retrying in ${delay}s."
              sleep "$delay"
            fi
          done

          if [ "$LOGIN_SUCCESS" != "true" ]; then
            echo "ACR login failed after 5 attempts for ${ACR_NAME}. Verify deploy principal object id ${DEPLOY_SP_OBJECT_ID} has AcrPush on ${ACR_SCOPE}, then rerun after RBAC propagation completes. OIDC-only auth is required; do not enable admin-user or add static registry credentials." >&2
            exit 1
          fi

      - name: Build lib wheel
        shell: bash
        run: |
          set -euo pipefail
          pip install --quiet uv
          cd lib/src
          uv build --wheel --out-dir "$GITHUB_WORKSPACE/.tmp/lib-dist"
          echo "Built lib wheel:"
          ls -la "$GITHUB_WORKSPACE/.tmp/lib-dist/"

      - name: Resolve Docker build inputs
        id: docker_inputs
        shell: bash
        run: |
          set -euo pipefail

          python3 - <<'PY'
          import os
          import re
          import sys
          from pathlib import Path

          service_name = os.environ["SERVICE_NAME"]
          github_output = Path(os.environ["GITHUB_OUTPUT"])
          repo_root = Path.cwd()
          azure_yaml = repo_root / "azure.yaml"

          with azure_yaml.open(encoding="utf-8") as handle:
              lines = handle.readlines()

          in_services = False
          current_service = None
          project_path = None
          docker_path = None
          docker_context = None
          in_docker_block = False

          for raw in lines:
              line = raw.rstrip("\n")
              if not in_services:
                  if re.match(r"^services:\s*$", line):
                      in_services = True
                  continue

              if re.match(r"^[^\s]", line):
                  break

              service_match = re.match(r"^  ([a-z0-9\-]+):\s*$", line)
              if service_match:
                  if current_service == service_name:
                      break
                  current_service = service_match.group(1)
                  project_path = None
                  docker_path = None
                  docker_context = None
                  in_docker_block = False
                  continue

              if current_service != service_name:
                  continue

              if re.match(r"^    docker:\s*$", line):
                  in_docker_block = True
                  continue

              if re.match(r"^    [a-zA-Z]", line):
                  in_docker_block = False

              project_match = re.match(r"^    project:\s*(\S+)\s*$", line)
              if project_match:
                  project_path = project_match.group(1)
                  continue

              if in_docker_block:
                  docker_path_match = re.match(r"^      path:\s*(\S+)\s*$", line)
                  if docker_path_match:
                      docker_path = docker_path_match.group(1)
                      continue

                  docker_context_match = re.match(r"^      context:\s*(\S+)\s*$", line)
                  if docker_context_match:
                      docker_context = docker_context_match.group(1)
                      continue

          if project_path is None:
              raise SystemExit(f"Could not resolve azure.yaml project metadata for service '{service_name}'.")

          project_dir = repo_root / project_path
          dockerfile_path = (project_dir / (docker_path or "Dockerfile")).resolve()
          build_context = (project_dir / (docker_context or ".")).resolve()

          try:
              dockerfile_rel = dockerfile_path.relative_to(repo_root).as_posix()
          except ValueError as exc:
              raise SystemExit(f"Resolved dockerfile path escapes repository root: {dockerfile_path}") from exc

          try:
              build_context_rel = build_context.relative_to(repo_root).as_posix()
          except ValueError as exc:
              raise SystemExit(f"Resolved build context path escapes repository root: {build_context}") from exc

          if not dockerfile_path.is_file():
              raise SystemExit(f"Dockerfile not found for service '{service_name}': {dockerfile_rel}")

          with github_output.open("a", encoding="utf-8") as output:
              output.write(f"dockerfile={dockerfile_rel}\n")
              output.write(f"context={build_context_rel}\n")
          PY
        env:
          SERVICE_NAME: ${{ matrix.service }}

      - name: Resolve or build immutable image
        id: image
        shell: bash
        run: |
          set -euo pipefail

          SERVICE_NAME="${{ matrix.service }}"
          SOURCE_SHA="${DEPLOY_SOURCE_SHA}"
          IMAGE_REPOSITORY="${{ needs.prepare-acr-build-access.outputs.login_server }}/${SERVICE_NAME}"
          IMAGE_TAGGED_REF="${IMAGE_REPOSITORY}:${SOURCE_SHA}"
          EXISTING_DIGEST=""

          if INSPECT_OUTPUT=$(docker buildx imagetools inspect "$IMAGE_TAGGED_REF" 2>/dev/null); then
            EXISTING_DIGEST=$(printf '%s\n' "$INSPECT_OUTPUT" | awk '/^Digest:/ {print $2; exit}')
          fi

          if [ -z "$EXISTING_DIGEST" ]; then
            # Inject freshly-built lib wheel into Docker build context
            CONTEXT_DIR="${{ steps.docker_inputs.outputs.context }}"
            cp "$GITHUB_WORKSPACE/.tmp/lib-dist"/holiday_peak_lib-*.whl "$CONTEXT_DIR/" 2>/dev/null || true

            EXTRA_TAG_ARGS=()
            if [ -n "${{ inputs.imageTag }}" ] && [ "${{ inputs.imageTag }}" != "$SOURCE_SHA" ] && [ "${{ inputs.imageTag }}" != "latest" ]; then
              EXTRA_TAG_ARGS+=( -t "${IMAGE_REPOSITORY}:${{ inputs.imageTag }}" )
            fi

            docker buildx build \
              --push \
              --pull \
              --cache-from=type=gha,scope="${SERVICE_NAME}" \
              --cache-to=type=gha,mode=max,scope="${SERVICE_NAME}" \
              -t "$IMAGE_TAGGED_REF" \
              "${EXTRA_TAG_ARGS[@]}" \
              -f "${{ steps.docker_inputs.outputs.dockerfile }}" \
              "${{ steps.docker_inputs.outputs.context }}"

            INSPECT_OUTPUT=$(docker buildx imagetools inspect "$IMAGE_TAGGED_REF")
            EXISTING_DIGEST=$(printf '%s\n' "$INSPECT_OUTPUT" | awk '/^Digest:/ {print $2; exit}')
          fi

          if [ -z "$EXISTING_DIGEST" ]; then
            echo "Failed to resolve immutable digest for ${IMAGE_TAGGED_REF}." >&2
            exit 1
          fi

          IMAGE_REF="${IMAGE_REPOSITORY}@${EXISTING_DIGEST}"
          ARTIFACT_DIR="${RUNNER_TEMP}/tested-image/${SERVICE_NAME}"
          mkdir -p "$ARTIFACT_DIR"
          printf '%s\n' "$IMAGE_REF" > "$ARTIFACT_DIR/image-ref.txt"
          echo "image_ref=${IMAGE_REF}" >> "$GITHUB_OUTPUT"

      - name: Verify prompt in built image
        if: ${{ success() && !inputs.skipPromptGates }}
        shell: bash
        env:
          SERVICE_NAME: ${{ matrix.service }}
          IMAGE_REF: ${{ steps.image.outputs.image_ref }}
        run: |
          set -euo pipefail
          echo "Verifying prompt sha256 match between repo and image for ${SERVICE_NAME}..."
          docker pull "$IMAGE_REF" >/dev/null
          python3 scripts/ci/verify_image_prompt.py \
            --service "$SERVICE_NAME" \
            --image-ref "$IMAGE_REF"

      - name: Upload tested image reference
        uses: actions/upload-artifact@v4
        with:
          name: tested-image-${{ matrix.service }}
          path: ${{ runner.temp }}/tested-image/${{ matrix.service }}/image-ref.txt
          if-no-files-found: error
          retention-days: 7

      - name: Cleanup temporary ACR runner IP allowlist (build)
        if: ${{ always() && steps.acr_preflight_build.outputs.runner_ip_allowlist_added == 'true' }}
        shell: bash
        run: |
          set -euo pipefail
          az acr network-rule remove \
            --name "${{ needs.prepare-acr-build-access.outputs.acr_name }}" \
            --resource-group "${{ needs.prepare-acr-build-access.outputs.rg_name }}" \
            --ip-address "${{ steps.acr_preflight_build.outputs.runner_ip }}" >/dev/null || true

  deploy-foundry-surfaces:
    runs-on: ubuntu-latest
    if: ${{ !inputs.uiOnly && inputs.deployFoundrySurfaces && needs.detect-changes.outputs.changed_agent_services_csv != '' }}
    needs:
      - detect-changes
      - provision
      - prepare-acr-build-access
      - build-aks-images
    environment: ${{ inputs.githubEnvironment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      PROJECT_ENDPOINT: ${{ needs.provision.outputs.PROJECT_ENDPOINT }}
      PROJECT_NAME: ${{ needs.provision.outputs.PROJECT_NAME }}
      APIM_BASE_URL: ${{ needs.provision.outputs.APIM_GATEWAY_URL }}
      AZURE_CONTAINER_REGISTRY: ${{ needs.prepare-acr-build-access.outputs.login_server }}
      REDIS_HOST: ${{ needs.provision.outputs.REDIS_HOST }}
      REDIS_URL: ""
      COSMOS_ACCOUNT_URI: ${{ needs.provision.outputs.COSMOS_ACCOUNT_URI }}
      COSMOS_DATABASE: ${{ needs.provision.outputs.COSMOS_DATABASE }}
      COSMOS_CONTAINER: ${{ needs.provision.outputs.COSMOS_CONTAINER }}
      BLOB_ACCOUNT_URL: ${{ needs.provision.outputs.BLOB_ACCOUNT_URL }}
      BLOB_CONTAINER: ${{ needs.provision.outputs.BLOB_CONTAINER }}
      EVENT_HUB_NAMESPACE: ${{ needs.provision.outputs.EVENT_HUB_NAMESPACE }}
      KEY_VAULT_URI: ${{ needs.provision.outputs.KEY_VAULT_URI }}
      APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ needs.provision.outputs.APPLICATIONINSIGHTS_CONNECTION_STRING }}
      MODEL_DEPLOYMENT_NAME_FAST: gpt-5-nano
      MODEL_DEPLOYMENT_NAME_RICH: gpt-5
    steps:
      - name: Checkout tested source
        uses: actions/checkout@v4
        with:
          ref: ${{ env.DEPLOY_SOURCE_CHECKOUT_REF }}

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.13'

      - name: Install Foundry surface registration dependencies
        shell: bash
        run: |
          set -euo pipefail
          python3 -m pip install --quiet \
            pyyaml \
            "azure-ai-projects>=2.0.1" \
            "azure-identity>=1.17.0"

      - name: Download tested image references
        uses: actions/download-artifact@v4
        with:
          pattern: tested-image-*
          path: ${{ runner.temp }}/tested-image-artifacts
          merge-multiple: false

      - name: Build Foundry image map
        shell: bash
        run: |
          set -euo pipefail
          python3 - <<'PY'
          import json
          import os
          from pathlib import Path

          artifact_root = Path(os.environ["RUNNER_TEMP"]) / "tested-image-artifacts"
          image_map = {}
          for image_file in artifact_root.glob("tested-image-*/image-ref.txt"):
              service = image_file.parent.name.removeprefix("tested-image-")
              image_map[service] = image_file.read_text(encoding="utf-8").strip()

          output_path = Path(os.environ["RUNNER_TEMP"]) / "foundry-image-map.json"
          output_path.write_text(json.dumps(image_map, indent=2, sort_keys=True) + "\n", encoding="utf-8")
          print(f"Wrote {len(image_map)} tested image refs to {output_path}")
          PY

      - name: Validate Hosted Agent ACR reachability policy
        shell: bash
        run: |
          set -euo pipefail
          MODE="${{ inputs.foundrySurfaceMode }}"
          if [ "$MODE" != "apply" ]; then
            exit 0
          fi

          PUBLIC_BEFORE="${{ needs.prepare-acr-build-access.outputs.public_network_access_before }}"
          DEFAULT_BEFORE="${{ needs.prepare-acr-build-access.outputs.default_action_before }}"
          if [ "$PUBLIC_BEFORE" != "Enabled" ] || [ "$DEFAULT_BEFORE" != "Allow" ]; then
            echo "Foundry Hosted Agents require an ACR public endpoint that the service can pull from." >&2
            echo "Current baseline publicNetworkAccess=${PUBLIC_BEFORE}, defaultAction=${DEFAULT_BEFORE}." >&2
            echo "Run in plan mode, then update the ACR network policy intentionally before apply." >&2
            exit 1
          fi

      - name: Register Foundry surfaces
        shell: bash
        run: |
          set -euo pipefail
          MODE="${{ inputs.foundrySurfaceMode }}"
          if [ "$MODE" != "plan" ] && [ "$MODE" != "apply" ]; then
            echo "foundrySurfaceMode must be 'plan' or 'apply'." >&2
            exit 1
          fi

          python3 scripts/ops/register_foundry_surfaces.py \
            --mode "$MODE" \
            --environment "${{ inputs.environment }}" \
            --project-endpoint "$PROJECT_ENDPOINT" \
            --project-name "$PROJECT_NAME" \
            --image-map-file "${RUNNER_TEMP}/foundry-image-map.json" \
            --acr-login-server "${AZURE_CONTAINER_REGISTRY}" \
            --image-tag "${DEPLOY_SOURCE_SHA}" \
            --apim-base-url "$APIM_BASE_URL" \
            --services "${{ needs.detect-changes.outputs.changed_agent_services_csv }}" \
            --output "${RUNNER_TEMP}/foundry-surface-plan.json"

      - name: Upload Foundry surface plan
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: foundry-surface-plan-${{ inputs.environment }}
          path: ${{ runner.temp }}/foundry-surface-plan.json
          if-no-files-found: warn
          retention-days: 14

  restore-acr-build-access:
    runs-on: ubuntu-latest
    if: ${{ always() && !inputs.uiOnly && needs.detect-changes.outputs.changed_aks_services_csv != '' }}
    needs:
      - detect-changes
      - prepare-acr-build-access
      - build-aks-images
    environment: ${{ inputs.githubEnvironment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Restore ACR public access state
        shell: bash
        run: |
          set -euo pipefail

          if [ "${{ needs.prepare-acr-build-access.outputs.public_access_temporarily_enabled }}" != "true" ]; then
            echo "ACR public access state did not require temporary changes."
            exit 0
          fi

          ACR_NAME="${{ needs.prepare-acr-build-access.outputs.acr_name }}"
          RG_NAME="${{ needs.prepare-acr-build-access.outputs.rg_name }}"
          PUBLIC_NETWORK_ACCESS_BEFORE="${{ needs.prepare-acr-build-access.outputs.public_network_access_before }}"
          DEFAULT_ACTION_BEFORE="${{ needs.prepare-acr-build-access.outputs.default_action_before }}"
          EXPORT_POLICY_BEFORE="${{ needs.prepare-acr-build-access.outputs.export_policy_before }}"

          if [ "$PUBLIC_NETWORK_ACCESS_BEFORE" = "Disabled" ]; then
            # Disable public network access first; defaultAction is irrelevant
            # when public access is off (Azure may report any value).
            az acr update \
              --name "$ACR_NAME" \
              --resource-group "$RG_NAME" \
              --public-network-enabled false >/dev/null
          else
            # Public access stays enabled — restore the original defaultAction
            if [ -n "$DEFAULT_ACTION_BEFORE" ] && [ "$DEFAULT_ACTION_BEFORE" != "null" ]; then
              az acr update \
                --name "$ACR_NAME" \
                --resource-group "$RG_NAME" \
                --default-action "$DEFAULT_ACTION_BEFORE" >/dev/null
            fi
          fi

          # Restore export policy after public network access is disabled
          if [ "$EXPORT_POLICY_BEFORE" = "disabled" ]; then
            az resource update \
              --resource-group "$RG_NAME" \
              --name "$ACR_NAME" \
              --resource-type "Microsoft.ContainerRegistry/registries" \
              --set "properties.policies.exportPolicy.status=disabled" >/dev/null
          fi

          CURRENT_PUBLIC_NETWORK_ACCESS=$(az acr show \
            --resource-group "$RG_NAME" \
            --name "$ACR_NAME" \
            --query "publicNetworkAccess" -o tsv)

          if [ "$CURRENT_PUBLIC_NETWORK_ACCESS" != "$PUBLIC_NETWORK_ACCESS_BEFORE" ]; then
            echo "Failed to restore ACR public network access state for ${ACR_NAME}: expected ${PUBLIC_NETWORK_ACCESS_BEFORE}, got ${CURRENT_PUBLIC_NETWORK_ACCESS}." >&2
            exit 1
          fi

          # Only validate defaultAction when public access is enabled (it is
          # meaningless when public access is Disabled — Azure can report any value).
          if [ "$PUBLIC_NETWORK_ACCESS_BEFORE" != "Disabled" ]; then
            CURRENT_DEFAULT_ACTION=$(az acr show \
              --resource-group "$RG_NAME" \
              --name "$ACR_NAME" \
              --query "networkRuleSet.defaultAction" -o tsv)

            if [ -z "$CURRENT_DEFAULT_ACTION" ] || [ "$CURRENT_DEFAULT_ACTION" = "null" ]; then
              CURRENT_DEFAULT_ACTION="Allow"
            fi

            if [ "$CURRENT_DEFAULT_ACTION" != "$DEFAULT_ACTION_BEFORE" ]; then
              echo "Failed to restore ACR default network action for ${ACR_NAME}: expected ${DEFAULT_ACTION_BEFORE}, got ${CURRENT_DEFAULT_ACTION}." >&2
              exit 1
            fi
          fi

  deploy-foundry-models:
    runs-on: ubuntu-latest
    needs:
      - provision
      - detect-changes
    if: ${{ !inputs.skipProvision && needs.detect-changes.outputs.infra_changed == 'true' }}
    environment: ${{ inputs.githubEnvironment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ env.DEPLOY_SOURCE_CHECKOUT_REF }}

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Deploy AI model deployments
        run: |
          bash .infra/azd/hooks/deploy-foundry-models.sh \
            "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            "${{ needs.provision.outputs.AI_SERVICES_NAME }}"
        env:
          AZURE_RESOURCE_GROUP: ${{ inputs.projectName }}-${{ inputs.environment }}-rg
          AI_SERVICES_NAME: ${{ needs.provision.outputs.AI_SERVICES_NAME }}

  deploy-crud:
    runs-on: ubuntu-latest
    needs:
      - provision
      - detect-changes
      - build-aks-images
      - restore-acr-build-access
    if: ${{ !inputs.uiOnly && needs.detect-changes.outputs.crud_changed == 'true' }}
    environment: ${{ inputs.githubEnvironment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ env.DEPLOY_SOURCE_CHECKOUT_REF }}

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Install kubelogin
        run: |
          for attempt in 1 2 3 4 5; do
            if az aks install-cli --kubelogin-version latest; then
              exit 0
            fi
            echo "kubelogin install failed on attempt ${attempt}; retrying..."
            sleep 15
          done
          echo "kubelogin install failed after retries"
          exit 1

      - name: Ensure hook scripts executable
        run: chmod +x ./.infra/azd/hooks/*.sh

      - name: Get AKS credentials
        run: |
          az aks get-credentials \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --overwrite-existing
          kubelogin convert-kubeconfig -l azurecli

      - name: Verify AKS API connectivity
        shell: bash
        run: |
          set -euo pipefail
          AKS_FQDN=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed 's|https://||;s|:.*||')
          echo "Probing AKS API server DNS: $AKS_FQDN"
          for attempt in $(seq 1 10); do
            if kubectl cluster-info >/dev/null 2>&1; then
              echo "AKS API reachable (attempt $attempt)."
              exit 0
            fi
            BACKOFF=$((5 * attempt))
            echo "Attempt $attempt/10 failed (DNS or TCP). Retrying in ${BACKOFF}s..."
            sleep "$BACKOFF"
          done
          echo "AKS API DNS resolution failed after 10 attempts for $AKS_FQDN." >&2
          nslookup "$AKS_FQDN" || true
          exit 1

      - name: Ensure target namespaces exist
        shell: bash
        run: |
          set -euo pipefail
          for ns in holiday-peak-crud holiday-peak-agents; do
            if ! kubectl get namespace "$ns" >/dev/null 2>&1; then
              kubectl create namespace "$ns"
              echo "Created namespace $ns"
            fi
            kubectl label namespace "$ns" holiday-peak/ingress-allowed=true --overwrite
          done

      - name: Resolve AKS managed identity client ID
        run: |
          AKS_MI_CLIENT_ID=$(az aks show \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --query "identityProfile.kubeletidentity.clientId" -o tsv)
          echo "WORKLOAD_AZURE_CLIENT_ID=${AKS_MI_CLIENT_ID}" >> "$GITHUB_ENV"

      - name: Resolve publication context
        shell: bash
        run: |
          set -euo pipefail

          AGC_GATEWAY_CLASS_VALUE="${{ needs.provision.outputs.AGC_GATEWAY_CLASS || 'azure-alb-external' }}"
          AGC_SUBNET_ID_VALUE="${{ needs.provision.outputs.AGC_SUBNET_ID }}"
          SHARED_GATEWAY_JSON="$(kubectl get gateway holiday-peak-agc -n holiday-peak-crud -o json 2>/dev/null || true)"

          get_first_alb_association() {
            local alb_name="$1"
            local alb_namespace="$2"

            if [ -z "$alb_name" ] || [ -z "$alb_namespace" ]; then
              return 0
            fi

            kubectl get applicationloadbalancer "$alb_name" -n "$alb_namespace" -o json 2>/dev/null \
              | jq -r '.spec.associations[]? | select(type == "string" and . != "")' \
              | head -n 1 || true
          }

          if [ -z "$AGC_SUBNET_ID_VALUE" ]; then
            AGC_SUBNET_ID_VALUE="$(get_first_alb_association "holiday-peak-agc" "holiday-peak-crud")"
            if [ -n "$AGC_SUBNET_ID_VALUE" ]; then
              echo "::notice::Recovered AGC render subnet from the live shared ApplicationLoadBalancer association."
            fi
          fi

          if [ -z "$AGC_SUBNET_ID_VALUE" ] && [ -n "$SHARED_GATEWAY_JSON" ]; then
            AGC_SHARED_ALB_NAME="$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '.metadata.annotations["alb.networking.azure.io/alb-name"] // empty' || true)"
            AGC_SHARED_ALB_NAMESPACE="$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '.metadata.annotations["alb.networking.azure.io/alb-namespace"] // empty' || true)"
            if [ -z "$AGC_SHARED_ALB_NAMESPACE" ]; then
              AGC_SHARED_ALB_NAMESPACE="holiday-peak-crud"
            fi
            AGC_SUBNET_ID_VALUE="$(get_first_alb_association "$AGC_SHARED_ALB_NAME" "$AGC_SHARED_ALB_NAMESPACE")"
            if [ -n "$AGC_SUBNET_ID_VALUE" ]; then
              echo "::notice::Recovered AGC render subnet from the ApplicationLoadBalancer referenced by the live shared Gateway."
            fi
          fi

          echo "PUBLICATION_MODE=agc" >> "$GITHUB_ENV"
          echo "AGC_SUPPORT_ENABLED=${{ needs.provision.outputs.AGC_SUPPORT_ENABLED || 'false' }}" >> "$GITHUB_ENV"
          echo "AGC_GATEWAY_CLASS=$AGC_GATEWAY_CLASS_VALUE" >> "$GITHUB_ENV"
          echo "AGC_SUBNET_ID=$AGC_SUBNET_ID_VALUE" >> "$GITHUB_ENV"

          if [ -z "$AGC_SUBNET_ID_VALUE" ]; then
            echo "::notice::No live AGC subnet association was available during render publication-context resolution. Later AGC readiness validation remains authoritative."
          fi

          kubectl get gatewayclass -o wide || true
          kubectl get ingressclass -o wide || true

          if [ "${{ needs.provision.outputs.AGC_SUPPORT_ENABLED || 'false' }}" = "true" ] && kubectl get gatewayclass "$AGC_GATEWAY_CLASS_VALUE" >/dev/null 2>&1; then
            echo "Using AGC gateway class: $AGC_GATEWAY_CLASS_VALUE"
          else
            echo "AGC gateway class '$AGC_GATEWAY_CLASS_VALUE' is not ready yet. APIM sync will be blocked later by the AGC readiness gate if this remains unresolved." >&2
          fi
          echo "Legacy ingress classes are ignored by default. Use PUBLICATION_MODE=legacy or dual with an explicit LEGACY_INGRESS_CLASS_NAME only for rollback."

      - name: Download tested image reference
        uses: actions/download-artifact@v4
        with:
          name: tested-image-crud-service
          path: ${{ runner.temp }}/tested-image/crud-service

      - name: Validate PostgreSQL auth contract
        shell: bash
        run: |
          set -euo pipefail

          if [ -z "${POSTGRES_HOST:-}" ] || [ -z "${POSTGRES_AUTH_MODE:-}" ]; then
            echo "Missing PostgreSQL deployment outputs required for auth validation." >&2
            exit 1
          fi

          POSTGRES_SERVER="${POSTGRES_HOST%%.*}"
          AUTH_JSON=$(az postgres flexible-server show \
            --resource-group "${RESOURCE_GROUP_NAME}" \
            --name "${POSTGRES_SERVER}" \
            --query '{passwordAuth:authConfig.passwordAuth,activeDirectoryAuth:authConfig.activeDirectoryAuth}' \
            -o json)

          PASSWORD_AUTH=$(python3 -c "import json,sys; print(json.load(sys.stdin).get('passwordAuth', ''))" <<<"${AUTH_JSON}")
          ACTIVE_DIRECTORY_AUTH=$(python3 -c "import json,sys; print(json.load(sys.stdin).get('activeDirectoryAuth', ''))" <<<"${AUTH_JSON}")

          echo "Configured POSTGRES_AUTH_MODE=${POSTGRES_AUTH_MODE}"
          echo "Live PostgreSQL auth config: passwordAuth=${PASSWORD_AUTH}, activeDirectoryAuth=${ACTIVE_DIRECTORY_AUTH}"

          case "${POSTGRES_AUTH_MODE}" in
            password)
              if [ "${PASSWORD_AUTH}" != "Enabled" ]; then
                echo "Configured password auth, but live PostgreSQL server '${POSTGRES_SERVER}' does not have passwordAuth enabled." >&2
                exit 1
              fi
              ;;
            entra)
              if [ "${ACTIVE_DIRECTORY_AUTH}" != "Enabled" ]; then
                echo "Configured Entra auth, but live PostgreSQL server '${POSTGRES_SERVER}' does not have activeDirectoryAuth enabled." >&2
                exit 1
              fi
              ;;
            *)
              echo "Unsupported POSTGRES_AUTH_MODE '${POSTGRES_AUTH_MODE}'." >&2
              exit 1
              ;;
          esac
        env:
          POSTGRES_AUTH_MODE: ${{ needs.provision.outputs.POSTGRES_AUTH_MODE }}
          POSTGRES_HOST: ${{ needs.provision.outputs.POSTGRES_HOST }}
          RESOURCE_GROUP_NAME: ${{ inputs.projectName }}-${{ inputs.environment }}-rg

      - name: Preflight — PostgreSQL admin password probe
        if: ${{ !inputs.skipPostgresPreflight }}
        shell: bash
        env:
          POSTGRES_AUTH_MODE: ${{ needs.provision.outputs.POSTGRES_AUTH_MODE }}
          POSTGRES_HOST: ${{ needs.provision.outputs.POSTGRES_HOST }}
          POSTGRES_ADMIN_USER: ${{ needs.provision.outputs.POSTGRES_ADMIN_USER }}
          KEY_VAULT_URI: ${{ needs.provision.outputs.KEY_VAULT_URI }}
          RESOURCE_GROUP_NAME: ${{ inputs.projectName }}-${{ inputs.environment }}-rg
          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
        run: |
          set -euo pipefail

          # Skip gracefully when this is a greenfield deploy or an Entra-only
          # server — the probe is only meaningful when a password-mode server
          # and Key Vault secret exist.
          if [ "${POSTGRES_AUTH_MODE:-}" != "password" ]; then
            echo "::notice::POSTGRES_AUTH_MODE is '${POSTGRES_AUTH_MODE:-unset}'; skipping password preflight probe."
            exit 0
          fi
          if [ -z "${POSTGRES_HOST:-}" ] || [ -z "${KEY_VAULT_URI:-}" ]; then
            echo "::warning::POSTGRES_HOST or KEY_VAULT_URI not yet resolved in the azd environment — skipping preflight (greenfield deploy)."
            exit 0
          fi

          SERVER_NAME="${POSTGRES_HOST%%.*}"
          KV_NAME="$(printf '%s' "${KEY_VAULT_URI}" | sed -E 's#https?://([^.]+).*#\1#')"
          ADMIN_USER="${POSTGRES_ADMIN_USER:-crud_admin}"

          if ! command -v psql >/dev/null 2>&1; then
            sudo apt-get update -qq >/dev/null
            sudo apt-get install -y -qq postgresql-client >/dev/null
          fi

          chmod +x ./scripts/ops/reconcile-postgres-password.sh
          set +e
          ./scripts/ops/reconcile-postgres-password.sh \
            --subscription-id "${AZURE_SUBSCRIPTION_ID}" \
            --resource-group "${RESOURCE_GROUP_NAME}" \
            --server-name "${SERVER_NAME}" \
            --admin-user "${ADMIN_USER}" \
            --key-vault-name "${KV_NAME}" \
            --secret-name postgres-admin-password \
            --mode probe
          probe_rc=$?
          set -e

          case "$probe_rc" in
            0)
              echo "::notice::Postgres admin password probe succeeded."
              ;;
            2)
              echo "::error::Postgres admin password probe FAILED with invalid_password." >&2
              echo "::error::Key Vault secret 'postgres-admin-password' is out of sync with the Flexible Server admin password." >&2
              echo "::error::Run: scripts/ops/reconcile-postgres-password.sh --mode rotate-from-keyvault --resource-group ${RESOURCE_GROUP_NAME} --server-name ${SERVER_NAME} --key-vault-name ${KV_NAME}" >&2
              exit 1
              ;;
            3)
              echo "::warning::Postgres admin password probe could not reach the server (network / VNet isolation). Skipping gate."
              ;;
            *)
              echo "::warning::Postgres admin password probe returned rc=${probe_rc}; treating as non-blocking."
              ;;
          esac

      - name: Deploy CRUD service
        timeout-minutes: 25
        shell: bash
        run: |
          set -euo pipefail
          export AZURE_CLIENT_ID="${WORKLOAD_AZURE_CLIENT_ID}"
          CRUD_IMAGE_REF=$(tr -d '\r\n' < "${RUNNER_TEMP}/tested-image/crud-service/image-ref.txt")

          # Phase B (ADR-017): Render only — Flux reconciles from Git
          export SERVICE_CRUD_SERVICE_IMAGE_NAME="$CRUD_IMAGE_REF"
          bash .infra/azd/hooks/render-helm.sh crud-service
          echo "Rendered CRUD manifest for Flux reconciliation (image: ${CRUD_IMAGE_REF})."
        env:
          DEPLOY_ENV: ${{ inputs.environment }}
          AZURE_TENANT_ID: ${{ env.AZURE_TENANT_ID }}
          AI_SEARCH_ENDPOINT: ${{ needs.provision.outputs.AI_SEARCH_ENDPOINT }}
          AI_SEARCH_INDEX: ${{ needs.provision.outputs.AI_SEARCH_INDEX }}
          AI_SEARCH_VECTOR_INDEX: ${{ needs.provision.outputs.AI_SEARCH_VECTOR_INDEX }}
          AI_SEARCH_INDEXER_NAME: ${{ needs.provision.outputs.AI_SEARCH_INDEXER_NAME }}
          EMBEDDING_DEPLOYMENT_NAME: ${{ needs.provision.outputs.EMBEDDING_DEPLOYMENT_NAME }}
          AI_SEARCH_AUTH_MODE: ${{ needs.provision.outputs.AI_SEARCH_AUTH_MODE }}
          CATALOG_SEARCH_REQUIRE_AI_SEARCH: "false"
          PROJECT_ENDPOINT: ${{ needs.provision.outputs.PROJECT_ENDPOINT }}
          PROJECT_NAME: ${{ needs.provision.outputs.PROJECT_NAME }}
          MODEL_DEPLOYMENT_NAME_FAST: gpt-5-nano
          MODEL_DEPLOYMENT_NAME_RICH: gpt-5
          POSTGRES_AUTH_MODE: ${{ needs.provision.outputs.POSTGRES_AUTH_MODE }}
          COSMOS_ACCOUNT_URI: ${{ needs.provision.outputs.COSMOS_ACCOUNT_URI }}
          COSMOS_DATABASE: ${{ needs.provision.outputs.COSMOS_DATABASE }}
          COSMOS_CONTAINER: ${{ needs.provision.outputs.COSMOS_CONTAINER }}
          BLOB_ACCOUNT_URL: ${{ needs.provision.outputs.BLOB_ACCOUNT_URL }}
          BLOB_CONTAINER: ${{ needs.provision.outputs.BLOB_CONTAINER }}
          REDIS_HOST: ${{ needs.provision.outputs.REDIS_HOST }}
          EVENT_HUB_NAMESPACE: ${{ needs.provision.outputs.EVENT_HUB_NAMESPACE }}
          KEY_VAULT_URI: ${{ needs.provision.outputs.KEY_VAULT_URI }}
          APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ needs.provision.outputs.APPLICATIONINSIGHTS_CONNECTION_STRING }}
          POSTGRES_HOST: ${{ needs.provision.outputs.POSTGRES_HOST }}
          POSTGRES_USER: ${{ needs.provision.outputs.POSTGRES_AUTH_MODE == 'password' && needs.provision.outputs.POSTGRES_ADMIN_USER || needs.provision.outputs.POSTGRES_USER }}
          POSTGRES_DATABASE: ${{ needs.provision.outputs.POSTGRES_DATABASE }}

      - name: Upload rendered CRUD manifest for verification
        uses: actions/upload-artifact@v4
        with:
          name: rendered-manifest-crud-service
          path: .kubernetes/rendered/crud-service/all.yaml
          retention-days: 1

  deploy-ui:
    runs-on: ubuntu-latest
    if: ${{ always() && inputs.deployStatic && !inputs.uiOnly && needs.detect-changes.outputs.ui_changed == 'true' && (needs.deploy-crud.result == 'success' || needs.deploy-crud.result == 'skipped') && (needs.deploy-agents.result == 'success' || needs.deploy-agents.result == 'skipped') && (needs.sync-apim.result == 'success' || needs.sync-apim.result == 'skipped') && (needs.smoke-apim.result == 'success' || needs.smoke-apim.result == 'skipped') }}
    needs:
      - provision
      - detect-changes
      - deploy-crud
      - deploy-agents
      - sync-apim
      - smoke-apim
    environment: ${{ inputs.githubEnvironment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup azd
        uses: Azure/setup-azd@v2

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Resolve API URL
        id: api
        shell: bash
        run: |
          set -euo pipefail

          RESOURCE_GROUP="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          EXPECTED_APIM_URL="${{ needs.provision.outputs.APIM_GATEWAY_URL }}"

          APIM_URL=$(az apim show \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-apim" \
            --resource-group "$RESOURCE_GROUP" \
            --query "gatewayUrl" -o tsv)

          APIM_URL="$(echo "$APIM_URL" | xargs)"
          EXPECTED_APIM_URL="$(echo "$EXPECTED_APIM_URL" | xargs)"

          APIM_URL="${APIM_URL%/}"
          EXPECTED_APIM_URL="${EXPECTED_APIM_URL%/}"

          if [ "$EXPECTED_APIM_URL" = "null" ]; then
            EXPECTED_APIM_URL=""
          fi

          if [ -z "$APIM_URL" ]; then
            echo "Failed to resolve APIM gatewayUrl for resource group $RESOURCE_GROUP. Ensure APIM exists and is provisioned before deploying UI." >&2
            exit 1
          fi

          RESOLVED_API_URL="$APIM_URL"

          if [ -z "$EXPECTED_APIM_URL" ]; then
            echo "::warning::Provision output APIM_GATEWAY_URL is empty; using live APIM gateway URL '$APIM_URL' for UI build."
          elif [ "$APIM_URL" != "$EXPECTED_APIM_URL" ]; then
            echo "APIM URL drift detected between provision output and live APIM gateway URL." >&2
            echo "Expected: $EXPECTED_APIM_URL" >&2
            echo "Actual:   $APIM_URL" >&2
            exit 1
          fi

          if [ -n "${{ inputs.apiBaseUrl }}" ]; then
            OVERRIDE_API_URL="$(echo "${{ inputs.apiBaseUrl }}" | xargs)"
            OVERRIDE_API_URL="${OVERRIDE_API_URL%/}"

            if [ "$OVERRIDE_API_URL" != "$APIM_URL" ]; then
              echo "apiBaseUrl override drift detected. Provided URL does not match live APIM gateway URL." >&2
              echo "Provided: $OVERRIDE_API_URL" >&2
              echo "Expected: $APIM_URL" >&2
              exit 1
            fi

            RESOLVED_API_URL="$OVERRIDE_API_URL"
          fi

          echo "api_url=$RESOLVED_API_URL" >> "$GITHUB_OUTPUT"
          echo "crud_api_url=$RESOLVED_API_URL" >> "$GITHUB_OUTPUT"
          echo "agent_api_url=${RESOLVED_API_URL}/agents" >> "$GITHUB_OUTPUT"

      - name: Smoke test APIM API health before UI deploy
        if: ${{ !(inputs.skipApiSmokeChecks == true || inputs.skipApiSmokeChecks == 'true') }}
        shell: bash
        run: |
          set -euo pipefail

          API_BASE_URL="${{ steps.api.outputs.api_url }}"
          API_BASE_URL="${API_BASE_URL%/}"

          smoke_endpoint() {
            local url="$1"
            local label="$2"

            for attempt in $(seq 1 20); do
              if ! STATUS_CODE=$(curl -sS -o /tmp/api-health.json -w "%{http_code}" "$url"); then
                STATUS_CODE="000"
              fi
              if [ "$STATUS_CODE" = "200" ]; then
                echo "[OK] $label: $url"
                return 0
              fi
              echo "Attempt $attempt/20: $label returned HTTP $STATUS_CODE"
              sleep 10
            done

            echo "[FAIL] $label: $url" >&2
            cat /tmp/api-health.json 2>/dev/null || true
            return 1
          }

          smoke_endpoint "${API_BASE_URL}/api/ready" "crud-ready"
          smoke_endpoint "${API_BASE_URL}/api/products?limit=1" "crud-products"
          smoke_endpoint "${API_BASE_URL}/api/categories" "crud-categories"

      - name: Resolve SWA deployment token
        id: swa
        shell: bash
        run: |
          set -euo pipefail

          RESOURCE_GROUP="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          SWA_NAME="${{ inputs.projectName }}-ui-${{ inputs.environment }}"

          mapfile -t AZD_UI_SWAS < <(az resource list \
            --resource-group "$RESOURCE_GROUP" \
            --resource-type Microsoft.Web/staticSites \
            --query "[?tags.\"azd-service-name\"=='ui'].name" -o tsv)

          if [ "${#AZD_UI_SWAS[@]}" -ne 1 ] || [ "${AZD_UI_SWAS[0]}" != "$SWA_NAME" ]; then
            echo "UI resource ownership contract violation in $RESOURCE_GROUP." >&2
            echo "Expected exactly one azd-owned SWA named $SWA_NAME." >&2
            echo "Found: ${AZD_UI_SWAS[*]:-<none>}" >&2
            exit 1
          fi

          az staticwebapp show --name "$SWA_NAME" --resource-group "$RESOURCE_GROUP" >/dev/null

          SWA_TOKEN=$(az staticwebapp secrets list \
            --name "$SWA_NAME" \
            --resource-group "$RESOURCE_GROUP" \
            --query "properties.apiKey" -o tsv)

          SWA_TOKEN="$(echo "$SWA_TOKEN" | xargs)"
          if [ -z "$SWA_TOKEN" ]; then
            echo "Failed to resolve deployment token for SWA $SWA_NAME"
            exit 1
          fi

          echo "::add-mask::$SWA_TOKEN"
          echo "swa_name=$SWA_NAME" >> "$GITHUB_OUTPUT"
          echo "swa_token=$SWA_TOKEN" >> "$GITHUB_OUTPUT"

      - name: Deploy UI via Static Web Apps action
        uses: Azure/static-web-apps-deploy@v1
        with:
          action: upload
          azure_static_web_apps_api_token: ${{ steps.swa.outputs.swa_token }}
          app_location: apps/ui
          output_location: ''
          skip_api_build: true
          app_build_command: yarn install --frozen-lockfile && yarn build
        env:
          NEXT_PUBLIC_API_URL: ${{ steps.api.outputs.api_url }}
          NEXT_PUBLIC_CRUD_API_URL: ${{ steps.api.outputs.crud_api_url }}
          NEXT_PUBLIC_AGENT_API_URL: ${{ steps.api.outputs.agent_api_url }}
          NEXT_PUBLIC_DEV_AUTH_MOCK: ${{ inputs.environment == 'dev' && 'true' || vars.NEXT_PUBLIC_DEV_AUTH_MOCK }}
          NEXT_PUBLIC_DEV_AUTH_MOCK_ALLOW_PROD: ${{ inputs.environment == 'dev' && 'false' || vars.NEXT_PUBLIC_DEV_AUTH_MOCK_ALLOW_PROD }}
          NEXT_PUBLIC_ENTRA_CLIENT_ID: ${{ vars.NEXT_PUBLIC_ENTRA_CLIENT_ID }}
          NEXT_PUBLIC_ENTRA_TENANT_ID: ${{ vars.NEXT_PUBLIC_ENTRA_TENANT_ID }}

      - name: Smoke test UI host and API health after deploy
        shell: bash
        run: |
          set -euo pipefail

          RESOURCE_GROUP="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          SWA_NAME="${{ steps.swa.outputs.swa_name }}"
          API_BASE_URL="${{ steps.api.outputs.api_url }}"

          SWA_HOSTNAME=$(az staticwebapp show \
            --name "$SWA_NAME" \
            --resource-group "$RESOURCE_GROUP" \
            --query "defaultHostname" -o tsv)

          SWA_HOSTNAME="$(echo "$SWA_HOSTNAME" | xargs)"
          if [ -z "$SWA_HOSTNAME" ]; then
            echo "Could not resolve Static Web App hostname for $SWA_NAME" >&2
            exit 1
          fi

          UI_URL="https://$SWA_HOSTNAME"
          for attempt in $(seq 1 20); do
            if ! STATUS_CODE=$(curl -sS -o /tmp/ui-homepage.html -w "%{http_code}" "$UI_URL"); then
              STATUS_CODE="000"
            fi
            if [ "$STATUS_CODE" = "200" ]; then
              echo "UI homepage is reachable: $UI_URL"
              break
            fi

            if [ "$attempt" -eq 20 ]; then
              echo "UI smoke test failed with HTTP $STATUS_CODE at $UI_URL" >&2
              exit 1
            fi
            sleep 10
          done

          API_HEALTH_URL="${API_BASE_URL%/}/api/ready"
          if ! STATUS_CODE=$(curl -sS -o /tmp/ui-api-health.json -w "%{http_code}" "$API_HEALTH_URL"); then
            echo "API health check failed after UI deploy due to transport error at $API_HEALTH_URL" >&2
            cat /tmp/ui-api-health.json 2>/dev/null || true
            exit 1
          fi
          if [ "$STATUS_CODE" != "200" ]; then
            echo "API health check failed after UI deploy with HTTP $STATUS_CODE at $API_HEALTH_URL" >&2
            cat /tmp/ui-api-health.json 2>/dev/null || true
            exit 1
          fi

          PRODUCTS_URL="${API_BASE_URL%/}/api/products?limit=1"
          if ! STATUS_CODE=$(curl -sS -o /tmp/ui-api-products.json -w "%{http_code}" "$PRODUCTS_URL"); then
            echo "Products smoke check failed after UI deploy due to transport error at $PRODUCTS_URL" >&2
            cat /tmp/ui-api-products.json 2>/dev/null || true
            exit 1
          fi
          if [ "$STATUS_CODE" != "200" ]; then
            echo "Products smoke check failed after UI deploy with HTTP $STATUS_CODE at $PRODUCTS_URL" >&2
            cat /tmp/ui-api-products.json 2>/dev/null || true
            exit 1
          fi

          CATEGORIES_URL="${API_BASE_URL%/}/api/categories"
          if ! STATUS_CODE=$(curl -sS -o /tmp/ui-api-categories.json -w "%{http_code}" "$CATEGORIES_URL"); then
            echo "Categories smoke check failed after UI deploy due to transport error at $CATEGORIES_URL" >&2
            cat /tmp/ui-api-categories.json 2>/dev/null || true
            exit 1
          fi
          if [ "$STATUS_CODE" != "200" ]; then
            echo "Categories smoke check failed after UI deploy with HTTP $STATUS_CODE at $CATEGORIES_URL" >&2
            cat /tmp/ui-api-categories.json 2>/dev/null || true
            exit 1
          fi

  deploy-ui-token:
    runs-on: ubuntu-latest
    if: ${{ inputs.deployStatic && inputs.uiOnly }}
    environment: ${{ inputs.githubEnvironment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Resolve API URL
        id: api
        shell: bash
        run: |
          set -euo pipefail

          RESOURCE_GROUP="${{ inputs.projectName }}-${{ inputs.environment }}-rg"

          APIM_URL=$(az apim show \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-apim" \
            --resource-group "$RESOURCE_GROUP" \
            --query "gatewayUrl" -o tsv)

          APIM_URL="$(echo "$APIM_URL" | xargs)"
          APIM_URL="${APIM_URL%/}"

          if [ -z "$APIM_URL" ]; then
            echo "Could not resolve APIM gateway URL for UI-only deployment." >&2
            exit 1
          fi

          RESOLVED_API_URL="$APIM_URL"

          if [ -n "${{ inputs.apiBaseUrl }}" ]; then
            OVERRIDE_API_URL="$(echo "${{ inputs.apiBaseUrl }}" | xargs)"
            OVERRIDE_API_URL="${OVERRIDE_API_URL%/}"

            if [ "$OVERRIDE_API_URL" != "$APIM_URL" ]; then
              echo "apiBaseUrl override drift detected. Provided URL does not match live APIM gateway URL." >&2
              echo "Provided: $OVERRIDE_API_URL" >&2
              echo "Expected: $APIM_URL" >&2
              exit 1
            fi

            RESOLVED_API_URL="$OVERRIDE_API_URL"
          fi

          if [ -z "$RESOLVED_API_URL" ]; then
            echo "UI-only deployment could not resolve API base URL. Provide 'apiBaseUrl' input or ensure the default APIM host is valid." >&2
            exit 1
          fi

          echo "api_url=$RESOLVED_API_URL" >> "$GITHUB_OUTPUT"
          echo "crud_api_url=$RESOLVED_API_URL" >> "$GITHUB_OUTPUT"
          echo "agent_api_url=${RESOLVED_API_URL}/agents" >> "$GITHUB_OUTPUT"

      - name: Smoke test APIM API health before UI deploy
        shell: bash
        run: |
          set -euo pipefail

          API_BASE_URL="${{ steps.api.outputs.api_url }}"
          API_BASE_URL="${API_BASE_URL%/}"

          smoke_endpoint() {
            local url="$1"
            local label="$2"
            if ! STATUS_CODE=$(curl -sS -o /tmp/ui-token-health-pre.json -w "%{http_code}" "$url"); then
              echo "$label check failed before UI-only deployment due to transport error at $url" >&2
              cat /tmp/ui-token-health-pre.json 2>/dev/null || true
              exit 1
            fi
            if [ "$STATUS_CODE" != "200" ]; then
              echo "$label check failed before UI-only deployment with HTTP $STATUS_CODE at $url" >&2
              cat /tmp/ui-token-health-pre.json 2>/dev/null || true
              exit 1
            fi
          }

          smoke_endpoint "${API_BASE_URL}/api/ready" "APIM ready"
          smoke_endpoint "${API_BASE_URL}/api/products?limit=1" "Products"
          smoke_endpoint "${API_BASE_URL}/api/categories" "Categories"

      - name: Resolve canonical SWA deployment token (token mode)
        id: swa_ui_only
        shell: bash
        run: |
          set -euo pipefail

          RESOURCE_GROUP="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          SWA_NAME="${{ inputs.projectName }}-ui-${{ inputs.environment }}"

          mapfile -t AZD_UI_SWAS < <(az resource list \
            --resource-group "$RESOURCE_GROUP" \
            --resource-type Microsoft.Web/staticSites \
            --query "[?tags.\"azd-service-name\"=='ui'].name" -o tsv)

          if [ "${#AZD_UI_SWAS[@]}" -ne 1 ] || [ "${AZD_UI_SWAS[0]}" != "$SWA_NAME" ]; then
            echo "UI resource ownership contract violation in $RESOURCE_GROUP." >&2
            echo "Expected exactly one azd-owned SWA named $SWA_NAME." >&2
            echo "Found: ${AZD_UI_SWAS[*]:-<none>}" >&2
            exit 1
          fi

          az staticwebapp show --name "$SWA_NAME" --resource-group "$RESOURCE_GROUP" >/dev/null

          SWA_TOKEN=$(az staticwebapp secrets list \
            --name "$SWA_NAME" \
            --resource-group "$RESOURCE_GROUP" \
            --query "properties.apiKey" -o tsv)

          SWA_TOKEN="$(echo "$SWA_TOKEN" | xargs)"
          if [ -z "$SWA_TOKEN" ]; then
            echo "Failed to resolve deployment token for canonical SWA $SWA_NAME in $RESOURCE_GROUP." >&2
            exit 1
          fi

          echo "::add-mask::$SWA_TOKEN"
          echo "swa_name=$SWA_NAME" >> "$GITHUB_OUTPUT"
          echo "swa_token=$SWA_TOKEN" >> "$GITHUB_OUTPUT"

      - name: Deploy UI via Static Web Apps action (token mode)
        uses: Azure/static-web-apps-deploy@v1
        with:
          action: upload
          azure_static_web_apps_api_token: ${{ steps.swa_ui_only.outputs.swa_token }}
          app_location: apps/ui
          output_location: ''
          skip_api_build: true
          app_build_command: yarn install --frozen-lockfile && yarn build
        env:
          NEXT_PUBLIC_API_URL: ${{ steps.api.outputs.api_url }}
          NEXT_PUBLIC_CRUD_API_URL: ${{ steps.api.outputs.crud_api_url }}
          NEXT_PUBLIC_AGENT_API_URL: ${{ steps.api.outputs.agent_api_url }}
          NEXT_PUBLIC_DEV_AUTH_MOCK: ${{ inputs.environment == 'dev' && 'true' || vars.NEXT_PUBLIC_DEV_AUTH_MOCK }}
          NEXT_PUBLIC_DEV_AUTH_MOCK_ALLOW_PROD: ${{ inputs.environment == 'dev' && 'false' || vars.NEXT_PUBLIC_DEV_AUTH_MOCK_ALLOW_PROD }}
          NEXT_PUBLIC_ENTRA_CLIENT_ID: ${{ vars.NEXT_PUBLIC_ENTRA_CLIENT_ID }}
          NEXT_PUBLIC_ENTRA_TENANT_ID: ${{ vars.NEXT_PUBLIC_ENTRA_TENANT_ID }}

      - name: Smoke test API health after UI-only deploy
        if: ${{ !(inputs.skipApiSmokeChecks == true || inputs.skipApiSmokeChecks == 'true') }}
        shell: bash
        run: |
          set -euo pipefail

          API_BASE_URL="${{ steps.api.outputs.api_url }}"
          API_BASE_URL="${API_BASE_URL%/}"

          smoke_endpoint() {
            local url="$1"
            local label="$2"
            if ! STATUS_CODE=$(curl -sS -o /tmp/ui-token-health-post.json -w "%{http_code}" "$url"); then
              echo "$label check failed after UI-only deployment due to transport error at $url" >&2
              cat /tmp/ui-token-health-post.json 2>/dev/null || true
              exit 1
            fi
            if [ "$STATUS_CODE" != "200" ]; then
              echo "$label check failed after UI-only deployment with HTTP $STATUS_CODE at $url" >&2
              cat /tmp/ui-token-health-post.json 2>/dev/null || true
              exit 1
            fi
          }

          smoke_endpoint "${API_BASE_URL}/api/health" "APIM health"
          smoke_endpoint "${API_BASE_URL}/api/products?limit=1" "Products"
          smoke_endpoint "${API_BASE_URL}/api/categories" "Categories"

  debug-detect-outputs:
    runs-on: ubuntu-latest
    if: ${{ !inputs.uiOnly }}
    needs:
      - detect-changes
    steps:
      - name: Dump detect-changes outputs
        shell: bash
        run: |
          echo "=== detect-changes outputs ==="
          echo "crud_changed=${{ needs.detect-changes.outputs.crud_changed }}"
          echo "ui_changed=${{ needs.detect-changes.outputs.ui_changed }}"
          echo "agents_changed=${{ needs.detect-changes.outputs.agents_changed }}"
          echo "changed_agent_services_csv=${{ needs.detect-changes.outputs.changed_agent_services_csv }}"
          echo "changed_aks_services_csv=${{ needs.detect-changes.outputs.changed_aks_services_csv }}"
          echo "lib_changed=${{ needs.detect-changes.outputs.lib_changed }}"
          echo "infra_changed=${{ needs.detect-changes.outputs.infra_changed }}"
          echo "changed_agents_matrix=${{ needs.detect-changes.outputs.changed_agents_matrix }}"
          echo "changed_aks_matrix=${{ needs.detect-changes.outputs.changed_aks_matrix }}"
          echo "=== end ==="

  deploy-agents:
    runs-on: ubuntu-latest
    # Use always() to ensure condition is evaluated even with skipped dependencies (deploy-foundry-models)
    if: ${{ always() && !cancelled() && !inputs.uiOnly && needs.detect-changes.outputs.changed_agent_services_csv != '' && needs.detect-changes.result == 'success' && (needs.deploy-crud.result == 'success' || needs.deploy-crud.result == 'skipped') && (needs.deploy-foundry-models.result == 'success' || needs.deploy-foundry-models.result == 'skipped') && (needs.build-aks-images.result == 'success' || needs.build-aks-images.result == 'skipped') && (needs.restore-acr-build-access.result == 'success' || needs.restore-acr-build-access.result == 'skipped') }}
    needs:
      - provision
      - deploy-crud
      - deploy-foundry-models
      - detect-changes
      - build-aks-images
      - restore-acr-build-access
    environment: ${{ inputs.githubEnvironment }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.detect-changes.outputs.changed_agents_matrix) }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ env.DEPLOY_SOURCE_CHECKOUT_REF }}

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Install kubelogin
        run: |
          for attempt in 1 2 3 4 5; do
            if az aks install-cli --kubelogin-version latest; then
              exit 0
            fi
            echo "kubelogin install failed on attempt ${attempt}; retrying..."
            sleep 15
          done
          echo "kubelogin install failed after retries"
          exit 1

      - name: Get AKS credentials
        run: |
          az aks get-credentials \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --overwrite-existing
          kubelogin convert-kubeconfig -l azurecli

      - name: Verify AKS API connectivity
        shell: bash
        run: |
          set -euo pipefail
          AKS_FQDN=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed 's|https://||;s|:.*||')
          echo "Probing AKS API server DNS: $AKS_FQDN"
          for attempt in $(seq 1 10); do
            if kubectl cluster-info >/dev/null 2>&1; then
              echo "AKS API reachable (attempt $attempt)."
              exit 0
            fi
            BACKOFF=$((5 * attempt))
            echo "Attempt $attempt/10 failed (DNS or TCP). Retrying in ${BACKOFF}s..."
            sleep "$BACKOFF"
          done
          echo "AKS API DNS resolution failed after 10 attempts for $AKS_FQDN." >&2
          nslookup "$AKS_FQDN" || true
          exit 1

      - name: Ensure agents namespace exists
        shell: bash
        run: |
          set -euo pipefail
          if ! kubectl get namespace holiday-peak-agents >/dev/null 2>&1; then
            kubectl create namespace holiday-peak-agents
            echo "Created namespace holiday-peak-agents"
          fi
          kubectl label namespace holiday-peak-agents holiday-peak/ingress-allowed=true --overwrite

      - name: Resolve AKS managed identity client ID
        run: |
          AKS_MI_CLIENT_ID=$(az aks show \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --query "identityProfile.kubeletidentity.clientId" -o tsv)
          echo "WORKLOAD_AZURE_CLIENT_ID=${AKS_MI_CLIENT_ID}" >> "$GITHUB_ENV"

      - name: Resolve publication context
        shell: bash
        run: |
          set -euo pipefail

          AGC_GATEWAY_CLASS_VALUE="${{ needs.provision.outputs.AGC_GATEWAY_CLASS || 'azure-alb-external' }}"
          AGC_SUBNET_ID_VALUE="${{ needs.provision.outputs.AGC_SUBNET_ID }}"
          SHARED_GATEWAY_JSON="$(kubectl get gateway holiday-peak-agc -n holiday-peak-crud -o json 2>/dev/null || true)"

          get_first_alb_association() {
            local alb_name="$1"
            local alb_namespace="$2"

            if [ -z "$alb_name" ] || [ -z "$alb_namespace" ]; then
              return 0
            fi

            kubectl get applicationloadbalancer "$alb_name" -n "$alb_namespace" -o json 2>/dev/null \
              | jq -r '.spec.associations[]? | select(type == "string" and . != "")' \
              | head -n 1 || true
          }

          if [ -z "$AGC_SUBNET_ID_VALUE" ]; then
            AGC_SUBNET_ID_VALUE="$(get_first_alb_association "holiday-peak-agc" "holiday-peak-crud")"
            if [ -n "$AGC_SUBNET_ID_VALUE" ]; then
              echo "::notice::Recovered AGC render subnet from the live shared ApplicationLoadBalancer association."
            fi
          fi

          if [ -z "$AGC_SUBNET_ID_VALUE" ] && [ -n "$SHARED_GATEWAY_JSON" ]; then
            AGC_SHARED_ALB_NAME="$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '.metadata.annotations["alb.networking.azure.io/alb-name"] // empty' || true)"
            AGC_SHARED_ALB_NAMESPACE="$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '.metadata.annotations["alb.networking.azure.io/alb-namespace"] // empty' || true)"
            if [ -z "$AGC_SHARED_ALB_NAMESPACE" ]; then
              AGC_SHARED_ALB_NAMESPACE="holiday-peak-crud"
            fi
            AGC_SUBNET_ID_VALUE="$(get_first_alb_association "$AGC_SHARED_ALB_NAME" "$AGC_SHARED_ALB_NAMESPACE")"
            if [ -n "$AGC_SUBNET_ID_VALUE" ]; then
              echo "::notice::Recovered AGC render subnet from the ApplicationLoadBalancer referenced by the live shared Gateway."
            fi
          fi

          echo "PUBLICATION_MODE=agc" >> "$GITHUB_ENV"
          echo "AGC_SUPPORT_ENABLED=${{ needs.provision.outputs.AGC_SUPPORT_ENABLED || 'false' }}" >> "$GITHUB_ENV"
          echo "AGC_GATEWAY_CLASS=$AGC_GATEWAY_CLASS_VALUE" >> "$GITHUB_ENV"
          echo "AGC_SUBNET_ID=$AGC_SUBNET_ID_VALUE" >> "$GITHUB_ENV"

          if [ -z "$AGC_SUBNET_ID_VALUE" ]; then
            echo "::notice::No live AGC subnet association was available during render publication-context resolution. Later AGC readiness validation remains authoritative."
          fi

      - name: Download tested image reference
        uses: actions/download-artifact@v4
        with:
          name: tested-image-${{ matrix.service }}
          path: ${{ runner.temp }}/tested-image/${{ matrix.service }}

      - name: Deploy service
        shell: bash
        run: |
          set -euo pipefail
          export AZURE_CLIENT_ID="${WORKLOAD_AZURE_CLIENT_ID}"
          SERVICE_NAME="${{ matrix.service }}"
          IMAGE_REF=$(tr -d '\r\n' < "${RUNNER_TEMP}/tested-image/${SERVICE_NAME}/image-ref.txt")
          SERVICE_IMAGE_VAR_NAME="SERVICE_$(printf '%s' "$SERVICE_NAME" | tr '[:lower:]-' '[:upper:]_')_IMAGE_NAME"
          export "${SERVICE_IMAGE_VAR_NAME}=${IMAGE_REF}"

          # Phase B (ADR-017): Render only — Flux reconciles from Git
          bash .infra/azd/hooks/render-helm.sh "$SERVICE_NAME"
          echo "Rendered ${SERVICE_NAME} manifest for Flux reconciliation (image: ${IMAGE_REF})."
        env:
          DEPLOY_ENV: ${{ inputs.environment }}
          AZURE_TENANT_ID: ${{ env.AZURE_TENANT_ID }}
          AI_SEARCH_ENDPOINT: ${{ needs.provision.outputs.AI_SEARCH_ENDPOINT }}
          AI_SEARCH_INDEX: ${{ needs.provision.outputs.AI_SEARCH_INDEX }}
          AI_SEARCH_VECTOR_INDEX: ${{ needs.provision.outputs.AI_SEARCH_VECTOR_INDEX }}
          AI_SEARCH_INDEXER_NAME: ${{ needs.provision.outputs.AI_SEARCH_INDEXER_NAME }}
          EMBEDDING_DEPLOYMENT_NAME: ${{ needs.provision.outputs.EMBEDDING_DEPLOYMENT_NAME }}
          AI_SEARCH_AUTH_MODE: ${{ needs.provision.outputs.AI_SEARCH_AUTH_MODE }}
          CATALOG_SEARCH_REQUIRE_AI_SEARCH: ${{ matrix.service == 'ecommerce-catalog-search' && 'true' || 'false' }}
          PROJECT_ENDPOINT: ${{ needs.provision.outputs.PROJECT_ENDPOINT }}
          PROJECT_NAME: ${{ needs.provision.outputs.PROJECT_NAME }}
          FOUNDRY_AGENT_NAME_FAST: ${{ matrix.service }}-fast
          FOUNDRY_AGENT_NAME_RICH: ${{ matrix.service }}-rich
          FOUNDRY_AUTO_ENSURE_ON_STARTUP: ${{ env.FOUNDRY_RUNTIME_AUTO_ENSURE_ON_STARTUP }}
          FOUNDRY_STRICT_ENFORCEMENT: ${{ env.FOUNDRY_RUNTIME_STRICT_ENFORCEMENT }}
          MODEL_DEPLOYMENT_NAME_FAST: gpt-5-nano
          MODEL_DEPLOYMENT_NAME_RICH: gpt-5
          POSTGRES_AUTH_MODE: ${{ needs.provision.outputs.POSTGRES_AUTH_MODE }}
          COSMOS_ACCOUNT_URI: ${{ needs.provision.outputs.COSMOS_ACCOUNT_URI }}
          COSMOS_DATABASE: ${{ needs.provision.outputs.COSMOS_DATABASE }}
          COSMOS_CONTAINER: ${{ needs.provision.outputs.COSMOS_CONTAINER }}
          BLOB_ACCOUNT_URL: ${{ needs.provision.outputs.BLOB_ACCOUNT_URL }}
          BLOB_CONTAINER: ${{ needs.provision.outputs.BLOB_CONTAINER }}
          REDIS_HOST: ${{ needs.provision.outputs.REDIS_HOST }}
          EVENT_HUB_NAMESPACE: ${{ needs.provision.outputs.EVENT_HUB_NAMESPACE }}
          KEY_VAULT_URI: ${{ needs.provision.outputs.KEY_VAULT_URI }}
          APPLICATIONINSIGHTS_CONNECTION_STRING: ${{ needs.provision.outputs.APPLICATIONINSIGHTS_CONNECTION_STRING }}
          POSTGRES_HOST: ${{ needs.provision.outputs.POSTGRES_HOST }}
          POSTGRES_USER: ${{ needs.provision.outputs.POSTGRES_AUTH_MODE == 'password' && needs.provision.outputs.POSTGRES_ADMIN_USER || needs.provision.outputs.POSTGRES_USER }}
          POSTGRES_DATABASE: ${{ needs.provision.outputs.POSTGRES_DATABASE }}

      - name: Upload rendered agent manifest for verification
        if: ${{ success() }}
        uses: actions/upload-artifact@v4
        with:
          name: rendered-manifest-${{ matrix.service }}
          path: .kubernetes/rendered/${{ matrix.service }}/all.yaml
          retention-days: 1

  # ADR-017, Phase 2b — image-tag bridge (DEFERRED, see issue #1099 follow-up).
  #
  # The previous ``commit-rendered-manifests`` job pushed bot-generated YAML
  # straight at ``refs/heads/main``, which the ``main-governance-baseline``
  # ruleset (GH013) rejects. Flux now reconciles HelmRelease CRDs from
  # ``.kubernetes/releases/{crud,agents}`` and the helm-controller renders the
  # chart in-cluster on every reconciliation. The image tag pinned in each
  # HelmRelease YAML remains the source of truth for desired state.
  #
  # A previous attempt (PR #1097) embedded a ``open-image-tag-bump-pr`` job
  # here that requested ``permissions.pull-requests: write``. GitHub rejects
  # nested-workflow callees that escalate permissions beyond what the caller
  # grants (``permissions can only be maintained or reduced — not elevated``).
  # The 27 per-service entrypoints grant only ``id-token | contents | issues``
  # to the ``uses:`` job, so the entire reusable workflow ``startup_failure``-d
  # at the GitHub orchestrator before any runner allocation — and the failure
  # is not detectable by ``actionlint`` or ``yaml.safe_load`` because it is a
  # cross-file semantic rule.
  #
  # The architecturally correct implementation per ADR-017 §"Phase 2b" is
  # Flux ``ImageRepository`` + ``ImagePolicy`` + ``ImageUpdateAutomation`` CRDs
  # with the Notification Controller GitHub provider opening the bridge PR.
  # That work is tracked separately so the deploy chain is restored without
  # locking the workflow into a GHA-resident bridge that fights the protected
  # branch model. Until Phase 2b lands properly, new image tags continue to
  # roll via the existing ``azd deploy`` path, and HelmRelease YAML is updated
  # through normal PRs reviewed by humans.

  wait-flux-reconciliation:
    runs-on: ubuntu-latest
    if: ${{ always() && !cancelled() && !inputs.uiOnly && (needs.deploy-crud.result == 'success' || needs.deploy-agents.result == 'success') }}
    needs:
      - deploy-crud
      - deploy-agents
      - detect-changes
    environment: ${{ inputs.githubEnvironment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Install kubelogin
        run: |
          for attempt in 1 2 3 4 5; do
            if az aks install-cli --kubelogin-version latest; then
              exit 0
            fi
            echo "kubelogin install failed on attempt ${attempt}; retrying..."
            sleep 15
          done
          echo "kubelogin install failed after retries"
          exit 1

      - name: Get AKS credentials
        run: |
          az aks get-credentials \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --overwrite-existing
          kubelogin convert-kubeconfig -l azurecli

      - name: Trigger Flux reconciliation
        shell: bash
        run: |
          set -euo pipefail
          echo "Triggering Flux kustomization reconciliation..."

          DEFAULT_BRANCH="${REPOSITORY_DEFAULT_BRANCH}"
          SOURCE_REF="${DEPLOY_SOURCE_REF}"
          FLUX_SOURCE="${FLUX_SOURCE_NAME}"
          FLUX_NAMESPACE="${FLUX_SOURCE_NAMESPACE}"

          if [ -z "$DEFAULT_BRANCH" ]; then
            echo "::error::Repository default branch is required for Flux reconciliation."
            exit 1
          fi

          PREVIEW_BRANCH=""
          if [ "${{ inputs.environment }}" != "prod" ] && [[ "$SOURCE_REF" == refs/heads/* ]]; then
            CANDIDATE_BRANCH="${SOURCE_REF#refs/heads/}"
            if [ -n "$CANDIDATE_BRANCH" ] && [ "$CANDIDATE_BRANCH" != "$DEFAULT_BRANCH" ]; then
              PREVIEW_BRANCH="$CANDIDATE_BRANCH"
            fi
          fi

          if [ -n "$PREVIEW_BRANCH" ]; then
            echo "Switching Flux source '${FLUX_SOURCE}' to preview branch '${PREVIEW_BRANCH}'."
            PATCH=$(printf '{"spec":{"ref":{"branch":"%s"}}}' "$PREVIEW_BRANCH")
            kubectl patch gitrepository.source.toolkit.fluxcd.io "$FLUX_SOURCE" -n "$FLUX_NAMESPACE" --type=merge -p "$PATCH"

            ACTIVE_BRANCH=$(kubectl get gitrepository.source.toolkit.fluxcd.io "$FLUX_SOURCE" -n "$FLUX_NAMESPACE" -o jsonpath='{.spec.ref.branch}')
            if [ "$ACTIVE_BRANCH" != "$PREVIEW_BRANCH" ]; then
              echo "::error::Flux source '${FLUX_SOURCE}' expected branch '${PREVIEW_BRANCH}' but found '${ACTIVE_BRANCH}'."
              exit 1
            fi
          else
            echo "Using Flux source '${FLUX_SOURCE}' on default branch '${DEFAULT_BRANCH}'."
          fi

          # Install Flux CLI if not present
          if ! command -v flux &>/dev/null; then
            curl -s https://fluxcd.io/install.sh | bash 2>/dev/null || true
          fi

          # Trigger reconciliation for both kustomizations
          flux reconcile source git "$FLUX_SOURCE" -n "$FLUX_NAMESPACE" --timeout=2m || true
          flux reconcile kustomization holiday-peak-gitops-holiday-peak-crud -n flux-system --timeout=5m || true
          flux reconcile kustomization holiday-peak-gitops-holiday-peak-agents -n flux-system --timeout=5m || true

          echo "Flux reconciliation triggered."

      - name: Wait for CRUD rollout
        if: ${{ needs.detect-changes.outputs.crud_changed == 'true' }}
        shell: bash
        run: |
          set -euo pipefail
          echo "Waiting for CRUD deployment rollout via Flux..."
          DEPLOYMENT_NAME=$(kubectl get deployment -n holiday-peak-crud -l 'app=crud-service,canary=false' -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
          if [ -n "$DEPLOYMENT_NAME" ]; then
            kubectl rollout status "deployment/${DEPLOYMENT_NAME}" -n holiday-peak-crud --timeout=300s
            echo "CRUD rollout completed: ${DEPLOYMENT_NAME}"
          else
            echo "No CRUD deployment found — may be first deploy."
          fi

      - name: Wait for agent rollouts
        if: ${{ needs.detect-changes.outputs.changed_agent_services_csv != '' }}
        shell: bash
        run: |
          set -euo pipefail
          echo "Waiting for agent deployment rollouts via Flux..."
          CHANGED="${{ needs.detect-changes.outputs.changed_agent_services_csv }}"
          if [ -z "$CHANGED" ]; then
            echo "No specific changed agents detected; checking all deployments."
            kubectl rollout status deployment --all -n holiday-peak-agents --timeout=600s || true
          else
            IFS=',' read -ra SERVICES <<< "$CHANGED"
            for svc in "${SERVICES[@]}"; do
              svc=$(echo "$svc" | xargs)
              DEPLOYMENT_NAME=$(kubectl get deployment -n holiday-peak-agents -l "app=${svc},canary=false" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
              if [ -n "$DEPLOYMENT_NAME" ]; then
                kubectl rollout status "deployment/${DEPLOYMENT_NAME}" -n holiday-peak-agents --timeout=300s
                echo "Agent rollout completed: ${DEPLOYMENT_NAME}"
              else
                echo "No deployment found for ${svc} — may be first deploy."
              fi
            done
          fi

      - name: Checkout repo for prompt verification scripts
        if: ${{ success() && !inputs.skipPromptGates && needs.detect-changes.outputs.changed_agent_services_csv != '' }}
        uses: actions/checkout@v4
        with:
          ref: ${{ env.DEPLOY_SOURCE_CHECKOUT_REF }}

      - name: Download rendered agent manifests
        if: ${{ success() && !inputs.skipPromptGates && needs.detect-changes.outputs.changed_agent_services_csv != '' }}
        uses: actions/download-artifact@v4
        with:
          path: ${{ runner.temp }}/rendered-manifests
          pattern: rendered-manifest-*
          merge-multiple: false

      - name: Stage rendered manifests into workspace
        if: ${{ success() && !inputs.skipPromptGates && needs.detect-changes.outputs.changed_agent_services_csv != '' }}
        shell: bash
        env:
          CHANGED_AGENT_SERVICES_CSV: ${{ needs.detect-changes.outputs.changed_agent_services_csv }}
        run: |
          set -euo pipefail
          IFS=',' read -ra SERVICES <<< "$CHANGED_AGENT_SERVICES_CSV"
          for raw_svc in "${SERVICES[@]}"; do
            svc=$(echo "$raw_svc" | xargs)
            [ -z "$svc" ] && continue
            src="${RUNNER_TEMP}/rendered-manifests/rendered-manifest-${svc}/all.yaml"
            if [ -f "$src" ]; then
              mkdir -p ".kubernetes/rendered/${svc}"
              cp "$src" ".kubernetes/rendered/${svc}/all.yaml"
              echo "Staged rendered manifest for ${svc}."
            else
              echo "::warning::rendered manifest artifact missing for ${svc}: ${src}"
            fi
          done

      - name: Verify Foundry instructions match repo
        if: ${{ success() && !inputs.skipPromptGates && needs.detect-changes.outputs.changed_agent_services_csv != '' }}
        shell: bash
        env:
          CHANGED_AGENT_SERVICES_CSV: ${{ needs.detect-changes.outputs.changed_agent_services_csv }}
        run: |
          set -euo pipefail
          python3 -m pip install --quiet --disable-pip-version-check \
            "azure-ai-projects>=1.0.0b6" "azure-identity>=1.17"

          FAIL=0
          IFS=',' read -ra SERVICES <<< "$CHANGED_AGENT_SERVICES_CSV"
          for raw_svc in "${SERVICES[@]}"; do
            svc=$(echo "$raw_svc" | xargs)
            [ -z "$svc" ] && continue

            pkg="${svc//-/_}"
            repo_prompt="apps/${svc}/src/${pkg}/prompts/instructions.md"
            if [ ! -f "$repo_prompt" ]; then
              echo "[foundry-gate] ${svc}: no repo prompt at ${repo_prompt} — skipping."
              continue
            fi

            rendered=".kubernetes/rendered/${svc}/all.yaml"
            if [ ! -f "$rendered" ]; then
              echo "::warning::[foundry-gate] ${svc}: rendered manifest not found at ${rendered}; skipping."
              continue
            fi

            if ! grep -q "FOUNDRY_AGENT_NAME_" "$rendered"; then
              echo "[foundry-gate] ${svc}: no FOUNDRY_AGENT_NAME_* envs in rendered manifest — non-Foundry service, skipping."
              continue
            fi

            endpoint=$(awk '
              $1=="-" && $2=="name:" && $3=="PROJECT_ENDPOINT" {found=1; next}
              found && $1=="value:" {sub(/^[^:]*:[[:space:]]*/, ""); gsub(/"/,""); print; exit}
            ' "$rendered" || true)
            project_name=$(awk '
              $1=="-" && $2=="name:" && $3=="PROJECT_NAME" {found=1; next}
              found && $1=="value:" {sub(/^[^:]*:[[:space:]]*/, ""); gsub(/"/,""); print; exit}
            ' "$rendered" || true)

            if [ -z "$endpoint" ]; then
              echo "::warning::[foundry-gate] ${svc}: PROJECT_ENDPOINT missing in rendered manifest; skipping."
              continue
            fi

            for role in FAST RICH; do
              agent_name=$(awk -v key="FOUNDRY_AGENT_NAME_${role}" '
                $1=="-" && $2=="name:" && $3==key {found=1; next}
                found && $1=="value:" {sub(/^[^:]*:[[:space:]]*/, ""); gsub(/"/,""); print; exit}
              ' "$rendered" || true)
              [ -z "$agent_name" ] && continue

              echo "[foundry-gate] ${svc}: verifying agent ${agent_name} against repo prompt..."
              args=(--service "$svc" --project-endpoint "$endpoint" --agent-name "$agent_name")
              if [ -n "$project_name" ]; then
                args+=(--project-name "$project_name")
              fi
              if ! python3 scripts/ci/verify_foundry_prompt.py "${args[@]}"; then
                FAIL=1
              fi
            done
          done

          if [ "$FAIL" -ne 0 ]; then
            echo "::error::Foundry prompt verification failed for one or more agents. See hints above."
            exit 1
          fi

      - name: Verify Flux kustomization health
        shell: bash
        run: |
          set -euo pipefail
          echo "Flux kustomization status:"
          kubectl get kustomizations -n flux-system
          echo ""
          # Report but don't fail — health checks may fail on first deploy (no images yet)
          CRUD_READY=$(kubectl get kustomization holiday-peak-gitops-holiday-peak-crud -n flux-system -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown")
          AGENTS_READY=$(kubectl get kustomization holiday-peak-gitops-holiday-peak-agents -n flux-system -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown")
          echo "CRUD kustomization ready: ${CRUD_READY}"
          echo "Agents kustomization ready: ${AGENTS_READY}"

  validate-agc-readiness:
    runs-on: ubuntu-latest
    if: ${{ always() && !cancelled() && !inputs.uiOnly && (inputs.forceApimSync || needs.detect-changes.outputs.changed_agent_services_csv != '' || needs.detect-changes.outputs.crud_changed == 'true' || needs.detect-changes.outputs.ui_changed == 'true') && (needs.deploy-crud.result == 'success' || needs.deploy-crud.result == 'skipped') && (needs.deploy-agents.result == 'success' || needs.deploy-agents.result == 'skipped') && (needs.wait-flux-reconciliation.result == 'success' || needs.wait-flux-reconciliation.result == 'skipped') }}
    needs:
      - provision
      - deploy-crud
      - deploy-agents
      - detect-changes
      - wait-flux-reconciliation
    environment: ${{ inputs.githubEnvironment }}
    outputs:
      approved_backend_hostnames: ${{ steps.finalize.outputs.approved_backend_hostnames }}
      approved_backend_scheme: ${{ steps.finalize.outputs.approved_backend_scheme }}
      agc_gateway_class: ${{ steps.finalize.outputs.agc_gateway_class }}
      agc_frontend_url: ${{ steps.finalize.outputs.agc_frontend_url }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      CHANGED_AGENT_SERVICES: ${{ needs.detect-changes.outputs.changed_agent_services_csv }}
    steps:
      - uses: actions/checkout@v4

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup azd
        uses: Azure/setup-azd@v2

      - name: Resolve AGC readiness inputs
        id: resolve
        shell: bash
        run: |
          set -euo pipefail
          AGC_ENABLED="${{ needs.provision.outputs.AGC_SUPPORT_ENABLED || 'false' }}"
          AGC_GATEWAY_CLASS_VALUE="${{ needs.provision.outputs.AGC_GATEWAY_CLASS || 'azure-alb-external' }}"

          if [ "$AGC_ENABLED" != "true" ]; then
            echo "AGC support is not enabled for environment '${{ inputs.environment }}'. APIM sync cannot proceed without the canonical AGC edge." >&2
            exit 1
          fi

          if [ -z "$AGC_GATEWAY_CLASS_VALUE" ]; then
            echo "AGC_GATEWAY_CLASS is empty. AGC readiness cannot be validated." >&2
            exit 1
          fi

          echo "agc_gateway_class=$AGC_GATEWAY_CLASS_VALUE" >> "$GITHUB_OUTPUT"

      - name: Install kubelogin
        run: |
          for attempt in 1 2 3 4 5; do
            if az aks install-cli --kubelogin-version latest; then
              exit 0
            fi
            echo "kubelogin install failed on attempt ${attempt}; retrying..."
            sleep 15
          done
          echo "kubelogin install failed after retries"
          exit 1

      - name: Get AKS credentials
        run: |
          az aks get-credentials \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --overwrite-existing
          kubelogin convert-kubeconfig -l azurecli

      - name: Verify AKS API connectivity
        shell: bash
        run: |
          set -euo pipefail
          AKS_FQDN=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed 's|https://||;s|:.*||')
          echo "Probing AKS API server DNS: $AKS_FQDN"
          for attempt in $(seq 1 10); do
            if kubectl cluster-info >/dev/null 2>&1; then
              echo "AKS API reachable (attempt $attempt)."
              exit 0
            fi
            BACKOFF=$((5 * attempt))
            echo "Attempt $attempt/10 failed (DNS or TCP). Retrying in ${BACKOFF}s..."
            sleep "$BACKOFF"
          done
          echo "AKS API DNS resolution failed after 10 attempts for $AKS_FQDN." >&2
          nslookup "$AKS_FQDN" || true
          exit 1

      - name: Ensure preview Flux reconciliation for AGC readiness
        shell: bash
        run: |
          set -euo pipefail

          DEFAULT_BRANCH="${REPOSITORY_DEFAULT_BRANCH}"
          SOURCE_REF="${DEPLOY_SOURCE_REF}"
          FLUX_SOURCE="${FLUX_SOURCE_NAME}"
          FLUX_NAMESPACE="${FLUX_SOURCE_NAMESPACE}"

          if [ -z "$DEFAULT_BRANCH" ]; then
            echo "::error::Repository default branch is required for AGC readiness Flux reconciliation."
            exit 1
          fi

          if [ "${{ inputs.environment }}" = "prod" ] || [ "${{ inputs.environment }}" = "production" ]; then
            echo "Skipping preview Flux reconciliation for production environment '${{ inputs.environment }}'."
            exit 0
          fi

          if [[ "$SOURCE_REF" != refs/heads/* ]]; then
            echo "Skipping preview Flux reconciliation because DEPLOY_SOURCE_REF '$SOURCE_REF' is not a branch ref."
            exit 0
          fi

          PREVIEW_BRANCH="${SOURCE_REF#refs/heads/}"
          if [ -z "$PREVIEW_BRANCH" ] || [ "$PREVIEW_BRANCH" = "$DEFAULT_BRANCH" ]; then
            echo "Skipping preview Flux reconciliation because source branch '${PREVIEW_BRANCH:-<empty>}' matches the default branch '${DEFAULT_BRANCH}'."
            exit 0
          fi

          echo "Ensuring Flux source '${FLUX_SOURCE}' is pinned to preview branch '${PREVIEW_BRANCH}' before AGC readiness validation."
          PATCH=$(printf '{"spec":{"ref":{"branch":"%s"}}}' "$PREVIEW_BRANCH")
          kubectl patch gitrepository.source.toolkit.fluxcd.io "$FLUX_SOURCE" -n "$FLUX_NAMESPACE" --type=merge -p "$PATCH"

          ACTIVE_BRANCH=$(kubectl get gitrepository.source.toolkit.fluxcd.io "$FLUX_SOURCE" -n "$FLUX_NAMESPACE" -o jsonpath='{.spec.ref.branch}')
          if [ "$ACTIVE_BRANCH" != "$PREVIEW_BRANCH" ]; then
            echo "::error::Flux source '${FLUX_SOURCE}' expected branch '${PREVIEW_BRANCH}' but found '${ACTIVE_BRANCH}'."
            exit 1
          fi

          if ! command -v flux >/dev/null 2>&1; then
            curl -s https://fluxcd.io/install.sh | bash
            export PATH="$HOME/.local/bin:$PATH"
          fi

          if ! command -v flux >/dev/null 2>&1; then
            echo "::error::Flux CLI is unavailable after installation attempt."
            exit 1
          fi

          flux reconcile source git "$FLUX_SOURCE" -n "$FLUX_NAMESPACE" --timeout=2m

          SOURCE_REVISION=$(kubectl get gitrepository.source.toolkit.fluxcd.io "$FLUX_SOURCE" -n "$FLUX_NAMESPACE" -o jsonpath='{.status.artifact.revision}')
          if [ -z "$SOURCE_REVISION" ]; then
            echo "::error::Flux source '${FLUX_SOURCE}' did not publish an artifact revision after preview reconcile."
            exit 1
          fi

          if [[ "$SOURCE_REVISION" != "$PREVIEW_BRANCH@"* ]]; then
            echo "::error::Flux source '${FLUX_SOURCE}' fetched unexpected revision '${SOURCE_REVISION}' while preview branch '${PREVIEW_BRANCH}' was required."
            exit 1
          fi

          request_preview_reconcile() {
            local kustomization_name="$1"
            local request_stamp="$2"
            local strict_mode="$3"

            if kubectl annotate kustomization.kustomize.toolkit.fluxcd.io "$kustomization_name" \
              -n flux-system \
              reconcile.fluxcd.io/requestedAt="$request_stamp" \
              --overwrite >/dev/null; then
              echo "Requested Flux reconcile for kustomization '${kustomization_name}'."
              return 0
            fi

            if [ "$strict_mode" = "strict" ]; then
              echo "::error::Failed to request preview reconcile for Flux kustomization '${kustomization_name}'."
              exit 1
            fi

            echo "::warning::Failed to request preview reconcile for advisory Flux kustomization '${kustomization_name}'."
            return 1
          }

          get_preview_observation_window() {
            local kustomization_name="$1"
            local configured_timeout=""

            configured_timeout=$(kubectl get kustomization.kustomize.toolkit.fluxcd.io "$kustomization_name" -n flux-system -o jsonpath='{.spec.timeout}' 2>/dev/null || true)
            if [ -n "$configured_timeout" ]; then
              echo "${configured_timeout}|spec.timeout"
              return 0
            fi

            configured_timeout=$(kubectl get kustomization.kustomize.toolkit.fluxcd.io "$kustomization_name" -n flux-system -o jsonpath='{.spec.interval}' 2>/dev/null || true)
            if [ -n "$configured_timeout" ]; then
              echo "${configured_timeout}|spec.interval"
              return 0
            fi

            echo "5m0s|workflow-fallback"
          }

          duration_to_seconds() {
            local duration="$1"
            local total=0
            local remaining="$duration"

            if [ -z "$remaining" ]; then
              echo "::error::Flux preview observation duration is empty." >&2
              return 1
            fi

            while [ -n "$remaining" ]; do
              if [[ "$remaining" =~ ^([0-9]+)h(.*)$ ]]; then
                total=$((total + ${BASH_REMATCH[1]} * 3600))
                remaining="${BASH_REMATCH[2]}"
                continue
              fi

              if [[ "$remaining" =~ ^([0-9]+)m(.*)$ ]]; then
                total=$((total + ${BASH_REMATCH[1]} * 60))
                remaining="${BASH_REMATCH[2]}"
                continue
              fi

              if [[ "$remaining" =~ ^([0-9]+)s(.*)$ ]]; then
                total=$((total + ${BASH_REMATCH[1]}))
                remaining="${BASH_REMATCH[2]}"
                continue
              fi

              echo "::error::Unsupported Flux preview observation duration '${duration}'." >&2
              return 1
            done

            echo "$total"
          }

          wait_for_preview_revision() {
            local kustomization_name="$1"
            local strict_mode="$2"
            local observation_window=""
            local observation_timeout=""
            local observation_timeout_source=""
            local observation_timeout_seconds=0
            local poll_interval_seconds=10
            local deadline_seconds=0
            local sleep_seconds=0
            local last_attempted_revision=""
            local last_applied_revision=""
            local ready_status=""
            local ready_message=""

            observation_window=$(get_preview_observation_window "$kustomization_name")
            observation_timeout="${observation_window%%|*}"
            observation_timeout_source="${observation_window#*|}"
            observation_timeout_seconds=$(duration_to_seconds "$observation_timeout")
            deadline_seconds=$((SECONDS + observation_timeout_seconds))

            echo "Observing Flux kustomization '${kustomization_name}' for preview revision '${SOURCE_REVISION}' for up to '${observation_timeout}' from ${observation_timeout_source}."
            if [ "$observation_timeout_source" != "spec.timeout" ]; then
              echo "::notice::Flux kustomization '${kustomization_name}' does not declare spec.timeout; preview observation window falls back to '${observation_timeout}' from ${observation_timeout_source}."
            fi

            while :; do
              last_attempted_revision=$(kubectl get kustomization.kustomize.toolkit.fluxcd.io "$kustomization_name" -n flux-system -o jsonpath='{.status.lastAttemptedRevision}' 2>/dev/null || true)
              last_applied_revision=$(kubectl get kustomization.kustomize.toolkit.fluxcd.io "$kustomization_name" -n flux-system -o jsonpath='{.status.lastAppliedRevision}' 2>/dev/null || true)
              ready_status=$(kubectl get kustomization.kustomize.toolkit.fluxcd.io "$kustomization_name" -n flux-system -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
              ready_message=$(kubectl get kustomization.kustomize.toolkit.fluxcd.io "$kustomization_name" -n flux-system -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || true)

              if [ "$last_attempted_revision" = "$SOURCE_REVISION" ] || [ "$last_applied_revision" = "$SOURCE_REVISION" ]; then
                if [ "$ready_status" = "True" ]; then
                  echo "Flux kustomization '${kustomization_name}' recorded preview revision '${SOURCE_REVISION}' and is Ready."
                else
                  echo "::notice::Flux kustomization '${kustomization_name}' recorded preview revision '${SOURCE_REVISION}' but is not Ready. Later AGC validation remains authoritative. Ready='${ready_status:-<empty>}' message='${ready_message:-<empty>}'"
                fi
                return 0
              fi

              if [ "$SECONDS" -ge "$deadline_seconds" ]; then
                break
              fi

              sleep_seconds=$((deadline_seconds - SECONDS))
              if [ "$sleep_seconds" -gt "$poll_interval_seconds" ]; then
                sleep_seconds="$poll_interval_seconds"
              fi

              sleep "$sleep_seconds"
            done

            if [ "$strict_mode" = "strict" ]; then
              echo "::error::Flux kustomization '${kustomization_name}' did not record preview revision '${SOURCE_REVISION}' within observation window '${observation_timeout}' (${observation_timeout_source}). Last attempted='${last_attempted_revision:-<empty>}' last applied='${last_applied_revision:-<empty>}' Ready='${ready_status:-<empty>}' message='${ready_message:-<empty>}'"
              exit 1
            fi

            echo "::warning::Flux kustomization '${kustomization_name}' did not record preview revision '${SOURCE_REVISION}' within observation window '${observation_timeout}' (${observation_timeout_source}). Last attempted='${last_attempted_revision:-<empty>}' last applied='${last_applied_revision:-<empty>}' Ready='${ready_status:-<empty>}' message='${ready_message:-<empty>}'"
          }

          REQUESTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)-${GITHUB_RUN_ID:-manual}"
          request_preview_reconcile holiday-peak-gitops-holiday-peak-crud "$REQUESTED_AT" strict
          request_preview_reconcile holiday-peak-gitops-holiday-peak-agents "$REQUESTED_AT" advisory || true

          wait_for_preview_revision holiday-peak-gitops-holiday-peak-crud strict
          wait_for_preview_revision holiday-peak-gitops-holiday-peak-agents advisory

          echo "Preview Flux reconciliation completed for AGC readiness validation."

      - name: Finalize AGC readiness target
        id: finalize
        shell: bash
        run: |
          set -euo pipefail
          AGC_GATEWAY_CLASS="${{ steps.resolve.outputs.agc_gateway_class }}"
          AGC_FRONTEND_HOST_VALUE="${{ needs.provision.outputs.AGC_FRONTEND_HOSTNAME }}"
          AGC_FRONTEND_SCHEME_VALUE="http"
          AKS_RESOURCE_GROUP="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          AKS_CLUSTER_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-aks"

          resolve_frontend_from_alb_resource_group() {
            local alb_resource_group="$1"
            local alb_name=""
            local frontend_fqdn=""

            alb_name=$(az network alb list -g "$alb_resource_group" --query "[0].name" -o tsv 2>/dev/null || true)
            if [ -z "${alb_name:-}" ]; then
              return 0
            fi

            frontend_fqdn=$(az network alb frontend list -g "$alb_resource_group" --alb-name "$alb_name" --query "[0].fqdn" -o tsv 2>/dev/null || true)
            if [ -n "${frontend_fqdn:-}" ]; then
              AGC_FRONTEND_HOST_VALUE="$frontend_fqdn"
              echo "::notice::Resolved AGC frontend hostname from ALB '$alb_name' in resource group '$alb_resource_group'."
            fi
          }

          if [ -z "$AGC_FRONTEND_HOST_VALUE" ]; then
            resolve_frontend_from_alb_resource_group "$AKS_RESOURCE_GROUP"
          fi

          if [ -z "$AGC_FRONTEND_HOST_VALUE" ]; then
            AKS_NODE_RESOURCE_GROUP=$(az aks show -g "$AKS_RESOURCE_GROUP" -n "$AKS_CLUSTER_NAME" --query nodeResourceGroup -o tsv 2>/dev/null || true)
            if [ -n "${AKS_NODE_RESOURCE_GROUP:-}" ] && [ "$AKS_NODE_RESOURCE_GROUP" != "$AKS_RESOURCE_GROUP" ]; then
              resolve_frontend_from_alb_resource_group "$AKS_NODE_RESOURCE_GROUP"
            fi
          fi

          if [ -z "$AGC_FRONTEND_HOST_VALUE" ]; then
            SHARED_GATEWAY_HOST=$(kubectl get gateway holiday-peak-agc -n holiday-peak-crud -o json 2>/dev/null \
              | jq -r '.status.addresses[]? | select(.value != null and .value != "") | .value' \
              | head -n 1 || true)
            if [ -n "$SHARED_GATEWAY_HOST" ]; then
              AGC_FRONTEND_HOST_VALUE="$SHARED_GATEWAY_HOST"
              echo "::notice::Resolved AGC frontend hostname from the live shared Gateway status address."
            fi
          fi

          if [ -z "$AGC_FRONTEND_HOST_VALUE" ]; then
            GATEWAY_CLASS_HOST=$(kubectl get gateway -A -o json 2>/dev/null \
              | jq -r --arg gateway_class "$AGC_GATEWAY_CLASS" '[.items[] | select(.spec.gatewayClassName == $gateway_class) | .status.addresses[]? | select(.value != null and .value != "") | .value] | unique | .[0] // empty' || true)
            if [ -n "$GATEWAY_CLASS_HOST" ]; then
              AGC_FRONTEND_HOST_VALUE="$GATEWAY_CLASS_HOST"
              echo "::notice::Resolved AGC frontend hostname from a live Gateway status address with GatewayClass '$AGC_GATEWAY_CLASS'."
            fi
          fi

          if [ -z "$AGC_FRONTEND_HOST_VALUE" ]; then
            echo "AGC_FRONTEND_HOSTNAME is empty and no approved AGC frontend could be resolved from Azure frontends or Gateway status addresses. APIM sync cannot proceed without an approved AGC hostname." >&2
            exit 1
          fi

          case "$AGC_FRONTEND_SCHEME_VALUE" in
            http|https)
              ;;
            *)
              echo "Unsupported AGC frontend scheme '$AGC_FRONTEND_SCHEME_VALUE'." >&2
              exit 1
              ;;
          esac

          echo "approved_backend_hostnames=$AGC_FRONTEND_HOST_VALUE" >> "$GITHUB_OUTPUT"
          echo "approved_backend_scheme=$AGC_FRONTEND_SCHEME_VALUE" >> "$GITHUB_OUTPUT"
          echo "agc_gateway_class=$AGC_GATEWAY_CLASS" >> "$GITHUB_OUTPUT"
          echo "agc_frontend_url=${AGC_FRONTEND_SCHEME_VALUE}://${AGC_FRONTEND_HOST_VALUE}" >> "$GITHUB_OUTPUT"

      - name: Validate AGC gateway class and direct CRUD health
        shell: bash
        run: |
          set -euo pipefail
          AGC_GATEWAY_CLASS="${{ steps.finalize.outputs.agc_gateway_class }}"
          AGC_FRONTEND_URL="${{ steps.finalize.outputs.agc_frontend_url }}"
          AKS_RESOURCE_GROUP="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          AKS_CLUSTER_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-aks"

          kubectl get gatewayclass "$AGC_GATEWAY_CLASS" >/dev/null 2>&1 || {
            echo "GatewayClass '$AGC_GATEWAY_CLASS' is not available in the cluster." >&2
            exit 1
          }

          SHARED_GATEWAY_NAME="holiday-peak-agc"
          SHARED_GATEWAY_NAMESPACE="holiday-peak-crud"

          # Architecturally approved contract (ADR-021):
          #   - Shared AGC uses an in-cluster ApplicationLoadBalancer CR that the
          #     ALB controller manages — creating and maintaining the ARM
          #     trafficController resource automatically.
          #   - The shared Helm chart renders the Gateway (+ HTTPRoutes) and binds
          #     the Gateway to the ALB CR via `alb.networking.azure.io/alb-name` +
          #     `alb.networking.azure.io/alb-namespace` annotations.
          #   - The BYO `alb.networking.azure.io/alb-id` annotation mode is NOT
          #     used and will generate a warning if observed.

          AKS_NODE_RESOURCE_GROUP=$(az aks show -g "$AKS_RESOURCE_GROUP" -n "$AKS_CLUSTER_NAME" --query nodeResourceGroup -o tsv)
          if [ -z "$AKS_NODE_RESOURCE_GROUP" ]; then
            echo "::error::AKS node resource group could not be resolved for cluster '$AKS_CLUSTER_NAME'." >&2
            exit 1
          fi

          # Validate the in-cluster ApplicationLoadBalancer CR
          SHARED_ALB_NAME="holiday-peak-agc"
          SHARED_ALB_NAMESPACE="holiday-peak-crud"

          ALB_CR_JSON=$(kubectl get applicationloadbalancer "$SHARED_ALB_NAME" -n "$SHARED_ALB_NAMESPACE" -o json 2>/dev/null || true)
          if [ -z "$ALB_CR_JSON" ]; then
            echo "::error::ApplicationLoadBalancer CR '$SHARED_ALB_NAME' in namespace '$SHARED_ALB_NAMESPACE' is not fully deployed before Gateway validation." >&2
            exit 1
          fi

          ALB_ACCEPTED_STATUS=$(printf '%s' "$ALB_CR_JSON" | jq -r '[.status.conditions[]? | select(.type == "Accepted") | .status] | first // empty')
          ALB_DEPLOYMENT_STATUS=$(printf '%s' "$ALB_CR_JSON" | jq -r '[.status.conditions[]? | select(.type == "Deployment") | .status] | first // empty')
          ALB_EVIDENCE=$(printf "Accepted='%s' Deployment='%s'" "${ALB_ACCEPTED_STATUS:-<empty>}" "${ALB_DEPLOYMENT_STATUS:-<empty>}")
          echo "Live shared ApplicationLoadBalancer evidence: ${ALB_EVIDENCE}"

          mapfile -t TRAFFIC_CONTROLLER_IDS < <(az resource list -g "$AKS_NODE_RESOURCE_GROUP" --resource-type Microsoft.ServiceNetworking/trafficControllers --query '[].id' -o tsv)
          if [ "${#TRAFFIC_CONTROLLER_IDS[@]}" -eq 0 ]; then
            echo "::error::No Azure Application Gateway for Containers trafficControllers were found in node resource group '$AKS_NODE_RESOURCE_GROUP'. Per ADR-021 the shared ALB must be ARM-provisioned by Bicep." >&2
            exit 1
          fi

          declare -A TRAFFIC_CONTROLLER_HEALTH
          HEALTHY_TRAFFIC_CONTROLLER_COUNT=0
          TRAFFIC_CONTROLLER_EVIDENCE=""
          for controller_id in "${TRAFFIC_CONTROLLER_IDS[@]}"; do
            [ -z "$controller_id" ] && continue
            controller_json=$(az resource show --ids "$controller_id" -o json)
            controller_name=$(printf '%s' "$controller_json" | jq -r '.name // empty')
            controller_state=$(printf '%s' "$controller_json" | jq -r '.properties.provisioningState // empty')
            controller_frontends=$(printf '%s' "$controller_json" | jq -r '(.properties.frontends // []) | length')
            controller_associations=$(printf '%s' "$controller_json" | jq -r '(.properties.associations // []) | length')

            if [ -n "$TRAFFIC_CONTROLLER_EVIDENCE" ]; then
              TRAFFIC_CONTROLLER_EVIDENCE="${TRAFFIC_CONTROLLER_EVIDENCE}; "
            fi

            TRAFFIC_CONTROLLER_EVIDENCE="${TRAFFIC_CONTROLLER_EVIDENCE}${controller_name:-<unknown>}: id='${controller_id}' provisioningState=${controller_state:-<empty>} frontends=${controller_frontends:-0} associations=${controller_associations:-0}"

            if [ "$controller_state" = "Succeeded" ] && [ "$controller_frontends" -gt 0 ] && [ "$controller_associations" -gt 0 ]; then
              TRAFFIC_CONTROLLER_HEALTH["$controller_id"]="healthy"
              HEALTHY_TRAFFIC_CONTROLLER_COUNT=$((HEALTHY_TRAFFIC_CONTROLLER_COUNT + 1))
            else
              TRAFFIC_CONTROLLER_HEALTH["$controller_id"]="unhealthy"
            fi
          done

          echo "Traffic controller evidence: ${TRAFFIC_CONTROLLER_EVIDENCE}"

          if [ "$HEALTHY_TRAFFIC_CONTROLLER_COUNT" -eq 0 ]; then
            echo "::error::No healthy Azure Application Gateway for Containers trafficController with populated frontends and associations was found before Gateway validation. ${TRAFFIC_CONTROLLER_EVIDENCE}" >&2
            exit 1
          fi

          if ! SHARED_GATEWAY_JSON=$(kubectl get gateway "$SHARED_GATEWAY_NAME" -n "$SHARED_GATEWAY_NAMESPACE" -o json 2>/dev/null); then
            echo "::error::Live shared Gateway '$SHARED_GATEWAY_NAME' in namespace '$SHARED_GATEWAY_NAMESPACE' could not be retrieved for AGC readiness validation." >&2
            exit 1
          fi

          LIVE_GATEWAY_CLASS=$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '.spec.gatewayClassName // empty')
          AGC_SHARED_ALB_ID=$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '.metadata.annotations["alb.networking.azure.io/alb-id"] // empty')
          AGC_SHARED_ALB_NAME=$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '.metadata.annotations["alb.networking.azure.io/alb-name"] // empty')
          AGC_SHARED_ALB_NAMESPACE=$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '.metadata.annotations["alb.networking.azure.io/alb-namespace"] // empty')
          GATEWAY_ACCEPTED_STATUS=$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '[.status.conditions[]? | select(.type == "Accepted") | .status] | first // empty')
          GATEWAY_ACCEPTED_MESSAGE=$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '[.status.conditions[]? | select(.type == "Accepted") | .message] | first // empty')
          GATEWAY_PROGRAMMED_STATUS=$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '[.status.conditions[]? | select(.type == "Programmed") | .status] | first // empty')
          GATEWAY_PROGRAMMED_MESSAGE=$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '[.status.conditions[]? | select(.type == "Programmed") | .message] | first // empty')
          GATEWAY_STATUS_ADDRESS=$(printf '%s' "$SHARED_GATEWAY_JSON" | jq -r '[.status.addresses[]? | select(.value != null and .value != "") | .value] | first // empty')
          GATEWAY_CONTRACT_EVIDENCE=$(printf "GatewayClass='%s' alb.networking.azure.io/alb-name='%s' alb.networking.azure.io/alb-namespace='%s' Accepted='%s' acceptedMessage='%s' Programmed='%s' programmedMessage='%s' address='%s'" \
            "${LIVE_GATEWAY_CLASS:-<empty>}" \
            "${AGC_SHARED_ALB_NAME:-<empty>}" \
            "${AGC_SHARED_ALB_NAMESPACE:-<empty>}" \
            "${GATEWAY_ACCEPTED_STATUS:-<empty>}" \
            "${GATEWAY_ACCEPTED_MESSAGE:-<empty>}" \
            "${GATEWAY_PROGRAMMED_STATUS:-<empty>}" \
            "${GATEWAY_PROGRAMMED_MESSAGE:-<empty>}" \
            "${GATEWAY_STATUS_ADDRESS:-<empty>}")

          echo "Live shared Gateway evidence: ${GATEWAY_CONTRACT_EVIDENCE}"

          # ADR-021 (updated): require in-cluster managed ALB binding via alb-name/alb-namespace.
          # The BYO alb-id annotation mode is no longer used — ALB controller manages the TC lifecycle.
          if [ -n "$AGC_SHARED_ALB_ID" ]; then
            echo "::warning::Live shared Gateway '$SHARED_GATEWAY_NAME' carries a BYO alb-id annotation ('${AGC_SHARED_ALB_ID}'). The approved binding mode is now managed (alb-name/alb-namespace). ${GATEWAY_CONTRACT_EVIDENCE}"
          fi

          if [ -z "$AGC_SHARED_ALB_NAME" ] || [ -z "$AGC_SHARED_ALB_NAMESPACE" ]; then
            echo "::error::Live shared Gateway '$SHARED_GATEWAY_NAME' in namespace '$SHARED_GATEWAY_NAMESPACE' is missing required AGC binding annotations before direct health validation. The Gateway must carry 'alb.networking.azure.io/alb-name' and 'alb.networking.azure.io/alb-namespace'. ${GATEWAY_CONTRACT_EVIDENCE}" >&2
            exit 1
          fi

          if [ "$LIVE_GATEWAY_CLASS" != "$AGC_GATEWAY_CLASS" ]; then
            echo "::error::Live shared Gateway '$SHARED_GATEWAY_NAME' uses GatewayClass '${LIVE_GATEWAY_CLASS:-<empty>}' but the approved class is '$AGC_GATEWAY_CLASS'. ${GATEWAY_CONTRACT_EVIDENCE}" >&2
            exit 1
          fi

          # The ALB controller resolves the TC from the CR; for health checks we match against discovered TCs
          MATCHED_CONTROLLER_ID=""
          for controller_id in "${TRAFFIC_CONTROLLER_IDS[@]}"; do
            [ -z "$controller_id" ] && continue
            if [ "${TRAFFIC_CONTROLLER_HEALTH[$controller_id]:-}" = "healthy" ]; then
              MATCHED_CONTROLLER_ID="$controller_id"
              break
            fi
          done

          if [ "$GATEWAY_ACCEPTED_STATUS" != "True" ]; then
            echo "::error::Live shared Gateway '$SHARED_GATEWAY_NAME' in namespace '$SHARED_GATEWAY_NAMESPACE' is not Accepted before direct health validation. ${GATEWAY_CONTRACT_EVIDENCE}" >&2
            exit 1
          fi

          if [ "$GATEWAY_PROGRAMMED_STATUS" != "True" ]; then
            echo "::error::Live shared Gateway '$SHARED_GATEWAY_NAME' in namespace '$SHARED_GATEWAY_NAMESPACE' is not Programmed before direct health validation. ${GATEWAY_CONTRACT_EVIDENCE}" >&2
            exit 1
          fi

          if [ -z "$GATEWAY_STATUS_ADDRESS" ]; then
            echo "::error::Live shared Gateway '$SHARED_GATEWAY_NAME' in namespace '$SHARED_GATEWAY_NAMESPACE' has no status address before direct health validation. ${GATEWAY_CONTRACT_EVIDENCE}" >&2
            exit 1
          fi

          route_namespace_for_service() {
            local service_name="$1"

            if [ "$service_name" = "crud-service" ]; then
              printf '%s' 'holiday-peak-crud'
              return 0
            fi

            printf '%s' 'holiday-peak-agents'
          }

          route_name_for_service() {
            local service_name="$1"
            printf '%s-%s-agc' "$service_name" "$service_name"
          }

          direct_health_path_for_service() {
            local service_name="$1"

            if [ "$service_name" = "crud-service" ]; then
              printf '%s' '/health'
              return 0
            fi

            printf '/%s/health' "$service_name"
          }

          validate_route_parent_status() {
            local service_name="$1"
            local route_namespace="$2"
            local route_name="$3"
            local route_json=""
            local route_parent_count=0
            local route_accepted_status=""
            local route_accepted_message=""
            local route_resolved_refs_status=""
            local route_resolved_refs_message=""
            local route_evidence=""

            if ! route_json=$(kubectl get httproute "$route_name" -n "$route_namespace" -o json 2>/dev/null); then
              echo "::error::Expected HTTPRoute '$route_name' in namespace '$route_namespace' was not found before direct AGC health validation for service '$service_name'." >&2
              exit 1
            fi

            route_parent_count=$(printf '%s' "$route_json" | jq -r --arg gateway_name "$SHARED_GATEWAY_NAME" --arg gateway_namespace "$SHARED_GATEWAY_NAMESPACE" '[.status.parents[]? | select((.parentRef.name // "") == $gateway_name and ((.parentRef.namespace // $gateway_namespace) == $gateway_namespace))] | length')
            route_accepted_status=$(printf '%s' "$route_json" | jq -r --arg gateway_name "$SHARED_GATEWAY_NAME" --arg gateway_namespace "$SHARED_GATEWAY_NAMESPACE" '[.status.parents[]? | select((.parentRef.name // "") == $gateway_name and ((.parentRef.namespace // $gateway_namespace) == $gateway_namespace)) | .conditions[]? | select(.type == "Accepted") | .status] | first // empty')
            route_accepted_message=$(printf '%s' "$route_json" | jq -r --arg gateway_name "$SHARED_GATEWAY_NAME" --arg gateway_namespace "$SHARED_GATEWAY_NAMESPACE" '[.status.parents[]? | select((.parentRef.name // "") == $gateway_name and ((.parentRef.namespace // $gateway_namespace) == $gateway_namespace)) | .conditions[]? | select(.type == "Accepted") | .message] | first // empty')
            route_resolved_refs_status=$(printf '%s' "$route_json" | jq -r --arg gateway_name "$SHARED_GATEWAY_NAME" --arg gateway_namespace "$SHARED_GATEWAY_NAMESPACE" '[.status.parents[]? | select((.parentRef.name // "") == $gateway_name and ((.parentRef.namespace // $gateway_namespace) == $gateway_namespace)) | .conditions[]? | select(.type == "ResolvedRefs") | .status] | first // empty')
            route_resolved_refs_message=$(printf '%s' "$route_json" | jq -r --arg gateway_name "$SHARED_GATEWAY_NAME" --arg gateway_namespace "$SHARED_GATEWAY_NAMESPACE" '[.status.parents[]? | select((.parentRef.name // "") == $gateway_name and ((.parentRef.namespace // $gateway_namespace) == $gateway_namespace)) | .conditions[]? | select(.type == "ResolvedRefs") | .message] | first // empty')
            route_evidence=$(printf "HTTPRoute='%s/%s' parentCount='%s' Accepted='%s' acceptedMessage='%s' ResolvedRefs='%s' resolvedRefsMessage='%s'" \
              "$route_namespace" \
              "$route_name" \
              "${route_parent_count:-0}" \
              "${route_accepted_status:-<empty>}" \
              "${route_accepted_message:-<empty>}" \
              "${route_resolved_refs_status:-<empty>}" \
              "${route_resolved_refs_message:-<empty>}")

            echo "Route evidence: ${route_evidence}"

            if [ "$route_parent_count" = "0" ]; then
              echo "::error::HTTPRoute '$route_name' in namespace '$route_namespace' has no parent status attached to shared Gateway '$SHARED_GATEWAY_NAME'. ${route_evidence}" >&2
              exit 1
            fi

            if [ "$route_accepted_status" != "True" ]; then
              echo "::error::HTTPRoute '$route_name' in namespace '$route_namespace' is not Accepted for shared Gateway '$SHARED_GATEWAY_NAME'. ${route_evidence}" >&2
              exit 1
            fi

            if [ -n "$route_resolved_refs_status" ] && [ "$route_resolved_refs_status" != "True" ]; then
              echo "::error::HTTPRoute '$route_name' in namespace '$route_namespace' has unresolved backend references for shared Gateway '$SHARED_GATEWAY_NAME'. ${route_evidence}" >&2
              exit 1
            fi
          }

          probe_direct_agc_health() {
            local path="$1"
            local label="$2"
            local response_file="/tmp/${label}.json"
            local status_code=""

            for attempt in $(seq 1 20); do
              if ! status_code=$(curl -sS -o "$response_file" -w "%{http_code}" "${AGC_FRONTEND_URL%/}${path}"); then
                status_code="000"
              fi

              if [ "$status_code" = "200" ]; then
                echo "Validated ${label} via ${AGC_FRONTEND_URL%/}${path}"
                return 0
              fi

              echo "Attempt $attempt/20 failed for ${label} with HTTP $status_code"
              sleep 10
            done

            echo "AGC readiness validation failed for ${AGC_FRONTEND_URL%/}${path}" >&2
            cat "$response_file" 2>/dev/null || true
            exit 1
          }

          echo "Live shared Gateway contract passed. Validating route attachment before direct AGC health checks."

          validate_route_parent_status "crud-service" "$(route_namespace_for_service crud-service)" "$(route_name_for_service crud-service)"

          if [ -n "${CHANGED_AGENT_SERVICES:-}" ]; then
            IFS=',' read -r -a agent_services <<< "${CHANGED_AGENT_SERVICES}"
            for svc in "${agent_services[@]}"; do
              [ -z "$svc" ] && continue
              validate_route_parent_status "$svc" "$(route_namespace_for_service "$svc")" "$(route_name_for_service "$svc")"
            done
          else
            echo "No changed agent services detected; skipping agent HTTPRoute attachment validation."
          fi

          probe_direct_agc_health "$(direct_health_path_for_service crud-service)" "direct-agc-crud-health"

          if [ -n "${CHANGED_AGENT_SERVICES:-}" ]; then
            for svc in "${agent_services[@]}"; do
              [ -z "$svc" ] && continue
              probe_direct_agc_health "$(direct_health_path_for_service "$svc")" "direct-agc-agent-health-${svc}"
            done
          else
            echo "No changed agent services detected; skipping direct AGC agent health validation."
          fi

  sync-apim:
    runs-on: ubuntu-latest
    if: ${{ always() && !cancelled() && !inputs.uiOnly && (inputs.forceApimSync || needs.detect-changes.outputs.changed_agent_services_csv != '' || needs.detect-changes.outputs.crud_changed == 'true' || needs.detect-changes.outputs.ui_changed == 'true') && (needs.deploy-crud.result == 'success' || needs.deploy-crud.result == 'skipped') && (needs.deploy-agents.result == 'success' || needs.deploy-agents.result == 'skipped') && needs.validate-agc-readiness.result == 'success' }}
    needs:
      - deploy-crud
      - deploy-agents
      - detect-changes
      - validate-agc-readiness
    environment: ${{ inputs.githubEnvironment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      CHANGED_SERVICES: ${{ needs.detect-changes.outputs.changed_aks_services_csv != '' && needs.detect-changes.outputs.changed_aks_services_csv || (needs.detect-changes.outputs.ui_changed == 'true' && 'crud-service' || '') }}
      APIM_APPROVED_BACKEND_HOSTNAMES: ${{ needs.validate-agc-readiness.outputs.approved_backend_hostnames }}
      APIM_APPROVED_BACKEND_SCHEME: ${{ needs.validate-agc-readiness.outputs.approved_backend_scheme }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ env.DEPLOY_SOURCE_CHECKOUT_REF }}

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Install kubelogin
        run: |
          for attempt in 1 2 3 4 5; do
            if az aks install-cli --kubelogin-version latest; then
              exit 0
            fi
            echo "kubelogin install failed on attempt ${attempt}; retrying..."
            sleep 15
          done
          echo "kubelogin install failed after retries"
          exit 1

      - name: Get AKS credentials
        run: |
          az aks get-credentials \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --overwrite-existing
          kubelogin convert-kubeconfig -l azurecli

      - name: Verify AKS API connectivity
        shell: bash
        run: |
          set -euo pipefail
          AKS_FQDN=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | sed 's|https://||;s|:.*||')
          echo "Probing AKS API server DNS: $AKS_FQDN"
          for attempt in $(seq 1 10); do
            if kubectl cluster-info >/dev/null 2>&1; then
              echo "AKS API reachable (attempt $attempt)."
              exit 0
            fi
            BACKOFF=$((5 * attempt))
            echo "Attempt $attempt/10 failed (DNS or TCP). Retrying in ${BACKOFF}s..."
            sleep "$BACKOFF"
          done
          echo "AKS API DNS resolution failed after 10 attempts for $AKS_FQDN." >&2
          nslookup "$AKS_FQDN" || true
          exit 1

      - name: Sync APIM agent APIs
        run: |
          bash .infra/azd/hooks/sync-apim-agents.sh
        env:
          AZURE_RESOURCE_GROUP: ${{ inputs.projectName }}-${{ inputs.environment }}-rg

  sync-apic:
    runs-on: ubuntu-latest
    if: ${{ always() && !cancelled() && !inputs.uiOnly && (inputs.forceApimSync || needs.detect-changes.outputs.changed_agent_services_csv != '' || needs.detect-changes.outputs.crud_changed == 'true') && (needs.sync-apim.result == 'success' || needs.sync-apim.result == 'skipped') }}
    needs:
      - detect-changes
      - sync-apim
    environment: ${{ inputs.githubEnvironment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ env.DEPLOY_SOURCE_CHECKOUT_REF }}

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Sync API Center
        run: |
          bash .infra/azd/hooks/sync-apic-apis.sh
        env:
          AZURE_RESOURCE_GROUP: ${{ inputs.projectName }}-${{ inputs.environment }}-rg

  smoke-apim:
    runs-on: ubuntu-latest
    if: ${{ always() && !cancelled() && !inputs.uiOnly && (inputs.forceApimSync || needs.detect-changes.outputs.changed_agent_services_csv != '' || needs.detect-changes.outputs.crud_changed == 'true' || needs.detect-changes.outputs.ui_changed == 'true') && needs.validate-agc-readiness.result == 'success' && (needs.sync-apim.result == 'success' || needs.sync-apim.result == 'skipped') }}
    needs:
      - detect-changes
      - sync-apim
      - validate-agc-readiness
    environment: ${{ inputs.githubEnvironment }}
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      CHANGED_AGENT_SERVICES: ${{ needs.detect-changes.outputs.changed_agent_services_csv }}
      APPROVED_BACKEND_HOSTNAMES: ${{ needs.validate-agc-readiness.outputs.approved_backend_hostnames }}
    steps:
      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Validate CRUD APIM backend stability
        shell: bash
        run: |
          set -euo pipefail

          RG_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-rg"
          APIM_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-apim"
          AKS_NAME="${{ inputs.projectName }}-${{ inputs.environment }}-aks"
          K8S_NAMESPACE="holiday-peak-crud"

          fetch_crud_endpoint_ips() {
            local logs
            logs=$(az aks command invoke \
              --resource-group "$RG_NAME" \
              --name "$AKS_NAME" \
              --command "kubectl get endpoints crud-service -n $K8S_NAMESPACE -o jsonpath='{.subsets[*].addresses[*].ip}'" \
              --query logs -o tsv)

            printf '%s' "$logs" | awk 'NF{line=$0} END{print line}'
          }

          resolve_crud_api_id() {
            if az apim api show --resource-group "$RG_NAME" --service-name "$APIM_NAME" --api-id "crud-service" --only-show-errors >/dev/null 2>&1; then
              printf 'crud-service'
              return 0
            fi

            if az apim api show --resource-group "$RG_NAME" --service-name "$APIM_NAME" --api-id "crud" --only-show-errors >/dev/null 2>&1; then
              printf 'crud'
              return 0
            fi

            echo "Neither APIM API id 'crud-service' nor 'crud' exists in '$APIM_NAME'." >&2
            return 1
          }

          extract_host() {
            local url="$1"
            python3 -c "from urllib.parse import urlparse; import sys; print(urlparse(sys.argv[1]).hostname or '')" "$url"
          }

          CRUD_ENDPOINT_IPS_RAW="$(fetch_crud_endpoint_ips)"
          CRUD_ENDPOINT_IPS="$(printf '%s\n' "$CRUD_ENDPOINT_IPS_RAW" | tr ' ' '\n' | sed '/^[[:space:]]*$/d' | sort -u)"

          if [ -z "$CRUD_ENDPOINT_IPS" ]; then
            echo "Failed to resolve current endpoint IPs for crud-service in namespace '$K8S_NAMESPACE'." >&2
            exit 1
          fi

          API_ID="$(resolve_crud_api_id)"
          SERVICE_URL="$(az apim api show --resource-group "$RG_NAME" --service-name "$APIM_NAME" --api-id "$API_ID" --query serviceUrl -o tsv --only-show-errors)"
          SERVICE_URL="$(printf '%s' "$SERVICE_URL" | xargs)"

          if [ -z "$SERVICE_URL" ]; then
            echo "APIM CRUD API '$API_ID' has an empty serviceUrl." >&2
            exit 1
          fi

          SERVICE_HOST="$(extract_host "$SERVICE_URL")"
          if [ -z "$SERVICE_HOST" ]; then
            echo "Unable to parse host from APIM CRUD serviceUrl '$SERVICE_URL'." >&2
            exit 1
          fi

          APPROVED_HOSTS="$(printf '%s' "$APPROVED_BACKEND_HOSTNAMES" | tr ',' '\n' | sed '/^[[:space:]]*$/d')"
          if [ -z "$APPROVED_HOSTS" ]; then
            echo "No approved AGC backend hostnames were provided to smoke validation." >&2
            exit 1
          fi

          if ! printf '%s\n' "$APPROVED_HOSTS" | grep -Fxq "$SERVICE_HOST"; then
            echo "APIM CRUD backend drift detected: API '$API_ID' serviceUrl '$SERVICE_URL' does not target an approved AGC hostname." >&2
            echo "Approved AGC hostnames: $(printf '%s\n' "$APPROVED_HOSTS" | paste -sd ',' -)" >&2
            exit 1
          fi

              if python3 -c "import ipaddress, sys; ipaddress.ip_address(sys.argv[1])" "$SERVICE_HOST" >/dev/null 2>&1; then
            echo "APIM CRUD backend drift detected: API '$API_ID' serviceUrl '$SERVICE_URL' resolves to IP literal '$SERVICE_HOST'." >&2
            exit 1
          fi

          if printf '%s\n' "$CRUD_ENDPOINT_IPS" | grep -Fxq "$SERVICE_HOST"; then
            echo "Unstable APIM backend detected: API '$API_ID' serviceUrl '$SERVICE_URL' points to current crud-service endpoint IP '$SERVICE_HOST'." >&2
            echo "Current crud-service endpoint IPs: $(printf '%s\n' "$CRUD_ENDPOINT_IPS" | paste -sd ',' -)" >&2
            exit 1
          fi

          echo "Validated APIM CRUD backend stability: API '$API_ID' serviceUrl host '$SERVICE_HOST' is not a current crud-service endpoint IP."

      - name: Smoke test direct AGC CRUD health
        shell: bash
        run: |
          set -euo pipefail

          AGC_BASE="${{ needs.validate-agc-readiness.outputs.agc_frontend_url }}"

          for attempt in $(seq 1 20); do
            if ! STATUS_CODE=$(curl -sS -o /tmp/agc-smoke-response.json -w "%{http_code}" "${AGC_BASE%/}/health"); then
              STATUS_CODE="000"
            fi

            if [ "$STATUS_CODE" = "200" ]; then
              echo "[OK] direct-agc-crud-health: ${AGC_BASE%/}/health"
              exit 0
            fi

            echo "Attempt $attempt/20 failed for direct AGC CRUD health with HTTP $STATUS_CODE"
            sleep 10
          done

          echo "[FAIL] direct-agc-crud-health: ${AGC_BASE%/}/health" >&2
          cat /tmp/agc-smoke-response.json 2>/dev/null || true
          exit 1

      - name: Resolve APIM gateway URL
        id: apim
        shell: bash
        run: |
          set -euo pipefail

          APIM_URL=$(az apim show \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-apim" \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --query "gatewayUrl" -o tsv)

          APIM_URL="$(echo "$APIM_URL" | xargs)"
          APIM_URL="${APIM_URL%/}"

          if [ -z "$APIM_URL" ]; then
            echo "Failed to resolve APIM gateway URL for smoke tests." >&2
            exit 1
          fi

          echo "gateway_url=$APIM_URL" >> "$GITHUB_OUTPUT"

      - name: Smoke test CRUD and changed agent health through APIM
        shell: bash
        run: |
          set -euo pipefail

          API_BASE="${{ steps.apim.outputs.gateway_url }}"

          smoke_health() {
            local url="$1"
            local label="$2"

            for attempt in $(seq 1 20); do
              if ! STATUS_CODE=$(curl -sS -o /tmp/smoke-response.json -w "%{http_code}" "$url"); then
                STATUS_CODE="000"
              fi
              if [ "$STATUS_CODE" = "200" ]; then
                echo "[OK] $label: $url"
                return 0
              fi
              echo "Attempt $attempt/20 failed for $label with HTTP $STATUS_CODE"
              sleep 10
            done

            echo "[FAIL] $label: $url" >&2
            cat /tmp/smoke-response.json 2>/dev/null || true
            return 1
          }

          smoke_health "${API_BASE}/api/ready" "crud-ready"
          smoke_health "${API_BASE}/api/products?limit=1" "crud-products"
          smoke_health "${API_BASE}/api/categories" "crud-categories"

          smoke_cors_preflight() {
            local url="$1"
            local label="$2"
            local headers_file="/tmp/apim-cors-headers.txt"
            local normalized_headers_file="/tmp/apim-cors-headers-normalized.txt"
            local body_file="/tmp/apim-cors-body.txt"
            local status_code=""

            for attempt in $(seq 1 20); do
              : > "$headers_file"
              : > "$normalized_headers_file"
              status_code=$(curl -sS -D "$headers_file" -o "$body_file" -w "%{http_code}" \
                -X OPTIONS "$url" \
                -H "Origin: http://localhost:3000" \
                -H "Access-Control-Request-Method: GET") || status_code="000"
              tr -d '\r' < "$headers_file" > "$normalized_headers_file" 2>/dev/null || true

              if { [ "$status_code" = "200" ] || [ "$status_code" = "204" ]; } \
                && grep -Eiq '^Access-Control-Allow-Origin: http://localhost:3000$' "$normalized_headers_file" \
                && grep -Eiq '^Access-Control-Allow-Methods: .*GET' "$normalized_headers_file"; then
                echo "[OK] $label: $url"
                return 0
              fi

              echo "Attempt $attempt/20 failed for $label with HTTP $status_code or missing CORS headers"
              cat "$headers_file" 2>/dev/null || true
              sleep 10
            done

            echo "[FAIL] $label: $url did not return expected CORS preflight headers" >&2
            cat "$headers_file" 2>/dev/null || true
            cat "$body_file" 2>/dev/null || true
            return 1
          }

          smoke_cors_preflight "${API_BASE}/api/products?limit=1" "crud-cors-preflight"

          NEGATIVE_STATUS=$(curl -sS -o /tmp/apim-negative-response.json -w "%{http_code}" "${API_BASE}/api/does-not-exist") || NEGATIVE_STATUS="000"
          if [ "$NEGATIVE_STATUS" = "500" ] || [ "$NEGATIVE_STATUS" = "502" ] || [ "$NEGATIVE_STATUS" = "503" ] || [ "$NEGATIVE_STATUS" = "000" ]; then
            echo "[FAIL] crud-negative-path: ${API_BASE}/api/does-not-exist returned unexpected HTTP $NEGATIVE_STATUS" >&2
            cat /tmp/apim-negative-response.json 2>/dev/null || true
            exit 1
          fi

          echo "[OK] crud-cors-preflight: ${API_BASE}/api/products?limit=1"
          echo "[OK] crud-negative-path: ${API_BASE}/api/does-not-exist returned HTTP $NEGATIVE_STATUS"

          if [ -n "${CHANGED_AGENT_SERVICES}" ]; then
            IFS=',' read -r -a agent_services <<< "${CHANGED_AGENT_SERVICES}"
            for svc in "${agent_services[@]}"; do
              [ -z "$svc" ] && continue
              smoke_health "${API_BASE}/agents/${svc}/health" "agent-health-$svc"
            done
          else
            echo "No changed agent services detected; skipping agent health smoke checks."
          fi

  watchdog-apim-agc-swa-drift:
    runs-on: ubuntu-latest
    continue-on-error: true
    if: ${{ always() && !inputs.uiOnly && inputs.environment == 'dev' && (needs.provision.result == 'success' || needs.provision.result == 'skipped') }}
    needs:
      - provision
      - detect-changes
      - deploy-crud
      - deploy-agents
      - wait-flux-reconciliation
      - sync-apim
      - sync-apic
      - smoke-apim
      - deploy-ui
    environment: ${{ inputs.githubEnvironment }}
    permissions:
      id-token: write
      contents: read
      issues: write
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ env.DEPLOY_SOURCE_CHECKOUT_REF }}

      - name: Azure login (OIDC)
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Setup azd
        uses: Azure/setup-azd@v2

      - name: Run APIM/AGC/SWA drift watchdog
        id: watchdog
        shell: bash
        run: |
          set -euo pipefail
          chmod +x scripts/ops/watchdog-apim-agc-swa-drift.sh
          scripts/ops/watchdog-apim-agc-swa-drift.sh
        env:
          PROJECT_NAME: ${{ inputs.projectName }}
          ENVIRONMENT: ${{ inputs.environment }}
          RESOURCE_GROUP: ${{ inputs.projectName }}-${{ inputs.environment }}-rg
          APIM_NAME: ${{ inputs.projectName }}-${{ inputs.environment }}-apim
          AKS_NAME: ${{ inputs.projectName }}-${{ inputs.environment }}-aks
          SWA_NAME: ${{ inputs.projectName }}-ui-${{ inputs.environment }}
          APIM_GATEWAY_URL: ${{ needs.provision.outputs.APIM_GATEWAY_URL }}
          AGC_FRONTEND_HOSTNAME: ${{ needs.provision.outputs.AGC_FRONTEND_HOSTNAME }}
          AGC_SUPPORT_ENABLED: ${{ needs.provision.outputs.AGC_SUPPORT_ENABLED }}
          REPORT_DIR: ${{ runner.temp }}/watchdog-drift

      - name: Append watchdog summary
        if: ${{ always() && steps.watchdog.outputs.summary_file != '' }}
        shell: bash
        run: |
          set -euo pipefail
          cat "${{ steps.watchdog.outputs.summary_file }}" >> "$GITHUB_STEP_SUMMARY"

      - name: Upload watchdog artifact
        if: ${{ always() && steps.watchdog.outputs.results_file != '' }}
        uses: actions/upload-artifact@v4
        with:
          name: watchdog-apim-agc-swa-drift-${{ github.run_id }}-attempt-${{ github.run_attempt }}
          path: ${{ runner.temp }}/watchdog-drift
          if-no-files-found: warn
          retention-days: 14

      - name: Comment on issue #298 when drift is detected
        if: ${{ always() && steps.watchdog.outputs.drift_detected == 'true' }}
        uses: actions/github-script@v7
        env:
          DEPLOY_ENV: ${{ inputs.environment }}
        with:
          script: |
            const failedChecks = `${{ steps.watchdog.outputs.failed_checks }}`;
            const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const body = [
              '⚠️ Watchdog drift detected after deploy.',
              '',
              `- Environment: ${process.env.DEPLOY_ENV}`,
              `- Failed checks: ${failedChecks || 'n/a'}`,
              `- Run: ${runUrl}`,
              '',
              'Auto-generated by `watchdog-apim-agc-swa-drift` job.'
            ].join('\n');

            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: 298,
              body,
            });

  restore-flux-source-default-branch:
    runs-on: ubuntu-latest
    if: ${{ always() && !cancelled() && !inputs.uiOnly }}
    needs:
      - wait-flux-reconciliation
      - validate-agc-readiness
      - sync-apim
      - sync-apic
      - smoke-apim
      - deploy-ui
      - watchdog-apim-agc-swa-drift
    environment: ${{ inputs.githubEnvironment }}
    permissions:
      id-token: write
    env:
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
    steps:
      - name: Resolve Flux preview cleanup
        id: preview
        shell: bash
        run: |
          set -euo pipefail

          DEFAULT_BRANCH="${REPOSITORY_DEFAULT_BRANCH}"
          SOURCE_REF="${DEPLOY_SOURCE_REF}"

          if [ -z "$DEFAULT_BRANCH" ]; then
            echo "::error::Repository default branch is required for Flux cleanup."
            exit 1
          fi

          if [ "${{ inputs.environment }}" = "prod" ] || [[ "$SOURCE_REF" != refs/heads/* ]]; then
            echo "preview_branch=" >> "$GITHUB_OUTPUT"
            echo "No Flux preview cleanup required."
            exit 0
          fi

          CANDIDATE_BRANCH="${SOURCE_REF#refs/heads/}"
          if [ -z "$CANDIDATE_BRANCH" ] || [ "$CANDIDATE_BRANCH" = "$DEFAULT_BRANCH" ]; then
            echo "preview_branch=" >> "$GITHUB_OUTPUT"
            echo "No Flux preview cleanup required."
            exit 0
          fi

          echo "preview_branch=$CANDIDATE_BRANCH" >> "$GITHUB_OUTPUT"
          echo "default_branch=$DEFAULT_BRANCH" >> "$GITHUB_OUTPUT"

      - name: Azure login (OIDC)
        if: ${{ steps.preview.outputs.preview_branch != '' }}
        uses: azure/login@v2
        with:
          client-id: ${{ env.AZURE_CLIENT_ID }}
          tenant-id: ${{ env.AZURE_TENANT_ID }}
          subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }}

      - name: Install kubelogin
        if: ${{ steps.preview.outputs.preview_branch != '' }}
        run: |
          for attempt in 1 2 3 4 5; do
            if az aks install-cli --kubelogin-version latest; then
              exit 0
            fi
            echo "kubelogin install failed on attempt ${attempt}; retrying..."
            sleep 15
          done
          echo "kubelogin install failed after retries"
          exit 1

      - name: Get AKS credentials
        if: ${{ steps.preview.outputs.preview_branch != '' }}
        run: |
          az aks get-credentials \
            --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
            --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
            --overwrite-existing
          kubelogin convert-kubeconfig -l azurecli

      - name: Restore Flux source to default branch
        if: ${{ steps.preview.outputs.preview_branch != '' }}
        shell: bash
        run: |
          set -euo pipefail

          FLUX_SOURCE="${FLUX_SOURCE_NAME}"
          FLUX_NAMESPACE="${FLUX_SOURCE_NAMESPACE}"
          DEFAULT_BRANCH="${{ steps.preview.outputs.default_branch }}"
          PREVIEW_BRANCH="${{ steps.preview.outputs.preview_branch }}"

          echo "Restoring Flux source '${FLUX_SOURCE}' from preview branch '${PREVIEW_BRANCH}' to default branch '${DEFAULT_BRANCH}'."
          PATCH=$(printf '{"spec":{"ref":{"branch":"%s"}}}' "$DEFAULT_BRANCH")

          restore_flux_source() {
            kubectl patch gitrepository.source.toolkit.fluxcd.io "$FLUX_SOURCE" -n "$FLUX_NAMESPACE" --type=merge -p "$PATCH"

            ACTIVE_BRANCH=$(kubectl get gitrepository.source.toolkit.fluxcd.io "$FLUX_SOURCE" -n "$FLUX_NAMESPACE" -o jsonpath='{.spec.ref.branch}')
            if [ "$ACTIVE_BRANCH" != "$DEFAULT_BRANCH" ]; then
              echo "::error::Flux source '${FLUX_SOURCE}' expected branch '${DEFAULT_BRANCH}' but found '${ACTIVE_BRANCH}'."
              return 1
            fi
          }

          if ! restore_flux_source; then
            echo "::warning::Direct kubectl Flux restore failed; retrying through az aks command invoke."
            REMOTE_COMMAND=$(cat <<EOF
          set -euo pipefail
          kubectl patch gitrepository.source.toolkit.fluxcd.io "$FLUX_SOURCE" -n "$FLUX_NAMESPACE" --type=merge -p '$PATCH'
          ACTIVE_BRANCH=\$(kubectl get gitrepository.source.toolkit.fluxcd.io "$FLUX_SOURCE" -n "$FLUX_NAMESPACE" -o jsonpath='{.spec.ref.branch}')
          if [ "\$ACTIVE_BRANCH" != "$DEFAULT_BRANCH" ]; then
            echo "::error::Flux source '$FLUX_SOURCE' expected branch '$DEFAULT_BRANCH' but found '\$ACTIVE_BRANCH'."
            exit 1
          fi
          EOF
          )

            az aks command invoke \
              --resource-group "${{ inputs.projectName }}-${{ inputs.environment }}-rg" \
              --name "${{ inputs.projectName }}-${{ inputs.environment }}-aks" \
              --command "$REMOTE_COMMAND" \
              --query logs -o tsv
          fi

          if ! command -v flux &>/dev/null; then
            curl -s https://fluxcd.io/install.sh | bash 2>/dev/null || true
          fi

          flux reconcile source git "$FLUX_SOURCE" -n "$FLUX_NAMESPACE" --timeout=2m || true
          flux reconcile kustomization holiday-peak-gitops-holiday-peak-crud -n flux-system --timeout=5m || true
          flux reconcile kustomization holiday-peak-gitops-holiday-peak-agents -n flux-system --timeout=5m || true