ci-main.yml

name: ci main

# Full pipeline on push to main: format, build/test, publish artifacts + Docker,
# release snapshot, notify. PRs use ci-pr.yml (format + build/test only).
#
# Job graph:
#
#   check-format ──────────────────────────────────────────────────────────────────┐
#   build (linux, asan debug) ── publish (docker+smoke) ── release ────────────────┼── notify
#   microbenchmark (linux, macos) ────────────────────────────────────────────────-┘

on:
  push:
    branches: [main, release/**]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  # Cancel superseded runs on release/** branches (rapid-iteration churn).
  # Keep main's serialize-don't-cancel behavior so external-state steps
  # (Docker push, release create, deploy) aren't interrupted mid-flight.
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

permissions:
  contents: write
  checks: write
  packages: write

jobs:
  # ════════════════════════════════════════════════════════════════════════════
  # Verify: format + build/test matrix
  # ════════════════════════════════════════════════════════════════════════════

  check-format:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Install clang-format
        run: pip install clang-format==19.1.7

      - name: Check formatting
        run: bash scripts/format.sh --check

  build:
    needs: [check-format]
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: linux
            preset: default
            build_dir: build
            runner: ubuntu-22.04
          # arm64 is validated by the publish-side docker build (publish arm64
          # entry below); skipped here to avoid the ubuntu-runner ↔ bookworm-tarball
          # glog ABI mismatch. arm64 issues surface at the publish step.
          - name: macos
            preset: default
            build_dir: build
            runner: macos-15
          - name: asan debug
            preset: san
            build_dir: build-san
            runner: [self-hosted, linode]
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.runner }}
    steps:
      - name: Generate app token
        id: app-token
        uses: actions/create-github-app-token@v2
        with:
          app-id: ${{ secrets.OMOQ_APP_ID }}
          private-key: ${{ secrets.OMOQ_APP_PRIV_KEY }}

      - uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Install system dependencies
        run: |
          if [[ "$(uname)" == "Darwin" ]]; then
            deps/moxygen/standalone/install-system-deps.sh
            brew install coreutils
            echo "/opt/homebrew/opt/coreutils/libexec/gnubin" >> "$GITHUB_PATH"
          else
            sudo deps/moxygen/standalone/install-system-deps.sh
          fi

      - name: Setup dependencies
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
          MOQX_PLATFORM: ${{ matrix.platform || '' }}
        run: |
          # Three-mode setup resolution:
          # 1. .moxygen-release file (release branches): use pinned tag.
          # 2. Submodule SHA reachable from moxygen main: tarball mode via
          #    snapshot-latest. Robust to drift between moxygen merges and
          #    moqx's daily moxygen-sync.
          # 3. Submodule SHA NOT reachable from main (dev iterating against
          #    an unreleased moxygen feature branch): source build. Slow
          #    but supports cross-repo iteration without requiring a
          #    published moxygen tarball.
          if [ -f .moxygen-release ]; then
            export MOQX_MOXYGEN_RELEASE_TAG=$(cat .moxygen-release | tr -d '[:space:]')
            echo "==> Pinned moxygen tag: $MOQX_MOXYGEN_RELEASE_TAG"
            bash scripts/build.sh setup --no-fallback
          else
            SUB_SHA=$(git -C deps/moxygen rev-parse HEAD)
            AHEAD=$(gh api "repos/openmoq/moxygen/compare/main...$SUB_SHA" --jq .ahead_by 2>/dev/null || echo "1")
            if [ "$AHEAD" = "0" ]; then
              echo "==> moxygen submodule $SUB_SHA on main → snapshot tarball"
              bash scripts/build.sh setup --use-latest --no-fallback
            else
              echo "==> moxygen submodule $SUB_SHA $AHEAD commits diverged from main → source build"
              bash scripts/build.sh setup
            fi
          fi

      - name: Build
        run: bash scripts/build.sh --profile ${{ matrix.preset }} --build-dir ${{ matrix.build_dir }}

      - name: Test
        env:
          ASAN_OPTIONS: ${{ matrix.name == 'asan debug' && 'detect_leaks=1:abort_on_error=1' || '' }}
        run: bash scripts/build.sh test --build-dir ${{ matrix.build_dir }} -- --output-junit test-results.xml

      - name: Publish test results
        uses: dorny/test-reporter@v1.9.1
        if: success() || failure()
        with:
          name: "test (${{ matrix.name }})"
          path: ${{ matrix.build_dir }}/test-results.xml
          reporter: java-junit
          fail-on-empty: ${{ job.status == 'success' && 'true' || 'false' }}
          # Only list failing tests in the Check Run summary. Listing all
          # passing tests too blows GitHub's 64 KiB output-body cap on large
          # suites; the per-suite pass/fail counts at the top still render.
          list-tests: failed

  # ════════════════════════════════════════════════════════════════════════════
  # Conformance: mvfst × {d14, d16} × {Q, WT} + pico × {d14, d16} × Q
  # (mirrors ci-pr.yml)
  # ════════════════════════════════════════════════════════════════════════════

  conformance:
    needs: [check-format]
    strategy:
      fail-fast: false
      matrix:
        include:
          # mvfst — both transports for d14 and d16. d14+WT was unblocked by
          # facebookexperimental/moxygen#151 (Accept moq-00 in WT-Available-Protocols).
          # pico — raw QUIC for d14 and d16. WT cells deferred until pico
          # WT CONNECT (openmoq/moxygen#172 / PR #173) syncs in.
          - name: mvfst d14 Q
            versions: "14"
            transport: "Q"
            stack: "mvfst"
          - name: mvfst d14 WT
            versions: "14"
            transport: ""
            stack: "mvfst"
          - name: mvfst d16 Q
            versions: "16"
            transport: "Q"
            stack: "mvfst"
          - name: mvfst d16 WT
            versions: "16"
            transport: ""
            stack: "mvfst"
          - name: pico d14 Q
            versions: "14"
            transport: "Q"
            stack: "pico"
          - name: pico d16 Q
            versions: "16"
            transport: "Q"
            stack: "pico"
    name: conformance (${{ matrix.name }})
    runs-on: ubuntu-22.04
    steps:
      - name: Generate app token
        id: app-token
        uses: actions/create-github-app-token@v2
        with:
          app-id: ${{ secrets.OMOQ_APP_ID }}
          private-key: ${{ secrets.OMOQ_APP_PRIV_KEY }}

      - uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Install system dependencies
        run: sudo deps/moxygen/standalone/install-system-deps.sh

      - name: Setup dependencies
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
        run: |
          # Three-mode setup resolution (matches build job).
          if [ -f .moxygen-release ]; then
            export MOQX_MOXYGEN_RELEASE_TAG=$(cat .moxygen-release | tr -d '[:space:]')
            echo "==> Pinned moxygen tag: $MOQX_MOXYGEN_RELEASE_TAG"
            bash scripts/build.sh setup --no-fallback
          else
            SUB_SHA=$(git -C deps/moxygen rev-parse HEAD)
            AHEAD=$(gh api "repos/openmoq/moxygen/compare/main...$SUB_SHA" --jq .ahead_by 2>/dev/null || echo "1")
            if [ "$AHEAD" = "0" ]; then
              echo "==> moxygen submodule $SUB_SHA on main → snapshot tarball"
              bash scripts/build.sh setup --use-latest --no-fallback
            else
              echo "==> moxygen submodule $SUB_SHA $AHEAD commits diverged from main → source build"
              bash scripts/build.sh setup
            fi
          fi

      - name: Build
        run: bash scripts/build.sh

      - name: Run conformance tests
        run: bash test/test_conformance.sh ./build/moqx ${{ matrix.versions }} ${{ matrix.transport }} ${{ matrix.stack }}

  # ════════════════════════════════════════════════════════════════════════════
  # Microbenchmark: run in-process micro-benchmarks (independent of build/publish pipeline)
  # ════════════════════════════════════════════════════════════════════════════

  microbenchmark:
    needs: [check-format]
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: linux
            runner: ubuntu-22.04
          - name: macos
            # Pinned to macos-15 to match the moxygen publish runner / tarball
            # name (moxygen-macos-15-arm64.tar.gz), downloaded here with
            # --no-fallback. See ci-pr.yml microbenchmark job for the full rationale.
            runner: macos-15
    name: microbenchmark (${{ matrix.name }})
    runs-on: ${{ matrix.runner }}
    steps:
      - name: Generate app token
        id: app-token
        uses: actions/create-github-app-token@v2
        with:
          app-id: ${{ secrets.OMOQ_APP_ID }}
          private-key: ${{ secrets.OMOQ_APP_PRIV_KEY }}

      - uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Install system dependencies
        run: |
          if [[ "$(uname)" == "Darwin" ]]; then
            deps/moxygen/standalone/install-system-deps.sh
          else
            sudo deps/moxygen/standalone/install-system-deps.sh
          fi

      - name: Setup dependencies
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
        run: |
          # Three-mode setup resolution (mirrors the build/conformance jobs):
          # 1. .moxygen-release file (release branches): use pinned tag.
          # 2. Submodule SHA reachable from moxygen main: snapshot tarball
          #    (avoids a slow from-source dep build when moxygen has advanced
          #    past the local pin — the common case under the daily sync).
          # 3. Submodule SHA diverged from main: source build.
          if [ -f .moxygen-release ]; then
            export MOQX_MOXYGEN_RELEASE_TAG=$(cat .moxygen-release | tr -d '[:space:]')
            echo "==> Pinned moxygen tag: $MOQX_MOXYGEN_RELEASE_TAG"
            bash scripts/build.sh setup --no-fallback
          else
            SUB_SHA=$(git -C deps/moxygen rev-parse HEAD)
            AHEAD=$(gh api "repos/openmoq/moxygen/compare/main...$SUB_SHA" --jq .ahead_by 2>/dev/null || echo "1")
            if [ "$AHEAD" = "0" ]; then
              echo "==> moxygen submodule $SUB_SHA on main → snapshot tarball"
              bash scripts/build.sh setup --use-latest --no-fallback
            else
              echo "==> moxygen submodule $SUB_SHA $AHEAD commits diverged from main → source build"
              bash scripts/build.sh setup
            fi
          fi

      - name: Build microbenchmarks
        run: bash scripts/build.sh --benchmark

      - name: Run microbenchmarks
        run: |
          ./build/benchmark/moqx_benchmark \
            --bm_json_verbose=microbench-results.json \
            | tee microbench-output.txt

      - name: Render summary
        if: always()
        run: |
          {
            echo "## Microbenchmark results — ${{ matrix.name }}"
            echo ""
            echo '```'
            cat microbench-output.txt
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Upload microbenchmark artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: microbench-results-${{ matrix.name }}
          path: |
            microbench-results.json
            microbench-output.txt

  # ════════════════════════════════════════════════════════════════════════════
  # Publish: build moqx, Docker image + smoke test, push
  # ════════════════════════════════════════════════════════════════════════════

  publish:
    needs: [check-format, build]
    strategy:
      fail-fast: false
      matrix:
        include:
          - arch: amd64
            runner: ubuntu-22.04
            platform: bookworm-amd64
          - arch: arm64
            runner: ubuntu-22.04-arm
            platform: bookworm-arm64
    name: publish (${{ matrix.arch }})
    runs-on: ${{ matrix.runner }}
    steps:
      - name: Generate app token
        id: app-token
        uses: actions/create-github-app-token@v2
        with:
          app-id: ${{ secrets.OMOQ_APP_ID }}
          private-key: ${{ secrets.OMOQ_APP_PRIV_KEY }}

      - uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Download bookworm moxygen tarball
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
          MOQX_PLATFORM: ${{ matrix.platform }}
        run: |
          # Release branches pin a specific moxygen release tag via
          # .moxygen-release. Main uses snapshot-latest via --use-latest.
          if [ -f .moxygen-release ]; then
            export MOQX_MOXYGEN_RELEASE_TAG=$(cat .moxygen-release | tr -d '[:space:]')
            echo "Using pinned moxygen release tag: $MOQX_MOXYGEN_RELEASE_TAG"
            bash scripts/build.sh setup --no-fallback
          else
            bash scripts/build.sh setup --use-latest --no-fallback
          fi

      - name: Stage tarball for Docker build
        run: |
          mkdir -p .docker-deps
          cp -a .scratch/moxygen-install .docker-deps/moxygen

      - name: Log in to GHCR
        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin

      - name: Compute image tags
        id: tags
        run: |
          SHORT="${GITHUB_SHA:0:7}"
          BRANCH="${{ github.ref_name }}"
          if [[ "$BRANCH" == "main" ]]; then
            LABEL="main"
          else
            LABEL="${BRANCH#release/}"
          fi
          echo "short=$SHORT" >> "$GITHUB_OUTPUT"
          echo "rolling=${LABEL}-latest" >> "$GITHUB_OUTPUT"

      - name: Build Docker image
        run: |
          IMAGE="ghcr.io/${{ github.repository }}"
          ARCH="${{ matrix.arch }}"
          docker build -f docker/Dockerfile \
            -t "${IMAGE}:${{ steps.tags.outputs.short }}-${ARCH}" \
            -t "${IMAGE}:${{ steps.tags.outputs.rolling }}-${ARCH}" \
            .

      - name: Smoke test Docker image
        run: |
          IMAGE="ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.rolling }}-${{ matrix.arch }}"
          echo "==> ldd check"
          docker run --rm --entrypoint ldd "${IMAGE}" /usr/local/bin/moqx
          if docker run --rm --entrypoint ldd "${IMAGE}" /usr/local/bin/moqx 2>&1 | grep -q "not found"; then
            echo "ERROR: missing shared libraries"; exit 1
          fi
          echo "==> Start container with test config"
          docker run -d --name moqx-smoke --network host \
            -v "$PWD/test/test.config.yaml:/etc/moqx/config.yaml:ro" \
            "${IMAGE}" --config=/etc/moqx/config.yaml
          echo "==> Wait for admin /info"
          for i in $(seq 1 50); do
            if curl -sf http://[::1]:9669/info >/dev/null 2>&1; then break; fi
            sleep 0.1
            if [ "$i" -eq 50 ]; then
              echo "ERROR: admin server did not start"; docker logs moqx-smoke; exit 1
            fi
          done
          RESP=$(curl -sf http://[::1]:9669/info)
          echo "Response: $RESP"
          echo "$RESP" | grep -q '"service":"moqx"' || { echo "FAIL: bad /info response"; exit 1; }
          docker stop moqx-smoke && docker rm moqx-smoke
          echo "==> Smoke test passed"

      - name: Build interop client image
        run: |
          CLIENT_IMAGE="ghcr.io/${{ github.repository_owner }}/moqx-interop-client"
          ARCH="${{ matrix.arch }}"
          docker build -f docker/Dockerfile.interop-client \
            -t "${CLIENT_IMAGE}:${{ steps.tags.outputs.short }}-${ARCH}" \
            -t "${CLIENT_IMAGE}:${{ steps.tags.outputs.rolling }}-${ARCH}" \
            .

      - name: Smoke test interop client
        run: |
          CLIENT_IMAGE="ghcr.io/${{ github.repository_owner }}/moqx-interop-client:${{ steps.tags.outputs.rolling }}-${{ matrix.arch }}"
          echo "==> ldd check"
          docker run --rm --entrypoint ldd "${CLIENT_IMAGE}" /usr/local/bin/moq_interop_client
          if docker run --rm --entrypoint ldd "${CLIENT_IMAGE}" /usr/local/bin/moq_interop_client 2>&1 | grep -q "not found"; then
            echo "ERROR: missing shared libraries"; exit 1
          fi
          echo "==> List supported tests"
          docker run --rm "${CLIENT_IMAGE}" --list
          echo "==> Interop client smoke test passed"

      - name: Push Docker images
        run: |
          IMAGE="ghcr.io/${{ github.repository }}"
          CLIENT_IMAGE="ghcr.io/${{ github.repository_owner }}/moqx-interop-client"
          ARCH="${{ matrix.arch }}"
          docker push "${IMAGE}:${{ steps.tags.outputs.short }}-${ARCH}"
          docker push "${IMAGE}:${{ steps.tags.outputs.rolling }}-${ARCH}"
          docker push "${CLIENT_IMAGE}:${{ steps.tags.outputs.short }}-${ARCH}"
          docker push "${CLIENT_IMAGE}:${{ steps.tags.outputs.rolling }}-${ARCH}"

      - name: Package
        id: package
        run: |
          ARTIFACT="moqx-${{ matrix.platform }}.tar.gz"
          docker cp "$(docker create --name extract ghcr.io/${{ github.repository }}:${{ steps.tags.outputs.rolling }}-${{ matrix.arch }}):/usr/local/bin/moqx" .
          docker rm extract
          mkdir -p install/bin && mv moqx install/bin/
          tar czf "$ARTIFACT" -C install .
          echo "artifact=$ARTIFACT" >> "$GITHUB_OUTPUT"

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.package.outputs.artifact }}
          path: ${{ steps.package.outputs.artifact }}
          retention-days: 90

  # ════════════════════════════════════════════════════════════════════════════
  # Manifest: stitch per-arch images into multiplatform tags
  # ════════════════════════════════════════════════════════════════════════════

  manifest:
    needs: [publish]
    runs-on: ubuntu-22.04
    steps:
      - name: Log in to GHCR
        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin

      - name: Compute image tags
        id: tags
        run: |
          SHORT="${GITHUB_SHA:0:7}"
          BRANCH="${{ github.ref_name }}"
          # Every branch gets a rolling <label>-latest tag. main additionally
          # keeps the bare 'latest' for back-compat with anyone pulling it.
          if [[ "$BRANCH" == "main" ]]; then
            LABEL="main"
            ALIAS="latest"   # back-compat: main is also ':latest'
          else
            LABEL="${BRANCH#release/}"
            ALIAS=""
          fi
          echo "short=$SHORT" >> "$GITHUB_OUTPUT"
          echo "rolling=${LABEL}-latest" >> "$GITHUB_OUTPUT"
          echo "alias=$ALIAS" >> "$GITHUB_OUTPUT"

      - name: Create multiplatform manifests (moqx)
        run: |
          IMAGE="ghcr.io/${{ github.repository }}"
          SHORT="${{ steps.tags.outputs.short }}"
          ROLLING="${{ steps.tags.outputs.rolling }}"
          for TAG in "$SHORT" "$ROLLING"; do
            docker buildx imagetools create -t "${IMAGE}:${TAG}" \
              "${IMAGE}:${TAG}-amd64" \
              "${IMAGE}:${TAG}-arm64"
          done
          ALIAS="${{ steps.tags.outputs.alias }}"
          if [[ -n "$ALIAS" ]]; then
            docker buildx imagetools create -t "${IMAGE}:${ALIAS}" \
              "${IMAGE}:${SHORT}-amd64" \
              "${IMAGE}:${SHORT}-arm64"
          fi

      - name: Create multiplatform manifests (interop-client)
        run: |
          CLIENT_IMAGE="ghcr.io/${{ github.repository_owner }}/moqx-interop-client"
          SHORT="${{ steps.tags.outputs.short }}"
          ROLLING="${{ steps.tags.outputs.rolling }}"
          for TAG in "$SHORT" "$ROLLING"; do
            docker buildx imagetools create -t "${CLIENT_IMAGE}:${TAG}" \
              "${CLIENT_IMAGE}:${TAG}-amd64" \
              "${CLIENT_IMAGE}:${TAG}-arm64"
          done
          ALIAS="${{ steps.tags.outputs.alias }}"
          if [[ -n "$ALIAS" ]]; then
            docker buildx imagetools create -t "${CLIENT_IMAGE}:${ALIAS}" \
              "${CLIENT_IMAGE}:${SHORT}-amd64" \
              "${CLIENT_IMAGE}:${SHORT}-arm64"
          fi

  # ════════════════════════════════════════════════════════════════════════════
  # Release: create/update snapshot-latest pre-release (all green)
  # ════════════════════════════════════════════════════════════════════════════

  release:
    needs: [build, manifest]
    runs-on: ubuntu-22.04
    outputs:
      tag: ${{ steps.publish.outputs.tag }}
    steps:
      - uses: actions/checkout@v4

      - name: Download all artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: "*.tar.gz"
          path: artifacts/

      - name: Publish snapshot
        id: publish
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          SHORT="${GITHUB_SHA:0:7}"
          BRANCH="${{ github.ref_name }}"
          if [[ "$BRANCH" == "main" ]]; then
            LABEL="main"
            TAG="snapshot-latest"
          else
            # release/nab-demo → nab-demo → snapshot-nab-demo-latest
            LABEL="${BRANCH#release/}"
            TAG="snapshot-${LABEL}-latest"
          fi
          echo "tag=$TAG" >> "$GITHUB_OUTPUT"

          gh release delete "$TAG" --yes 2>/dev/null || true
          git tag -d "$TAG" 2>/dev/null || true
          git push origin ":refs/tags/$TAG" 2>/dev/null || true

          gh release create "$TAG" artifacts/**/* \
            --target "$GITHUB_SHA" \
            --title "Latest build — ${LABEL} (${SHORT})" \
            --prerelease \
            --notes "$(cat <<EOF
          Rolling snapshot of the latest build from \`${{ github.ref_name }}\`.

          **Commit:** \`${GITHUB_SHA}\`
          **Built:** $(date -u +%Y-%m-%dT%H:%M:%SZ)
          **Docker:** \`ghcr.io/${{ github.repository }}:${SHORT}\`

          This pre-release is automatically replaced on every push to \`${{ github.ref_name }}\`.
          EOF
          )"

  # ════════════════════════════════════════════════════════════════════════════
  # Deploy: auto-deploy main to moqx-main.ci.openmoq.org
  # (release branches are deployed manually via deploy-relay.yml with their
  # own branch-derived domain)
  # ════════════════════════════════════════════════════════════════════════════

  deploy:
    if: github.ref_name == 'main'
    needs: [manifest, release]
    runs-on: [self-hosted, linode]
    env:
      DOMAIN: moqx-main.ci.openmoq.org
      RELAY_PORT: 4433
      ADMIN_PORT: 8000
    steps:
      - uses: actions/checkout@v4

      - name: Ensure DNS A record
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.OMOQ_CERTBOT_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.OMOQ_CERTBOT_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: us-east-1
          HOSTED_ZONE_ID: Z0079758316CI99B99FLR
        run: |
          IP=$(curl -sf https://checkip.amazonaws.com | tr -d '[:space:]')
          if [[ -z "$IP" ]]; then
            echo "::error::Could not determine runner public IP"
            exit 1
          fi
          echo "Ensuring A record: $DOMAIN → $IP"
          CHANGE_FILE=$(mktemp)
          cat > "$CHANGE_FILE" <<EOF
          {
            "Comment": "moqx auto-deploy: $DOMAIN",
            "Changes": [
              {
                "Action": "UPSERT",
                "ResourceRecordSet": {
                  "Name": "$DOMAIN",
                  "Type": "A",
                  "TTL": 300,
                  "ResourceRecords": [{"Value": "$IP"}]
                }
              }
            ]
          }
          EOF
          CHANGE_ID=$(aws route53 change-resource-record-sets \
            --hosted-zone-id "$HOSTED_ZONE_ID" \
            --change-batch "file://$CHANGE_FILE" \
            --query 'ChangeInfo.Id' --output text)
          aws route53 wait resource-record-sets-changed --id "$CHANGE_ID"
          rm -f "$CHANGE_FILE"

      - name: Ensure TLS cert
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.OMOQ_CERTBOT_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.OMOQ_CERTBOT_SECRET_ACCESS_KEY }}
        run: |
          CERT="/etc/letsencrypt/live/${DOMAIN}/fullchain.pem"
          need_issue=false
          if [ ! -f "$CERT" ]; then
            echo "::warning::No cert for ${DOMAIN} — provisioning via Route53"
            need_issue=true
          elif ! sudo openssl x509 -in "$CERT" -noout -checkend 2592000 2>/dev/null; then
            echo "::warning::Cert for ${DOMAIN} expires within 30 days — renewing"
            need_issue=true
          else
            echo "Cert for ${DOMAIN} is valid"
          fi

          if $need_issue; then
            sudo -E certbot certonly --dns-route53 \
              --cert-name "$DOMAIN" \
              -d "$DOMAIN" \
              --non-interactive --agree-tos \
              --email gmarzot@openmoq.org
          fi

      - name: Log in to GHCR
        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin

      # Auto-deploy uses the shared core (docker/relay-deploy.sh) with stats ON,
      # so main keeps the stats stack + public read-only dashboard live. Pulls
      # the compose-pinned :latest (this run's just-published image).
      - name: Deploy (relay + stats + public dashboard)
        env:
          ENABLE_STATS: "true"
          STATS_USER: ${{ secrets.STATS_USER }}
          STATS_PASSWORD: ${{ secrets.STATS_PASSWORD }}
          GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
        run: bash docker/relay-deploy.sh

  # ════════════════════════════════════════════════════════════════════════════
  # Notify: aggregate status (runs after everything)
  # ════════════════════════════════════════════════════════════════════════════

  notify:
    needs: [check-format, build, conformance, publish, manifest, release, deploy]
    if: always()
    runs-on: ubuntu-22.04
    steps:
      - name: Notify Slack
        continue-on-error: true
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.OMOQ_SLACK_WEBHOOK_URL }}
        run: |
          SHORT="${GITHUB_SHA:0:7}"
          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

          # Map job results to status symbols
          fmt_status() {
            case "$1" in
              success)   echo "✓" ;;
              failure)   echo "✗" ;;
              cancelled) echo "⊘" ;;
              *)         echo "—" ;;
            esac
          }

          # verify = worst of check-format + build
          if [ "${{ needs.check-format.result }}" = "failure" ] || [ "${{ needs.build.result }}" = "failure" ]; then
            VER_RESULT="failure"
          elif [ "${{ needs.check-format.result }}" = "cancelled" ] || [ "${{ needs.build.result }}" = "cancelled" ]; then
            VER_RESULT="cancelled"
          elif [ "${{ needs.check-format.result }}" = "success" ] && [ "${{ needs.build.result }}" = "success" ]; then
            VER_RESULT="success"
          else
            VER_RESULT="skipped"
          fi
          VER=$(fmt_status "$VER_RESULT")
          CONF=$(fmt_status "${{ needs.conformance.result }}")
          PUB=$(fmt_status "${{ needs.publish.result }}")
          REL=$(fmt_status "${{ needs.release.result }}")
          DEP=$(fmt_status "${{ needs.deploy.result }}")

          STATUS="verify:${VER} conformance:${CONF} publish:${PUB} release:${REL} deploy:${DEP}"

          if [ "${{ needs.release.result }}" = "success" ]; then
            REL_URL="${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ needs.release.outputs.tag || 'snapshot-latest' }}"
            TEXT=":white_check_mark: *${{ github.repository }}* \`${{ github.ref_name }}\` \`${SHORT}\` — ${STATUS}  <${RUN_URL}|#${{ github.run_number }}> artifacts: <${REL_URL}|view>"
          else
            TEXT=":x: *${{ github.repository }}* \`${{ github.ref_name }}\` \`${SHORT}\` — ${STATUS}  <${RUN_URL}|#${{ github.run_number }}>"
          fi

          curl -s -X POST "$SLACK_WEBHOOK_URL" \
            -H "Content-Type: application/json" \
            --data "{\"text\": \"${TEXT}\"}"

      - name: Notify email
        continue-on-error: true
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.OMOQ_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.OMOQ_AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: us-east-1
        run: |
          SHORT="${GITHUB_SHA:0:7}"
          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

          # verify = worst of check-format + build
          CF="${{ needs.check-format.result }}"
          BLD="${{ needs.build.result }}"
          if [ "$CF" = "failure" ] || [ "$BLD" = "failure" ]; then
            VER="failure"
          elif [ "$CF" = "cancelled" ] || [ "$BLD" = "cancelled" ]; then
            VER="cancelled"
          elif [ "$CF" = "success" ] && [ "$BLD" = "success" ]; then
            VER="success"
          else
            VER="skipped"
          fi
          PUB="${{ needs.publish.result }}"
          REL="${{ needs.release.result }}"
          DEP="${{ needs.deploy.result }}"

          STATUS="verify:${VER} publish:${PUB} release:${REL} deploy:${DEP}"

          if [ "${{ needs.release.result }}" = "success" ]; then
            REL_URL="${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ needs.release.outputs.tag || 'snapshot-latest' }}"
            SUBJECT="[moqx] published ${{ github.ref_name }} ${SHORT}"
            BODY="Repository: ${{ github.repository }}\nBranch: ${{ github.ref_name }}\nCommit: ${GITHUB_SHA}\nStatus: ${STATUS}\nDocker: ghcr.io/${{ github.repository }}:${SHORT}\nArtifacts: ${REL_URL}\nRun: ${RUN_URL}"
          else
            SUBJECT="[moqx] pipeline failed ${{ github.ref_name }} ${SHORT}"
            BODY="Repository: ${{ github.repository }}\nBranch: ${{ github.ref_name }}\nCommit: ${GITHUB_SHA}\nStatus: ${STATUS}\nRun: ${RUN_URL}"
          fi

          aws ses send-email \
            --from "noreply@ci.openmoq.org" \
            --destination '{"ToAddresses":["github-notifications@openmoq.org"]}' \
            --message "{
              \"Subject\": {\"Data\": \"${SUBJECT}\"},
              \"Body\": {\"Text\": {\"Data\": \"${BODY}\"}}
            }"