perf-test.yml

name: perf test

# Standalone perf-regression workflow.
#
# Triggers
# --------
#   schedule         — nightly run against main HEAD. This is the canonical
#                      trend source; only scheduled runs publish to Pages.
#   workflow_dispatch — manual runs from the Actions tab or `gh workflow run`,
#                      available to members/maintainers. The `ref` input lets
#                      a maintainer point a run at any PR branch on demand.
#   workflow_call    — reusable, for invocation from another workflow.
#
# This workflow does NOT run automatically on push or pull_request — it never
# competes with per-PR CI on the shared self-hosted VMs, and untrusted PR code
# never executes here automatically.
#
# Concurrency
# -----------
# A single fixed group (`perf-test`) so scheduled, manual, and called runs all
# queue against the same physical Linode VMs. `cancel-in-progress: false`
# because a partial test leaves VM state behind; queueing is safer.
#
# Publishing
# ----------
# Results stage as a Pages artifact (180-run rolling window + dashboard) and
# only deploy on the nightly schedule. Manual / called runs upload the
# artifact for inspection but do not publish — keeps experiments from
# clobbering the main trend line at openmoq.org/moqx/.

on:
  schedule:
    # Nightly perf run against main HEAD (05:00 UTC). Scheduled runs always
    # execute on the default branch (main) and are the only runs that publish.
    - cron: '0 5 * * *'

  workflow_dispatch:
    # No `ref` input: the branch/tag under test is the one selected in the
    # native "Use workflow from" picker (github.ref). That keeps a single
    # branch control and makes the recorded branch label match what was tested.
    inputs:
      duration:
        description: 'test duration seconds'
        type: number
        default: 120
      subscribers:
        description: 'peak subscribers'
        type: number
        default: 600
      compare:
        description: 'compare against the published nightly baseline and render a report'
        type: boolean
        default: true
      pr:
        description: 'PR number to post the report to (blank = report only in the step summary)'
        type: string
        default: ''

  workflow_call:
    inputs:
      ref:
        type: string
        required: false
        default: ''
      duration:
        type: number
        required: false
        default: 120
      subscribers:
        type: number
        required: false
        default: 600
      compare:
        type: boolean
        required: false
        default: false
      pr:
        type: string
        required: false
        default: ''

concurrency:
  group: perf-test
  cancel-in-progress: false

permissions:
  contents: read
  pull-requests: write   # used only by the optional "post report to PR" step (pr input)

jobs:
  perf:
    name: perf-test
    runs-on: [self-hosted, linode]
    outputs:
      run-file: ${{ steps.collect.outputs.run-file }}
    steps:
      - name: Generate app token
        id: app-token
        uses: actions/create-github-app-token@v2
        with:
          app-id: ${{ secrets.OMOQ_APP_ID }}
          private-key: ${{ secrets.OMOQ_APP_PRIV_KEY }}

      - uses: actions/checkout@v4
        with:
          # workflow_call passes an explicit ref; dispatch/schedule leave it
          # empty, so checkout falls back to github.ref (the selected branch on
          # dispatch, main HEAD on schedule).
          ref: ${{ inputs.ref }}
          submodules: true

      - name: Determine SHA
        id: sha
        run: |
          SHA=$(git rev-parse HEAD)
          echo "sha=$SHA"          >> "$GITHUB_OUTPUT"
          echo "short=${SHA:0:7}"  >> "$GITHUB_OUTPUT"

      - name: Install system dependencies
        run: sudo deps/moxygen/standalone/install-system-deps.sh

      - name: Setup dependencies
        env:
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
        run: bash scripts/build.sh setup

      - name: Build moqx
        run: bash scripts/build.sh

      - name: Configure SSH for perf VMs
        env:
          PERF_SSH_KEY: ${{ secrets.PERF_SSH_KEY }}
        run: |
          install -d -m 700 ~/.ssh
          printf '%s\n' "$PERF_SSH_KEY" > ~/.ssh/perf-key
          chmod 600 ~/.ssh/perf-key
          echo "PERF_SSH_KEY_FILE=$HOME/.ssh/perf-key" >> "$GITHUB_ENV"

      - name: Run perf test on dedicated VMs
        id: collect
        env:
          PERF_RELAY_HOST:    ${{ secrets.PERF_RELAY_HOST }}
          PERF_CLIENT_HOST:   ${{ secrets.PERF_CLIENT_HOST }}
        run: |
          FILE="run-${{ steps.sha.outputs.short }}.json"
          # Fallbacks: schedule events carry no `inputs`, so apply the canonical
          # nightly load when invoked by cron.
          bash scripts/perf-test-ci.sh \
            --duration       "${{ inputs.duration || '120' }}" \
            --subscriber-max "${{ inputs.subscribers || '600' }}" \
            --ramp           150 \
            --io-threads     4 \
            --client-threads 8 \
            --client-args    "--first_object_size=26516 --other_object_size=3788 --objects_per_group 60 --delivery_timeout 5000" \
            --cooldown       10 \
            --output         "$FILE"
          echo "run-file=$FILE" >> "$GITHUB_OUTPUT"

      - name: Render step summary
        if: always()
        env:
          FILE: ${{ steps.collect.outputs.run-file }}
        run: |
          {
            echo "## perf-test — $(date -u +'%Y-%m-%d %H:%M UTC')"
            echo
            echo "ref: \`${{ steps.sha.outputs.short }}\` · branch: \`${{ github.ref_name }}\` · subs: ${{ inputs.subscribers || '600' }} · duration: ${{ inputs.duration || '120' }}s"
            echo
            if [ -f "$FILE" ]; then
              echo '| metric | value |'
              echo '|---|---|'
              jq -r '
                (if (.metrics | type) == "object" then .metrics
                 elif (.results | type) == "object" then .results
                 else {}
                 end)
                | to_entries[]
                | "| \(.key) | \(.value) |"
              ' "$FILE"
            else
              echo "_no results JSON produced_"
            fi
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Pull rolling baseline from currently-published Pages
        # Downloads index.json + the latest N run JSONs into data/ so
        # perf-compare.py can score the new run against history.
        # No-op on the very first deploy (index.json 404).
        env:
          BASE: https://openmoq.org/moqx/perf
          KEEP: 180
        run: |
          mkdir -p data
          if curl -fsSL "$BASE/index.json" -o data/index.json; then
            jq -r '.runs[].file' data/index.json | head -n $((KEEP - 1)) | \
              while read -r f; do
                curl -fsSL "$BASE/$f" -o "data/$f" 2>/dev/null || true
              done
          fi

      - name: Stage Pages payload (dashboard + rolling window + new run)
        env:
          KEEP: 180
          FILE: ${{ steps.collect.outputs.run-file }}
        run: |
          OUT=perf-out
          mkdir -p "$OUT/perf"
          # Dashboard (status/index.html) is in-repo and uses ./perf at runtime
          cp status/index.html "$OUT/index.html"
          # Copy historical runs, then the new run (overrides any stale copy
          # of the same SHA name)
          cp data/run-*.json "$OUT/perf/" 2>/dev/null || true
          cp "$FILE" "$OUT/perf/$FILE"
          # Rebuild manifest with per-run metadata (branch + timestamp). The
          # run filenames are SHA-based, so they cannot be ordered by name —
          # sort newest-first by timestamp. perf-compare.py filters entries by
          # branch and the dashboard relies on chronological ordering. Cap KEEP.
          for f in "$OUT/perf/"run-*.json; do
            [ -e "$f" ] || continue
            jq -c --arg file "$(basename "$f")" \
              '{file: $file, branch: (.branch // "unknown"), timestamp: (.timestamp // ""), commit: (.commit // "")}' "$f"
          done \
            | jq -s "sort_by(.timestamp) | reverse | .[0:${KEEP}] | {runs: .}" \
            > "$OUT/perf/index.json"

      - name: Upload Pages artifact
        # Picked up by actions/deploy-pages in the deploy job (gated to the
        # nightly schedule). Dispatch / called runs still upload it so reviewers
        # can download + inspect without publishing.
        uses: actions/upload-pages-artifact@v3
        with:
          path: perf-out

      - name: Upload workflow artifacts (always retained)
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: perf-results
          path: |
            ${{ steps.collect.outputs.run-file }}
            perf-out/
          retention-days: 30

      - name: Compare vs rolling baseline (report)
        # Opt-in via the `compare` input. perf-compare.py is non-blocking
        # (always exits 0) and renders a "no baseline yet" note on first run.
        # Always compares against the published main trend, so the report is
        # only meaningful when run at matching load (subs=1000).
        if: ${{ inputs.compare }}
        env:
          FILE: ${{ steps.collect.outputs.run-file }}
        run: |
          python3 scripts/perf-compare.py \
            --current "$FILE" \
            --data-dir data \
            --window 10 \
            --threshold 5 \
            --output comment.md
          {
            echo
            echo "## perf comparison vs published baseline (subs=${{ inputs.subscribers || '600' }})"
            echo
            cat comment.md
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Post comparison report to PR
        # Only when a PR number is supplied. Blank `pr` = zero PR footprint;
        # the report still lives in this run's step summary, whose URL can be
        # referenced from a PR by hand.
        if: ${{ inputs.compare && inputs.pr != '' }}
        env:
          GH_TOKEN:  ${{ github.token }}
          PR_NUMBER: ${{ inputs.pr }}
        run: |
          gh pr comment "$PR_NUMBER" \
            --repo "$GITHUB_REPOSITORY" \
            --body-file comment.md

  deploy:
    needs: perf
    # Only the nightly schedule publishes — manual dispatch (of any branch) and
    # reusable calls upload the artifact for inspection but leave the published
    # trend line untouched. Scheduled runs always execute on main HEAD.
    if: github.event_name == 'schedule'
    runs-on: ubuntu-latest
    permissions:
      pages: write
      id-token: write
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    steps:
      # Create the Pages deployment as the omoq-sync-bot App rather than under
      # the personal actor GitHub attributes scheduled runs to. Keeps the
      # deployments list owned by the bot, consistent with the rest of the org
      # automation. Requires the App installation to have Pages: write.
      - uses: actions/create-github-app-token@v2
        id: app-token
        with:
          app-id: ${{ secrets.OMOQ_APP_ID }}
          private-key: ${{ secrets.OMOQ_APP_PRIV_KEY }}
      - id: deployment
        uses: actions/deploy-pages@v4
        with:
          token: ${{ steps.app-token.outputs.token }}