perf test #39
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: perf test | |
| # Standalone perf-regression workflow. | |
| # | |
| # Triggers | |
| # -------- | |
| # schedule — nightly run against main HEAD. This is the canonical | |
| # trend source; only scheduled runs publish to Pages. | |
| # workflow_dispatch — manual runs from the Actions tab or `gh workflow run`, | |
| # available to members/maintainers. The `ref` input lets | |
| # a maintainer point a run at any PR branch on demand. | |
| # workflow_call — reusable, for invocation from another workflow. | |
| # | |
| # This workflow does NOT run automatically on push or pull_request — it never | |
| # competes with per-PR CI on the shared self-hosted VMs, and untrusted PR code | |
| # never executes here automatically. | |
| # | |
| # Concurrency | |
| # ----------- | |
| # A single fixed group (`perf-test`) so scheduled, manual, and called runs all | |
| # queue against the same physical Linode VMs. `cancel-in-progress: false` | |
| # because a partial test leaves VM state behind; queueing is safer. | |
| # | |
| # Publishing | |
| # ---------- | |
| # Results stage as a Pages artifact (180-run rolling window + dashboard) and | |
| # only deploy on the nightly schedule. Manual / called runs upload the | |
| # artifact for inspection but do not publish — keeps experiments from | |
| # clobbering the main trend line at openmoq.org/moqx/. | |
| on: | |
| schedule: | |
| # Nightly perf run against main HEAD (05:00 UTC). Scheduled runs always | |
| # execute on the default branch (main) and are the only runs that publish. | |
| - cron: '0 5 * * *' | |
| workflow_dispatch: | |
| # No `ref` input: the branch/tag under test is the one selected in the | |
| # native "Use workflow from" picker (github.ref). That keeps a single | |
| # branch control and makes the recorded branch label match what was tested. | |
| inputs: | |
| duration: | |
| description: 'test duration seconds' | |
| type: number | |
| default: 120 | |
| subscribers: | |
| description: 'peak subscribers' | |
| type: number | |
| default: 600 | |
| compare: | |
| description: 'compare against the published nightly baseline and render a report' | |
| type: boolean | |
| default: true | |
| pr: | |
| description: 'PR number to post the report to (blank = report only in the step summary)' | |
| type: string | |
| default: '' | |
| workflow_call: | |
| inputs: | |
| ref: | |
| type: string | |
| required: false | |
| default: '' | |
| duration: | |
| type: number | |
| required: false | |
| default: 120 | |
| subscribers: | |
| type: number | |
| required: false | |
| default: 600 | |
| compare: | |
| type: boolean | |
| required: false | |
| default: false | |
| pr: | |
| type: string | |
| required: false | |
| default: '' | |
| concurrency: | |
| group: perf-test | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| pull-requests: write # used only by the optional "post report to PR" step (pr input) | |
| jobs: | |
| perf: | |
| name: perf-test | |
| runs-on: [self-hosted, linode] | |
| outputs: | |
| run-file: ${{ steps.collect.outputs.run-file }} | |
| steps: | |
| - name: Generate app token | |
| id: app-token | |
| uses: actions/create-github-app-token@v2 | |
| with: | |
| app-id: ${{ secrets.OMOQ_APP_ID }} | |
| private-key: ${{ secrets.OMOQ_APP_PRIV_KEY }} | |
| - uses: actions/checkout@v4 | |
| with: | |
| # workflow_call passes an explicit ref; dispatch/schedule leave it | |
| # empty, so checkout falls back to github.ref (the selected branch on | |
| # dispatch, main HEAD on schedule). | |
| ref: ${{ inputs.ref }} | |
| submodules: true | |
| - name: Determine SHA | |
| id: sha | |
| run: | | |
| SHA=$(git rev-parse HEAD) | |
| echo "sha=$SHA" >> "$GITHUB_OUTPUT" | |
| echo "short=${SHA:0:7}" >> "$GITHUB_OUTPUT" | |
| - name: Install system dependencies | |
| run: sudo deps/moxygen/standalone/install-system-deps.sh | |
| - name: Setup dependencies | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| run: bash scripts/build.sh setup | |
| - name: Build moqx | |
| run: bash scripts/build.sh | |
| - name: Configure SSH for perf VMs | |
| env: | |
| PERF_SSH_KEY: ${{ secrets.PERF_SSH_KEY }} | |
| run: | | |
| install -d -m 700 ~/.ssh | |
| printf '%s\n' "$PERF_SSH_KEY" > ~/.ssh/perf-key | |
| chmod 600 ~/.ssh/perf-key | |
| echo "PERF_SSH_KEY_FILE=$HOME/.ssh/perf-key" >> "$GITHUB_ENV" | |
| - name: Run perf test on dedicated VMs | |
| id: collect | |
| env: | |
| PERF_RELAY_HOST: ${{ secrets.PERF_RELAY_HOST }} | |
| PERF_CLIENT_HOST: ${{ secrets.PERF_CLIENT_HOST }} | |
| run: | | |
| FILE="run-${{ steps.sha.outputs.short }}.json" | |
| # Fallbacks: schedule events carry no `inputs`, so apply the canonical | |
| # nightly load when invoked by cron. | |
| bash scripts/perf-test-ci.sh \ | |
| --duration "${{ inputs.duration || '120' }}" \ | |
| --subscriber-max "${{ inputs.subscribers || '600' }}" \ | |
| --ramp 150 \ | |
| --io-threads 4 \ | |
| --client-threads 8 \ | |
| --client-args "--first_object_size=26516 --other_object_size=3788 --objects_per_group 60 --delivery_timeout 5000" \ | |
| --cooldown 10 \ | |
| --output "$FILE" | |
| echo "run-file=$FILE" >> "$GITHUB_OUTPUT" | |
| - name: Render step summary | |
| if: always() | |
| env: | |
| FILE: ${{ steps.collect.outputs.run-file }} | |
| run: | | |
| { | |
| echo "## perf-test — $(date -u +'%Y-%m-%d %H:%M UTC')" | |
| echo | |
| echo "ref: \`${{ steps.sha.outputs.short }}\` · branch: \`${{ github.ref_name }}\` · subs: ${{ inputs.subscribers || '600' }} · duration: ${{ inputs.duration || '120' }}s" | |
| echo | |
| if [ -f "$FILE" ]; then | |
| echo '| metric | value |' | |
| echo '|---|---|' | |
| jq -r ' | |
| (if (.metrics | type) == "object" then .metrics | |
| elif (.results | type) == "object" then .results | |
| else {} | |
| end) | |
| | to_entries[] | |
| | "| \(.key) | \(.value) |" | |
| ' "$FILE" | |
| else | |
| echo "_no results JSON produced_" | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Pull rolling baseline from currently-published Pages | |
| # Downloads index.json + the latest N run JSONs into data/ so | |
| # perf-compare.py can score the new run against history. | |
| # No-op on the very first deploy (index.json 404). | |
| env: | |
| BASE: https://openmoq.org/moqx/perf | |
| KEEP: 180 | |
| run: | | |
| mkdir -p data | |
| if curl -fsSL "$BASE/index.json" -o data/index.json; then | |
| jq -r '.runs[].file' data/index.json | head -n $((KEEP - 1)) | \ | |
| while read -r f; do | |
| curl -fsSL "$BASE/$f" -o "data/$f" 2>/dev/null || true | |
| done | |
| fi | |
| - name: Stage Pages payload (dashboard + rolling window + new run) | |
| env: | |
| KEEP: 180 | |
| FILE: ${{ steps.collect.outputs.run-file }} | |
| run: | | |
| OUT=perf-out | |
| mkdir -p "$OUT/perf" | |
| # Dashboard (status/index.html) is in-repo and uses ./perf at runtime | |
| cp status/index.html "$OUT/index.html" | |
| # Copy historical runs, then the new run (overrides any stale copy | |
| # of the same SHA name) | |
| cp data/run-*.json "$OUT/perf/" 2>/dev/null || true | |
| cp "$FILE" "$OUT/perf/$FILE" | |
| # Rebuild manifest with per-run metadata (branch + timestamp). The | |
| # run filenames are SHA-based, so they cannot be ordered by name — | |
| # sort newest-first by timestamp. perf-compare.py filters entries by | |
| # branch and the dashboard relies on chronological ordering. Cap KEEP. | |
| for f in "$OUT/perf/"run-*.json; do | |
| [ -e "$f" ] || continue | |
| jq -c --arg file "$(basename "$f")" \ | |
| '{file: $file, branch: (.branch // "unknown"), timestamp: (.timestamp // ""), commit: (.commit // "")}' "$f" | |
| done \ | |
| | jq -s "sort_by(.timestamp) | reverse | .[0:${KEEP}] | {runs: .}" \ | |
| > "$OUT/perf/index.json" | |
| - name: Upload Pages artifact | |
| # Picked up by actions/deploy-pages in the deploy job (gated to the | |
| # nightly schedule). Dispatch / called runs still upload it so reviewers | |
| # can download + inspect without publishing. | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: perf-out | |
| - name: Upload workflow artifacts (always retained) | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: perf-results | |
| path: | | |
| ${{ steps.collect.outputs.run-file }} | |
| perf-out/ | |
| retention-days: 30 | |
| - name: Compare vs rolling baseline (report) | |
| # Opt-in via the `compare` input. perf-compare.py is non-blocking | |
| # (always exits 0) and renders a "no baseline yet" note on first run. | |
| # Always compares against the published main trend, so the report is | |
| # only meaningful when run at matching load (subs=1000). | |
| if: ${{ inputs.compare }} | |
| env: | |
| FILE: ${{ steps.collect.outputs.run-file }} | |
| run: | | |
| python3 scripts/perf-compare.py \ | |
| --current "$FILE" \ | |
| --data-dir data \ | |
| --window 10 \ | |
| --threshold 5 \ | |
| --output comment.md | |
| { | |
| echo | |
| echo "## perf comparison vs published baseline (subs=${{ inputs.subscribers || '600' }})" | |
| echo | |
| cat comment.md | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Post comparison report to PR | |
| # Only when a PR number is supplied. Blank `pr` = zero PR footprint; | |
| # the report still lives in this run's step summary, whose URL can be | |
| # referenced from a PR by hand. | |
| if: ${{ inputs.compare && inputs.pr != '' }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ inputs.pr }} | |
| run: | | |
| gh pr comment "$PR_NUMBER" \ | |
| --repo "$GITHUB_REPOSITORY" \ | |
| --body-file comment.md | |
| deploy: | |
| needs: perf | |
| # Only the nightly schedule publishes — manual dispatch (of any branch) and | |
| # reusable calls upload the artifact for inspection but leave the published | |
| # trend line untouched. Scheduled runs always execute on main HEAD. | |
| if: github.event_name == 'schedule' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pages: write | |
| id-token: write | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| steps: | |
| # Create the Pages deployment as the omoq-sync-bot App rather than under | |
| # the personal actor GitHub attributes scheduled runs to. Keeps the | |
| # deployments list owned by the bot, consistent with the rest of the org | |
| # automation. Requires the App installation to have Pages: write. | |
| - uses: actions/create-github-app-token@v2 | |
| id: app-token | |
| with: | |
| app-id: ${{ secrets.OMOQ_APP_ID }} | |
| private-key: ${{ secrets.OMOQ_APP_PRIV_KEY }} | |
| - id: deployment | |
| uses: actions/deploy-pages@v4 | |
| with: | |
| token: ${{ steps.app-token.outputs.token }} |