Benchmarks #29
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| # ------------------------------------------------------------------------------ | |
| # Workflow Settings | |
| # ------------------------------------------------------------------------------ | |
| on: | |
| schedule: | |
| - cron: "17 3 * * *" # 11:17 PM ET | |
| workflow_dispatch: | |
| inputs: | |
| threshold: | |
| description: "Regression threshold (fraction, e.g. 0.01 = 1%)" | |
| required: false | |
| type: string | |
| default: "0.1" | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: false | |
| permissions: {} | |
| env: | |
| CARGO_TERM_COLOR: always | |
| THRESHOLD: ${{ inputs.threshold || '0.10' }} | |
| jobs: | |
| # ---------------------------------------------------------------------------- | |
| # Comparative Benchmarks (Praxis vs Envoy) | |
| # ---------------------------------------------------------------------------- | |
| benchmark: | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Install stable Rust | |
| uses: dtolnay/rust-toolchain@6190aa5fb88a88ee71c12769924bbe63a9ab152e # 1.96.0 | |
| - name: Install benchmark tools | |
| run: | | |
| make tools | |
| echo "${{ github.workspace }}/target/praxis-binutils" >> "$GITHUB_PATH" | |
| - name: Download baseline | |
| id: baseline | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| run_id=$(gh api "repos/${{ github.repository }}/actions/workflows/benchmarks.yaml/runs?branch=main&status=success&per_page=1" \ | |
| --jq '.workflow_runs[0].id // empty') | |
| if [ -z "$run_id" ]; then | |
| echo "status=missing" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| artifact_id=$(gh api "repos/${{ github.repository }}/actions/runs/${run_id}/artifacts" \ | |
| --jq '.artifacts[] | select(.name == "benchmark-baseline") | .id // empty') | |
| if [ -z "$artifact_id" ]; then | |
| echo "status=missing" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| gh api "repos/${{ github.repository }}/actions/artifacts/${artifact_id}/zip" > baseline.zip | |
| unzip -o baseline.zip -d . | |
| mv benchmark-results.yaml baseline.yaml | |
| echo "status=ok" >> "$GITHUB_OUTPUT" | |
| - name: Run benchmarks | |
| env: | |
| THRESHOLD: ${{ env.THRESHOLD }} | |
| run: | | |
| cargo xtask benchmark \ | |
| --proxy envoy \ | |
| --threshold "$THRESHOLD" \ | |
| --runs 5 \ | |
| --duration 60 \ | |
| --warmup 30 \ | |
| --format yaml \ | |
| --output benchmark-results.yaml | |
| - name: Compare against baseline | |
| id: compare | |
| if: steps.baseline.outputs.status == 'ok' | |
| continue-on-error: true | |
| env: | |
| THRESHOLD: ${{ env.THRESHOLD }} | |
| run: | | |
| cargo xtask benchmark compare \ | |
| baseline.yaml benchmark-results.yaml \ | |
| --threshold "$THRESHOLD" | tee compare-output.txt | |
| - name: Post comparison to job summary | |
| if: steps.compare.outcome != 'skipped' | |
| run: | | |
| { | |
| echo "## Benchmark Comparison" | |
| echo '```' | |
| cat compare-output.txt 2>/dev/null || echo "No comparison output" | |
| echo '```' | |
| if [ "${{ steps.compare.outcome }}" = "failure" ]; then | |
| echo "" | |
| echo "> **Warning**: Regression detected. Review results for CI noise vs genuine regression." | |
| fi | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload baseline | |
| if: steps.compare.outcome != 'failure' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-baseline | |
| path: benchmark-results.yaml | |
| overwrite: true | |
| - name: Upload timestamped results | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-${{ github.sha }} | |
| path: benchmark-results.yaml |