Benchmarks #53
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Benchmark workflow - runs benchmarks and handles results flexibly | |
| # | |
| # Triggers: | |
| # - Manual dispatch only (during release prep) | |
| # | |
| # Output modes: | |
| # - pr: Create PR to main | |
| # - commit: Commit directly to the ref (for pre-release branches) | |
| # - artifact: Upload only, no commit (for batch backfill) | |
| # | |
| # Strategy: | |
| # - Matrix runs 10 packages in parallel (~5 min wall time) | |
| # - Results combined into benchmarks/benchmark-v1.X.Y.txt | |
| # | |
| # Usage: | |
| # During release prep, run with output_mode=commit on the pre-release branch. | |
| # The benchmark file will be included in the release PR before tagging. | |
| name: Benchmarks | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: 'Version for output file (e.g., v1.34.0)' | |
| required: true | |
| ref: | |
| description: 'Git ref to checkout (branch, tag, SHA). Defaults to version tag.' | |
| required: false | |
| output_mode: | |
| description: 'How to handle results' | |
| type: choice | |
| options: | |
| - pr # Create PR to main (tag push default) | |
| - commit # Commit directly to ref (pre-release branch) | |
| - artifact # Upload only (batch backfill) | |
| default: 'pr' | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| package: [parser, validator, fixer, httpvalidator, converter, joiner, differ, generator, builder, walker] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| # Priority: explicit ref > version input > triggered ref | |
| ref: ${{ inputs.ref || inputs.version || github.ref }} | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '1.24' | |
| cache: true | |
| - name: Run benchmarks for ${{ matrix.package }} | |
| run: | | |
| go test -bench=. -benchmem -benchtime=5s -timeout=15m \ | |
| ./${{ matrix.package }} > bench-${{ matrix.package }}.txt 2>&1 || true | |
| cat bench-${{ matrix.package }}.txt | |
| - name: Upload benchmark artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bench-${{ matrix.package }} | |
| path: bench-${{ matrix.package }}.txt | |
| retention-days: 7 | |
| collect: | |
| needs: benchmark | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| # For commit mode, checkout the target ref; otherwise main is fine | |
| ref: ${{ inputs.output_mode == 'commit' && inputs.ref || 'main' }} | |
| - name: Download all benchmark artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: bench-results | |
| - name: Combine benchmark results | |
| run: | | |
| VERSION="${{ inputs.version || github.ref_name }}" | |
| echo "Combining benchmarks for version: $VERSION" | |
| mkdir -p benchmarks | |
| cat bench-results/bench-parser/bench-parser.txt \ | |
| bench-results/bench-validator/bench-validator.txt \ | |
| bench-results/bench-fixer/bench-fixer.txt \ | |
| bench-results/bench-httpvalidator/bench-httpvalidator.txt \ | |
| bench-results/bench-converter/bench-converter.txt \ | |
| bench-results/bench-joiner/bench-joiner.txt \ | |
| bench-results/bench-differ/bench-differ.txt \ | |
| bench-results/bench-generator/bench-generator.txt \ | |
| bench-results/bench-builder/bench-builder.txt \ | |
| bench-results/bench-walker/bench-walker.txt \ | |
| > benchmarks/benchmark-${VERSION}.txt | |
| echo "Combined benchmark file:" | |
| wc -l benchmarks/benchmark-${VERSION}.txt | |
| head -50 benchmarks/benchmark-${VERSION}.txt | |
| - name: Upload combined benchmark artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-combined-${{ inputs.version || github.ref_name }} | |
| path: benchmarks/benchmark-*.txt | |
| retention-days: 30 | |
| - name: Commit directly to ref | |
| if: ${{ inputs.output_mode == 'commit' }} | |
| run: | | |
| VERSION="${{ inputs.version }}" | |
| REF="${{ inputs.ref }}" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add benchmarks/ | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit" | |
| exit 0 | |
| fi | |
| git commit -m "chore: add benchmark results for ${VERSION} | |
| Generated by CI benchmark workflow on ${REF} | |
| 🤖 Generated automatically" | |
| git push origin HEAD:${REF} | |
| echo "Benchmark committed directly to ${REF}" | |
| - name: Create PR with benchmark results | |
| if: ${{ inputs.output_mode == 'pr' }} | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| VERSION="${{ inputs.version || github.ref_name }}" | |
| BRANCH="chore/benchmark-${VERSION}" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add benchmarks/ | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit" | |
| exit 0 | |
| fi | |
| git checkout -b "$BRANCH" | |
| git commit -m "chore: add benchmark results for ${VERSION}" | |
| git push origin "$BRANCH" | |
| gh pr create \ | |
| --title "chore: add benchmark results for ${VERSION}" \ | |
| --body "## Automated Benchmark Results | |
| This PR adds benchmark results for ${VERSION}, generated automatically by the benchmark workflow. | |
| **Workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| --- | |
| 🤖 Generated automatically by the benchmark workflow" \ | |
| --base main \ | |
| --head "$BRANCH" | |
| echo "PR created for benchmark results" | |
| - name: Compare with previous version | |
| run: | | |
| VERSION="${{ inputs.version || github.ref_name }}" | |
| # Find previous benchmark file (in local/ for historical, or root for CI) | |
| PREV_FILE=$(ls -1 benchmarks/benchmark-v*.txt benchmarks/local/benchmark-v*.txt 2>/dev/null \ | |
| | grep -v "${VERSION}" | sort -V | tail -1 || true) | |
| if [ -n "$PREV_FILE" ] && [ -f "$PREV_FILE" ]; then | |
| echo "Comparing with previous: $PREV_FILE" | |
| go install golang.org/x/perf/cmd/benchstat@latest 2>/dev/null || true | |
| if command -v benchstat >/dev/null 2>&1; then | |
| benchstat "$PREV_FILE" "benchmarks/benchmark-${VERSION}.txt" || true | |
| else | |
| echo "benchstat not available, skipping comparison" | |
| fi | |
| else | |
| echo "No previous benchmark file found for comparison" | |
| fi |