Add persistent Benchee benchmarks #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: benchmark | |
| on: | |
| pull_request: | |
| push: | |
| branches: [master] | |
| schedule: | |
| - cron: "17 10 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| concurrency: | |
| group: benchmark-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| row-binary: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - id: beam | |
| uses: erlef/setup-beam@v1 | |
| with: | |
| elixir-version: "1.19" | |
| otp-version: "28" | |
| - uses: actions/cache@v6 | |
| with: | |
| path: | | |
| deps | |
| _build | |
| key: benchmark-${{ runner.os }}-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('mix.lock') }} | |
| restore-keys: benchmark-${{ runner.os }}-${{ steps.beam.outputs.elixir-version }}- | |
| - run: mix deps.get --only dev | |
| - run: mix compile --warnings-as-errors | |
| - run: mix benchmark | |
| - name: Locate benchmark result | |
| id: benchmark-result | |
| run: | | |
| result_path=$(cat bench/output/current-result-path) | |
| machine=$(jq --raw-output .machine "$result_path") | |
| version=$(jq --raw-output .benchmark_version "$result_path") | |
| echo "path=$result_path" >> "$GITHUB_OUTPUT" | |
| echo "machine=$machine" >> "$GITHUB_OUTPUT" | |
| echo "version=$version" >> "$GITHUB_OUTPUT" | |
| - name: Install ClickHouse Local | |
| env: | |
| DO_NOT_TRACK: "1" | |
| run: | | |
| curl --fail --silent --show-error https://clickhouse.com/cli | sh | |
| "$HOME/.local/bin/clickhousectl" local use latest | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Load benchmark history | |
| id: benchmark-history | |
| run: | | |
| result_path="${{ steps.benchmark-result.outputs.path }}" | |
| machine="${{ steps.benchmark-result.outputs.machine }}" | |
| if git fetch origin benchmark-results; then | |
| git worktree add bench-history FETCH_HEAD | |
| else | |
| git worktree add --detach bench-history | |
| git -C bench-history switch --orphan benchmark-results | |
| fi | |
| mkdir -p bench/compare | |
| cp "$result_path" bench/compare/current.json | |
| if test -f "bench-history/latest/$machine/benchmark.json"; then | |
| cp "bench-history/latest/$machine/benchmark.json" bench/compare/baseline.json | |
| else | |
| cp bench/compare/current.json bench/compare/baseline.json | |
| echo "note=No prior baseline exists; this run establishes it." >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Compare with latest master benchmark | |
| run: | | |
| version="${{ steps.benchmark-result.outputs.version }}" | |
| machine="${{ steps.benchmark-result.outputs.machine }}" | |
| baseline_note="${{ steps.benchmark-history.outputs.note }}" | |
| if test -z "$baseline_note"; then | |
| baseline_note="Compared with the latest master result from the same CI machine. Changes below 5% are reported as noise." | |
| fi | |
| clickhouse local --queries-file bench/compare.sql > bench/compare/comment.md | |
| { | |
| echo "## RowBinary benchmark" | |
| echo | |
| echo "$baseline_note" | |
| echo | |
| cat bench/compare/comment.md | |
| echo | |
| echo "Commit: \`$version\` · Machine: \`$machine\` · Profiler: TProf · Full JSON results are attached to this workflow run." | |
| } > bench/compare/pr-comment.md | |
| cat bench/compare/pr-comment.md | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: row-binary-benchmark-${{ steps.benchmark-result.outputs.version }} | |
| path: ${{ steps.benchmark-result.outputs.path }} | |
| - name: Comment on pull request | |
| if: github.event_name == 'pull_request' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: gh pr comment "${{ github.event.pull_request.number }}" --edit-last --create-if-none --body-file bench/compare/pr-comment.md | |
| - name: Store master result | |
| if: github.event_name == 'push' || github.event_name == 'schedule' | |
| run: | | |
| result_path="${{ steps.benchmark-result.outputs.path }}" | |
| machine="${{ steps.benchmark-result.outputs.machine }}" | |
| version="${{ steps.benchmark-result.outputs.version }}" | |
| destination="data/machine=$machine/version=$version" | |
| mkdir -p "bench-history/$destination" "bench-history/latest/$machine" | |
| cp "$result_path" "bench-history/$destination/benchmark.json" | |
| cp "bench-history/$destination/benchmark.json" "bench-history/latest/$machine/benchmark.json" | |
| git -C bench-history add data latest | |
| git -C bench-history config user.name github-actions[bot] | |
| git -C bench-history config user.email 41898282+github-actions[bot]@users.noreply.github.com | |
| git -C bench-history commit -m "Benchmark $version" | |
| git -C bench-history push origin HEAD:benchmark-results |