benchmark.yml

name: benchmark

on:
  pull_request:
  push:
    branches: [master]
  schedule:
    - cron: "17 10 * * *"
  workflow_dispatch:

permissions:
  contents: write
  pull-requests: write

concurrency:
  group: benchmark-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  row-binary:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

      - id: beam
        uses: erlef/setup-beam@v1
        with:
          elixir-version: "1.19"
          otp-version: "28"

      - uses: actions/cache@v6
        with:
          path: |
            deps
            _build
          key: benchmark-${{ runner.os }}-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('mix.lock') }}
          restore-keys: benchmark-${{ runner.os }}-${{ steps.beam.outputs.elixir-version }}-

      - run: mix deps.get --only dev
      - run: mix compile --warnings-as-errors
      - run: mix benchmark

      - name: Locate benchmark result
        id: benchmark-result
        run: |
          result_path=$(cat bench/output/current-result-path)
          machine=$(jq --raw-output .machine "$result_path")
          version=$(jq --raw-output .benchmark_version "$result_path")
          echo "path=$result_path" >> "$GITHUB_OUTPUT"
          echo "machine=$machine" >> "$GITHUB_OUTPUT"
          echo "version=$version" >> "$GITHUB_OUTPUT"

      - name: Install ClickHouse Local
        env:
          DO_NOT_TRACK: "1"
        run: |
          curl --fail --silent --show-error https://clickhouse.com/cli | sh
          "$HOME/.local/bin/clickhousectl" local use latest
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

      - name: Load benchmark history
        id: benchmark-history
        run: |
          result_path="${{ steps.benchmark-result.outputs.path }}"
          machine="${{ steps.benchmark-result.outputs.machine }}"

          if git fetch origin benchmark-results; then
            git worktree add bench-history FETCH_HEAD
          else
            git worktree add --detach bench-history
            git -C bench-history switch --orphan benchmark-results
          fi

          mkdir -p bench/compare
          cp "$result_path" bench/compare/current.json

          if test -f "bench-history/latest/$machine/benchmark.json"; then
            cp "bench-history/latest/$machine/benchmark.json" bench/compare/baseline.json
          else
            cp bench/compare/current.json bench/compare/baseline.json
            echo "note=No prior baseline exists; this run establishes it." >> "$GITHUB_OUTPUT"
          fi

      - name: Compare with latest master benchmark
        run: |
          version="${{ steps.benchmark-result.outputs.version }}"
          machine="${{ steps.benchmark-result.outputs.machine }}"
          baseline_note="${{ steps.benchmark-history.outputs.note }}"
          if test -z "$baseline_note"; then
            baseline_note="Compared with the latest master result from the same CI machine. Changes below 5% are reported as noise."
          fi
          clickhouse local --queries-file bench/compare.sql > bench/compare/comment.md
          {
            echo "## RowBinary benchmark"
            echo
            echo "$baseline_note"
            echo
            cat bench/compare/comment.md
            echo
            echo "Commit: \`$version\` · Machine: \`$machine\` · Profiler: TProf · Full JSON results are attached to this workflow run."
          } > bench/compare/pr-comment.md
          cat bench/compare/pr-comment.md

      - uses: actions/upload-artifact@v7
        with:
          name: row-binary-benchmark-${{ steps.benchmark-result.outputs.version }}
          path: ${{ steps.benchmark-result.outputs.path }}

      - name: Comment on pull request
        if: github.event_name == 'pull_request'
        env:
          GH_TOKEN: ${{ github.token }}
        run: gh pr comment "${{ github.event.pull_request.number }}" --edit-last --create-if-none --body-file bench/compare/pr-comment.md

      - name: Store master result
        if: github.event_name == 'push' || github.event_name == 'schedule'
        run: |
          result_path="${{ steps.benchmark-result.outputs.path }}"
          machine="${{ steps.benchmark-result.outputs.machine }}"
          version="${{ steps.benchmark-result.outputs.version }}"
          destination="data/machine=$machine/version=$version"
          mkdir -p "bench-history/$destination" "bench-history/latest/$machine"
          cp "$result_path" "bench-history/$destination/benchmark.json"
          cp "bench-history/$destination/benchmark.json" "bench-history/latest/$machine/benchmark.json"
          git -C bench-history add data latest
          git -C bench-history config user.name github-actions[bot]
          git -C bench-history config user.email 41898282+github-actions[bot]@users.noreply.github.com
          git -C bench-history commit -m "Benchmark $version"
          git -C bench-history push origin HEAD:benchmark-results