bench.yml

name: benchmark

on:
  push:
    branches: [main]
    paths:
      - "bin/katana/**"
      - "crates/**"
      - "!**/README.md"

  pull_request:
    types: [opened, synchronize, ready_for_review]
    paths:
      - "bin/katana/**"
      - "crates/**"
      - ".github/workflows/bench.yml"
      - "!**/README.md"

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

permissions:
  deployments: write
  contents: write
  pull-requests: write

jobs:
  detect-changes:
    runs-on: ubuntu-latest
    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
    outputs:
      broader-rust: ${{ steps.filter.outputs.broader-rust }}
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          # Benchmarks don't exercise the explorer UI, so skip when the only
          # changes are in crates/explorer or the submodule pointer.
          filters: |
            broader-rust:
              - 'bin/katana/**'
              - 'crates/**'
              - '!crates/explorer/**'
              - '.github/workflows/bench.yml'

  generate-artifacts:
    needs: [detect-changes]
    runs-on: ubuntu-latest
    if: |
      needs.detect-changes.outputs.broader-rust == 'true' &&
      (github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false))
    container:
      image: ghcr.io/dojoengine/katana-dev:latest
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive
      - run: git config --global --add safe.directory "*"

      - name: Restore cached artifacts
        id: cache
        uses: actions/cache@v4
        with:
          path: crates/contracts/build
          key: bench-contracts-${{ hashFiles('Makefile', 'crates/contracts/build.rs', 'crates/contracts/contracts/account/**', 'crates/contracts/contracts/legacy/**', 'crates/contracts/contracts/messaging/**', 'crates/contracts/contracts/test-contracts/**', 'crates/contracts/contracts/vrf/**', 'crates/contracts/contracts/avnu/**', 'crates/contracts/contracts/openzeppelin/**', 'crates/contracts/contracts/piltover/**', 'crates/contracts/contracts/Scarb.toml') }}

      - name: Build contracts
        if: steps.cache.outputs.cache-hit != 'true'
        run: make contracts

      - name: Upload contract artifacts
        uses: actions/upload-artifact@v4
        with:
          name: contracts
          overwrite: true
          retention-days: 14
          if-no-files-found: error
          path: crates/contracts/build

  bench-codec:
    needs: [detect-changes, generate-artifacts]
    runs-on: ubuntu-latest
    if: |
      needs.detect-changes.outputs.broader-rust == 'true' &&
      (github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false))
    container:
      image: ghcr.io/dojoengine/katana-dev:latest
    steps:
      - uses: actions/checkout@v3
      - run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
      - uses: Swatinem/rust-cache@v2
        with:
          key: bench-codec

      - name: Download contract artifacts
        uses: actions/download-artifact@v4
        with:
          name: contracts
          path: crates/contracts/build

      - name: Collect hardware specs
        run: |
          echo "cpu_model=$(lscpu | awk -F: '/Model name/ {gsub(/^[ \t]+/, "", $2); print $2}')" >> hw-specs.txt
          echo "cpu_cores=$(nproc)" >> hw-specs.txt
          echo "ram_total=$(free -h | awk '/Mem:/ {print $2}')" >> hw-specs.txt

      - name: Run codec benchmarks
        run: |
          cargo bench -p katana-db --features arbitrary --bench codec -- --output-format bencher 2>&1 | tee raw.txt
          grep '^test ' raw.txt > output.txt || true
          echo "::group::Codec benchmark output"
          cat output.txt
          echo "::endgroup::"
          if [ ! -s output.txt ]; then
            echo "::error::Codec benchmark produced no results. Raw output:"
            cat raw.txt
            exit 1
          fi

      - uses: actions/upload-artifact@v4
        with:
          name: bench-codec
          path: |
            output.txt
            hw-specs.txt

  # bench-commit:
  #   runs-on: ubuntu-latest
  #   if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
  #   container:
  #     image: ghcr.io/dojoengine/katana-dev:latest
  #   steps:
  #     - uses: actions/checkout@v3
  #     - run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
  #     - uses: Swatinem/rust-cache@v2
  #       with:
  #         key: bench-commit
  #
  #     - name: Run commit benchmarks
  #       run: cargo bench -p katana-core --bench commit -- --output-format bencher | tee output.txt
  #
  #     - uses: actions/upload-artifact@v4
  #       with:
  #         name: bench-commit
  #         path: output.txt

  # bench-startup:
  #   needs: [generate-artifacts]
  #   runs-on: ubuntu-latest
  #   if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
  #   container:
  #     image: ghcr.io/dojoengine/katana-dev:latest
  #   steps:
  #     - uses: actions/checkout@v3
  #     - run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
  #     - uses: Swatinem/rust-cache@v2
  #       with:
  #         key: bench-startup
  #
  #     - name: Download contract artifacts
  #       uses: actions/download-artifact@v4
  #       with:
  #         name: contracts
  #         path: crates/contracts/build
  #
  #     - name: Run startup benchmarks
  #       run: cargo bench -p katana-node-bindings --bench startup -- --output-format bencher | tee output.txt
  #
  #     - uses: actions/upload-artifact@v4
  #       with:
  #         name: bench-startup
  #         path: output.txt

  report:
    needs: [detect-changes, bench-codec]
    runs-on: ubuntu-latest
    if: |
      needs.detect-changes.outputs.broader-rust == 'true' &&
      (github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false))
    steps:
      - uses: actions/checkout@v3

      - uses: actions/download-artifact@v4
        with:
          name: bench-codec
          path: bench-results/bench-codec

      - name: Merge benchmark results
        run: |
          cat bench-results/bench-codec/output.txt > output.txt
          echo "::group::Merged benchmark output"
          cat output.txt
          echo "::endgroup::"
          if [ ! -s output.txt ]; then
            echo "::error::Benchmark output is empty"
            exit 1
          fi

      - name: Parse hardware specs
        id: hw
        run: |
          if [ -f bench-results/bench-codec/hw-specs.txt ]; then
            cpu_model=$(grep '^cpu_model=' bench-results/bench-codec/hw-specs.txt | cut -d= -f2-)
            cpu_cores=$(grep '^cpu_cores=' bench-results/bench-codec/hw-specs.txt | cut -d= -f2-)
            ram_total=$(grep '^ram_total=' bench-results/bench-codec/hw-specs.txt | cut -d= -f2-)
            echo "summary=**Runner:** ${cpu_model} (${cpu_cores} cores) · ${ram_total} RAM" >> "$GITHUB_OUTPUT"
          else
            echo "summary=Hardware specs not available" >> "$GITHUB_OUTPUT"
          fi

      # On push to main: update GitHub Pages with historical benchmark data
      - name: Store benchmark result (main)
        if: github.event_name == 'push'
        uses: benchmark-action/github-action-benchmark@v1
        with:
          tool: "cargo"
          output-file-path: output.txt
          benchmark-data-dir-path: "."
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          alert-threshold: "130%"
          comment-on-alert: true
          alert-comment-cc-users: "@kariy"

      # On PR: compare against main baseline and alert on regressions
      - name: Compare benchmark result (PR)
        if: github.event_name == 'pull_request'
        uses: benchmark-action/github-action-benchmark@v1
        with:
          tool: "cargo"
          output-file-path: output.txt
          benchmark-data-dir-path: "."
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: false
          alert-threshold: "130%"
          comment-on-alert: true
          comment-always: false
          alert-comment-cc-users: "@kariy"

      # On PR: post a percentage-diff comment against the main baseline
      - name: Fetch baseline benchmark data
        if: github.event_name == 'pull_request'
        run: |
          git fetch origin gh-pages --depth=1 || true
          git show origin/gh-pages:data.js > baseline.js 2>/dev/null || echo "window.BENCHMARK_DATA = { entries: {} };" > baseline.js

      - name: Build percentage diff comment
        if: github.event_name == 'pull_request'
        run: node scripts/bench-percent-diff.js baseline.js output.txt diff.md

      - name: Post percentage diff comment
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const body = fs.readFileSync('diff.md', 'utf8');
            const marker = '<!-- benchmark-percent-diff -->';
            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });
            const existing = comments.find(c => c.body.includes(marker));
            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body,
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body,
              });
            }

      - name: Post hardware specs comment
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const marker = '<!-- benchmark-hw-specs -->';
            const body = `${marker}\n> ${{ steps.hw.outputs.summary }}`;
            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });
            const existing = comments.find(c => c.body.includes(marker));
            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: existing.id,
                body,
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: context.issue.number,
                body,
              });
            }