asv.yaml

name: asv

on:
  workflow_dispatch:
  schedule:
    - cron: 0 4 * * *

concurrency:
  group: "pages"
  cancel-in-progress: false

env:
  # NOTE: For the first clean run, this should be a high-ish (~20) number to
  #       populate the history. For all subsequent runs where data is cached in
  #       `.asv/results/github-actions/`, it can be a lower number for better
  #       runtimes in CI
  NUM_BENCHMARK_COMMITS: 5
  BRANCH: main

jobs:
  asv:
    runs-on: ubuntu-latest
    permissions:
      contents: write
      pages: write
      id-token: write
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    steps:
      - uses: actions/checkout@v7
        # NOTE: This only grabs the HEAD commit of the main branch, so we can't
        #       use it for retrospective benchmarking. Instead, we manually
        #       pull the submodule in the next step
        # with:
        #   submodules: 'true'
      - name: Manually pull submodules
        run: |
          git config --global url."https://github.com/".insteadOf "git@github.com:" &&\
              git submodule update --init &&\
              cd xdsl &&\
              git fetch --all &&\
              git branch &&\
              git checkout $BRANCH &&\
              git pull

      - uses: astral-sh/setup-uv@v8.3.2
        with:
            enable-cache: true
            cache-dependency-glob: "uv.lock"
      # Use `actions/setup-python@v5` rather than `uv python install 3.x` to
      # ensure that it is correctly in the $PATH for `asv`
      - uses: actions/setup-python@v6
        with:
          python-version: |
              3.11
              pypy3.11
              3.13
      - name: Check Python installations
        run: |
          which python3.11 && python3.11 --version
          which pypy.11 && pypy3.11 --version
          which python3.13 && python3.13 --version
          uv python list

      - name: Include existing artefacts
        run: rm -rf xdsl/.asv && mv .asv xdsl/.asv

      - name: Install the project
        working-directory: ./xdsl
        run: uv sync --group bench

      - name: Fingerprint the machine
        working-directory: ./xdsl
        run: |
          echo "OS: `uname -sr`"
          echo "Architecture: `uname -m`"
          echo "CPU: `lscpu | grep 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1'`"
          echo "Cores: `nproc --all`"
          echo "RAM: `grep MemTotal /proc/meminfo | awk '{print $2}'`"

          uv run asv machine -v --yes \
            --machine github-action \
            --os "`uname -sr`" \
            --arch "`uname -m`" \
            --cpu "`lscpu | grep 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1'`" \
            --num_cpu `nproc --all` \
            --ram `grep MemTotal /proc/meminfo | awk '{print $2}'`

      - name: Run the ASV benchmarks
        id: asv_run
        continue-on-error: true
        working-directory: ./xdsl
        run: |
          mkdir -p ../asv-logs
          set -o pipefail
          uv run asv run $BRANCH~$NUM_BENCHMARK_COMMITS..$BRANCH 2>&1 | tee ../asv-logs/asv-run.log

      - name: Rerun failed benchmarks with stderr
        if: steps.asv_run.outcome == 'failure'
        working-directory: ./xdsl
        run: |
          python - <<'PY'
          import pathlib
          import re

          log_path = pathlib.Path("../asv-logs/asv-run.log")
          bench_path = pathlib.Path("../asv-logs/asv-failed-benches.txt")
          failed = []
          for line in log_path.read_text().splitlines():
              match = re.search(r"([A-Za-z0-9_.]+)\s+failed$", line)
              if match:
                  failed.append(match.group(1))
          unique_failed = sorted(set(failed))
          bench_path.write_text("\n".join(unique_failed) + ("\n" if unique_failed else ""))
          print(f"Found {len(unique_failed)} failed benchmarks.")
          PY

          if [ -s ../asv-logs/asv-failed-benches.txt ]; then
            while IFS= read -r bench; do
              echo "=== Rerun for $bench ===" | tee -a ../asv-logs/asv-rerun.log
              uv run asv run "$BRANCH~$NUM_BENCHMARK_COMMITS..$BRANCH" \
                --bench "$bench" \
                --quick \
                --show-stderr \
                --dry-run \
                -v 2>&1 | tee -a ../asv-logs/asv-rerun.log
            done < ../asv-logs/asv-failed-benches.txt
          else
            echo "No failed benchmark names were parsed from asv-run.log" | tee -a ../asv-logs/asv-rerun.log
          fi

      - name: Upload ASV debug logs
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
        with:
          name: asv-debug-logs
          path: asv-logs/

      - name: Fail job if ASV run failed
        if: steps.asv_run.outcome == 'failure'
        run: |
          echo "ASV benchmark run failed. See the asv-debug-logs artifact." >&2
          exit 1

      - name: Generate the ASV website
        working-directory: ./xdsl
        run: uv run asv publish

      - name: Copy back generated artefacts
        run: mv xdsl/.asv .asv

      - name: Commit the generated benchmark results
        uses: stefanzweifel/git-auto-commit-action@v7
        with:
          commit_message: "Add CI benchmark runs"
          # CI commits should not be attributed to action author, to avoid extraneous activity
          commit_author: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>"

      - name: Setup Pages
        uses: actions/configure-pages@v6

      - name: Upload artifact
        uses: actions/upload-pages-artifact@v5
        with:
          path: .asv/html/

      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v5

  notify-on-failure:
    needs: asv
    if: ${{ failure() }}
    runs-on: ubuntu-latest
    steps:
      - name: Send Zulip notification
        uses: zulip/github-actions-zulip/send-message@v2
        with:
          api-key: ${{ secrets.ZULIP_FAILURE_BOT_API_KEY }}
          email: ${{ secrets.ZULIP_FAILURE_BOT_EMAIL }}
          organization-url: "https://xdsl.zulipchat.com"
          to: "Framework"
          type: "stream"
          topic: "xdsl-bench Failure"
          content: |
            ❗ Workflow [${{ github.workflow }}](${{
              github.server_url }}/${{ github.repository }}/actions/runs/${{
              github.run_id }}) failed!