run_benchmarks_over_history.yml

# GitHub actions workflow that runs the benchmark suite in benchmarks/
# from "commit_start" to "commit_end".  It pushes the results to the
# pybamm-bench repo and updates the display website.

# This workflow is meant to be triggered manually, see
# https://docs.github.com/en/enterprise-server@3.0/actions/managing-workflow-runs/manually-running-a-workflow

name: Manual benchmarks
on:
  workflow_dispatch:
    inputs:
      commit_start:
        description: "Identifier of commit from which to start"
        default: "v0.1.0"
        type: string
        pattern: '^[a-zA-Z0-9._-]+$'
      commit_end:
        description: "Identifier of commit at which to end"
        default: "main"
        type: string
        pattern: '^[a-zA-Z0-9._-]+$'
      ncommits:
        description: "Number of commits to benchmark between commit_start and commit_end"
        default: "100"
        type: string
        pattern: '^[0-9]+$'


permissions: {}

env:
  PYBAMM_DISABLE_TELEMETRY: "true"

jobs:
  benchmarks:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false
      - name: Set up Python 3.14
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: 3.14

      - name: Install nox and asv
        run: pip install -U pip nox asv

      - name: Fetch main branch
        # Not required when workflow triggered
        # on main, but useful when
        # experimenting/developing on another branch.
        if: github.ref != 'refs/heads/main'
        run: |
          git fetch origin main:main

      - name: Validate commit_start
        id: validate_start
        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
        with:
          script: |
            const input = context.payload.inputs.commit_start;
            if (!input || !/^[a-zA-Z0-9._-]+$/.test(input)) {
              core.setFailed('Invalid commit_start format');
              return;
            }
            core.setOutput('commit_start', input);

      - name: Validate commit_end
        id: validate_end
        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
        with:
          script: |
            const input = context.payload.inputs.commit_end;
            if (!input || !/^[a-zA-Z0-9._-]+$/.test(input)) {
              core.setFailed('Invalid commit_end format');
              return;
            }
            core.setOutput('commit_end', input);

      - name: Validate ncommits
        id: validate_ncommits
        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
        with:
          script: |
            const input = context.payload.inputs.ncommits;
            if (!input || !/^[0-9]+$/.test(input)) {
              core.setFailed('Invalid ncommits format');
              return;
            }
            const numValue = parseInt(input, 10);
            if (numValue < 1 || numValue > 10000) {
              core.setFailed('ncommits must be between 1 and 10000');
              return;
            }
            if (numValue > 5000) {
              core.warning('Processing a large number of commits. This may take a while....');
            }
            core.setOutput('ncommits', numValue.toString());

      - name: Set environment variables
        env:
          COMMIT_START: ${{ steps.validate_start.outputs.commit_start }}
          COMMIT_END: ${{ steps.validate_end.outputs.commit_end }}
          NCOMMITS: ${{ steps.validate_ncommits.outputs.ncommits }}
        run: |
          echo "COMMIT_START=$COMMIT_START" >> $GITHUB_ENV
          echo "COMMIT_END=$COMMIT_END" >> $GITHUB_ENV
          echo "NCOMMITS=$NCOMMITS" >> $GITHUB_ENV

      - name: Run benchmarks
        run: |
          asv machine --machine "GitHubRunner"
          asv run -m "GitHubRunner" -s $NCOMMITS \
          $COMMIT_START..$COMMIT_END

      - name: Upload results as artifact
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: asv_over_history_results
          path: results
          if-no-files-found: error

  publish-results:
    if: github.repository == 'pybamm-team/PyBaMM'
    name: Push and publish results
    needs: benchmarks
    runs-on: ubuntu-latest
    environment: benchmarks
    permissions:
      contents: write
    steps:
      - name: Set up Python 3.14
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: 3.14

      - name: Install asv
        run: pip install asv

      - name: Checkout pybamm-bench repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: pybamm-team/pybamm-bench
          token: ${{ secrets.BENCH_PAT }}
          persist-credentials: false

      - name: Download results artifact(s)
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v4.6.2
        with:
          name: asv_over_history_results
          path: results

      - name: Copy new results and push to pybamm-bench repo
        env:
          PUSH_BENCH_EMAIL: ${{ secrets.PUSH_BENCH_EMAIL }}
          PUSH_BENCH_NAME: ${{ secrets.PUSH_BENCH_NAME }}
        run: |
          git config --global user.email "$PUSH_BENCH_EMAIL"
          git config --global user.name "$PUSH_BENCH_NAME"
          git add results
          git commit -am "Add new results"
          git push

      - name: Publish results
        run: |
          git fetch origin gh-pages:gh-pages
          asv gh-pages