Re-implement benchmarks in pytest #7
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks (PR) | |
| on: | |
| pull_request: | |
| permissions: {} | |
| env: | |
| PYBAMM_DISABLE_TELEMETRY: "true" | |
| FORCE_COLOR: 3 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: read | |
| contents: read | |
| outputs: | |
| solver: ${{ steps.filter.outputs.solver }} | |
| pybamm: ${{ steps.filter.outputs.pybamm }} | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 | |
| id: filter | |
| with: | |
| filters: | | |
| solver: | |
| - 'packages/pybammsolvers/**' | |
| - '.github/workflows/_build_solver_wheels.yml' | |
| pybamm: | |
| - 'packages/pybamm/**' | |
| - 'pyproject.toml' | |
| - 'uv.lock' | |
| - 'noxfile.py' | |
| - 'conftest.py' | |
| - '.github/workflows/benchmarks_pr.yml' | |
| # Build solver wheel from the PR's base branch (main). | |
| # Uses the same cache key as _build_solver_wheels.yml, so this is a free | |
| # cache hit on pybamm-only PRs where main's solver hasn't changed. | |
| build_solver_main: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.pybamm == 'true' || needs.changes.outputs.solver == 'true' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| ref: ${{ github.base_ref }} | |
| submodules: "recursive" | |
| persist-credentials: false | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.13" | |
| - name: Compute solver source hash | |
| id: solverhash | |
| run: echo "hash=$(git rev-parse HEAD:packages/pybammsolvers)" >> "$GITHUB_OUTPUT" | |
| - name: Restore cached solver wheels (main) | |
| id: cache | |
| uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 | |
| with: | |
| path: packages/pybammsolvers/wheelhouse | |
| key: solver-wheels-linux-${{ steps.solverhash.outputs.hash }} | |
| - name: Build solver wheels for main | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| working-directory: packages/pybammsolvers | |
| env: | |
| CIBW_BUILD_VERBOSITY: 2 | |
| CIBW_SKIP: "pp* *musllinux* cp314t*" | |
| CIBW_ARCHS_LINUX: x86_64 | |
| CIBW_BEFORE_ALL_LINUX: > | |
| yum -y --disablerepo=epel install openblas-devel lapack-devel && | |
| python install_KLU_Sundials.py | |
| CIBW_REPAIR_WHEEL_COMMAND_LINUX: > | |
| auditwheel repair --exclude libcasadi.so.3.7 -w {dest_dir} {wheel} | |
| run: pipx run cibuildwheel --output-dir wheelhouse | |
| - name: Upload main solver wheels | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: wheels_manylinux_main | |
| path: packages/pybammsolvers/wheelhouse/*.whl | |
| if-no-files-found: error | |
| # Build solver wheel from the PR HEAD. | |
| build_solver_pr: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.pybamm == 'true' || needs.changes.outputs.solver == 'true' }} | |
| permissions: | |
| contents: read | |
| uses: ./.github/workflows/_build_solver_wheels.yml | |
| with: | |
| cache_wheels: true | |
| build_sdist: false | |
| platforms: '["linux"]' | |
| macos_runners: '[]' | |
| benchmark_speed: | |
| needs: [changes, build_solver_main, build_solver_pr] | |
| if: ${{ needs.changes.outputs.pybamm == 'true' || needs.changes.outputs.solver == 'true' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| name: Speed benchmarks (vs main) | |
| steps: | |
| - name: Check out PyBaMM repository | |
| uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: Install Linux system dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 | |
| with: | |
| packages: gfortran gcc | |
| execute_install_scripts: true | |
| - name: Install OpenBLAS for Linux | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install libopenblas-dev | |
| - name: Set up Python 3.13 | |
| id: setup-python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.13" | |
| - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 | |
| with: | |
| version: "latest" | |
| enable-cache: true | |
| - name: Install nox | |
| run: uv tool install nox | |
| - name: Download main solver wheels | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: wheels_manylinux_main | |
| path: solver-wheels-main | |
| - name: Download PR solver wheels | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: wheels_manylinux | |
| path: solver-wheels-pr | |
| - name: Run baseline benchmarks at main HEAD | |
| continue-on-error: true | |
| env: | |
| PYBAMM_SOLVER_WHEELS: ${{ github.workspace }}/solver-wheels-main | |
| BASE_REF: ${{ github.base_ref }} | |
| run: | | |
| git fetch origin "$BASE_REF" | |
| git checkout "origin/$BASE_REF" -- packages/pybamm/ uv.lock pyproject.toml | |
| nox -s benchmark-speed -- --benchmark-save=main-baseline | |
| - name: Run PR benchmarks and compare against main | |
| env: | |
| PYBAMM_SOLVER_WHEELS: ${{ github.workspace }}/solver-wheels-pr | |
| # Only compare if the baseline step saved results (it won't on the first PR | |
| # that introduces benchmarks, since they don't exist on main yet). | |
| run: | | |
| git checkout HEAD -- packages/pybamm/ uv.lock pyproject.toml | |
| if ls .benchmarks/*/main-baseline*.json 1>/dev/null 2>&1; then | |
| nox -s benchmark-speed -- \ | |
| --benchmark-compare=main-baseline \ | |
| --benchmark-compare-fail=mean:125% | |
| else | |
| nox -s benchmark-speed | |
| fi | |
| benchmark_memory: | |
| needs: [changes, build_solver_pr] | |
| if: ${{ needs.changes.outputs.pybamm == 'true' || needs.changes.outputs.solver == 'true' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| name: Memory benchmarks (ubuntu-latest / Python 3.13) | |
| env: | |
| PYBAMM_SOLVER_WHEELS: ${{ github.workspace }}/solver-wheels | |
| steps: | |
| - name: Check out PyBaMM repository | |
| uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| - name: Install Linux system dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 | |
| with: | |
| packages: gfortran gcc graphviz | |
| execute_install_scripts: true | |
| - name: Install OpenBLAS for Linux | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install libopenblas-dev | |
| - name: Set up Python 3.13 | |
| id: setup-python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.13" | |
| - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 | |
| with: | |
| version: "latest" | |
| enable-cache: true | |
| - name: Install nox | |
| run: uv tool install nox | |
| - name: Download PR solver wheels | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: wheels_manylinux | |
| path: solver-wheels | |
| - name: Run memory benchmark tests for GNU/Linux with Python 3.13 | |
| run: nox -s benchmark-memory |