test: Benchmarks (#160)

# Summary
Adds a simple benchmark to the test suite.

# Changes
* Adds benchmark test on a simple 3-node model.
* Reports benchmark results on each new PR, so that the code changes can
be compared with main branch before merging.

Author: toby-coleman

.github/workflows/benchmarks.yaml

@@ -0,0 +1,80 @@
+name: Benchmarks
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+
+jobs:
+  lint:
+    name: Benchmark tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python_version: [3.12]
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v4
+        with:
+          path: pr
+
+      - name: Checkout main
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          path: main
+
+      - name: Install python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{matrix.python_version}}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+          cache-dependency-glob: "main/uv.lock"
+
+      - name: Setup benchmarks
+        run: |
+          echo "BASE_SHA=$(echo ${{ github.event.pull_request.base.sha }} | cut -c1-8)" >> $GITHUB_ENV
+          echo "HEAD_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-8)" >> $GITHUB_ENV
+          echo "PR_COMMENT=$(mktemp)" >> $GITHUB_ENV
+
+      - name: Run benchmarks on PR
+        working-directory: ./pr
+        run: |
+          uv sync --group test
+          uv run pytest --benchmark-only --benchmark-save=pr
+
+      - name: Run benchmarks on main
+        working-directory: ./main
+        continue-on-error: true
+        run: |
+          uv sync --group test
+          uv run pytest --benchmark-only --benchmark-save=base
+
+      - name: Compare results
+        continue-on-error: false
+        run: |
+          uvx pytest-benchmark compare **/.benchmarks/**/*.json | tee cmp_results
+
+          echo 'Benchmark comparison for [`${{ env.BASE_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.base.sha }}) (base) vs [`${{ env.HEAD_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.head.sha }}) (PR)' >> pr_comment
+          echo '```' >> pr_comment
+          cat cmp_results >> pr_comment
+          echo '```' >> pr_comment
+          cat pr_comment > ${{ env.PR_COMMENT }}
+      
+      - name: Comment on PR
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: require('fs').readFileSync('${{ env.PR_COMMENT }}').toString()
+            });
+

.gitignore

@@ -54,6 +54,9 @@ coverage.json
 .pytest_cache/
 cover/
 
+# Benchmarking
+.benchmarks/
+
 # Translations
 *.mo
 *.pot

pyproject.toml

@@ -75,6 +75,7 @@ test = [
     "optuna>=3.0,<4",
     "pytest>=8.3,<9",
     "pytest-asyncio>=1.0,<2",
+    "pytest-benchmark>=5.1.0",
     "pytest-cases>=3.8,<4",
     "pytest-env>=1.1,<2",
     "pytest-rerunfailures>=15.0,<16",

tests/benchmark/__init__.py

@@ -0,0 +1 @@
+"""Provides benchmark tests for Plugboard."""

tests/benchmark/test_benchmarking.py

@@ -0,0 +1,35 @@
+"""Simple benchmark tests for Plugboard models."""
+
+import asyncio
+
+from pytest_benchmark.fixture import BenchmarkFixture
+
+from plugboard.connector import AsyncioConnector
+from plugboard.process import LocalProcess, Process
+from plugboard.schemas import ConnectorSpec
+from tests.integration.test_process_with_components_run import A, B
+
+
+def _setup_process() -> tuple[tuple[Process], dict]:
+    comp_a = A(name="comp_a", iters=1000)
+    comp_b1 = B(name="comp_b1", factor=1)
+    comp_b2 = B(name="comp_b2", factor=2)
+    components = [comp_a, comp_b1, comp_b2]
+    connectors = [
+        AsyncioConnector(spec=ConnectorSpec(source="comp_a.out_1", target="comp_b1.in_1")),
+        AsyncioConnector(spec=ConnectorSpec(source="comp_b1.out_1", target="comp_b2.in_1")),
+    ]
+    process = LocalProcess(components=components, connectors=connectors)
+    # Initialise process so that this is excluded from the benchmark timing
+    asyncio.run(process.init())
+    # Return args and kwargs tuple for benchmark.pedantic
+    return (process,), {}
+
+
+def _run_process(process: Process) -> None:
+    asyncio.run(process.run())
+
+
+def test_benchmark_process_run(benchmark: BenchmarkFixture) -> None:
+    """Benchmark the running of a Plugboard Process."""
+    benchmark.pedantic(_run_process, setup=_setup_process, rounds=5)