Add benchmark suite and per-PR benchmark CI workflow (#185)

* Add pytest-benchmark suite and per-PR benchmark CI workflow

Adds a bench/ suite (element creation, component mount depth,
attribute update at depth, keyed/unkeyed v-for reconciliation, and
unkeyed list grow/shrink) using pytest-benchmark with DictRenderer and
synchronous event loop, mirroring the setup in the sibling observ repo.

The new benchmark workflow runs the suite twice on every PR - once
against the master version of collagraph/ and once against the PR
version - and fails when mean time regresses more than 5%. The CI test
job is scoped to the tests directory so the matrix does not execute
benchmarks.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Disable GC during timed list-benchmark calls to reduce variance

The allocation burst of list reconciliation triggers collection pauses
in some rounds, which inflated stddev on the grow benchmarks to ~40%
of the mean - far too noisy for the 5% regression gate in CI.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: berendkleinhaneveld

.github/workflows/benchmark.yml

@@ -0,0 +1,57 @@
+name: Benchmarks
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  benchmark:
+    name: Benchmarks
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+
+      - name: Install dependencies
+        run: uv sync
+
+      # On PRs: run benchmarks twice (PR code vs master code) and compare
+      - name: Run benchmarks
+        run: |
+          # Checkout master version of collagraph directory
+          git fetch origin master
+          git checkout origin/master -- collagraph/
+
+          # Run benchmarks with master code as baseline
+          uv run --no-sync pytest bench \
+              --benchmark-only \
+              --benchmark-save=master \
+              --benchmark-sort=mean || true
+
+          # Restore PR code
+          git checkout HEAD -- collagraph/
+
+          # Run benchmarks on PR code and compare
+          uv run --no-sync pytest bench \
+              --benchmark-only \
+              --benchmark-compare \
+              --benchmark-compare-fail=mean:5% \
+              --benchmark-save=branch \
+              --benchmark-sort=mean
+
+      - name: Upload benchmarks
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Benchmarks
+          path: .benchmarks/
+          include-hidden-files: true

.github/workflows/ci.yml

@@ -61,7 +61,7 @@ jobs:
       - name: Install dependencies
         run: uv sync --all-groups
       - name: Test
-        run: uv run pytest -v --cov=collagraph --cov-report=term-missing
+        run: uv run pytest tests -v --cov=collagraph --cov-report=term-missing
         env:
           QT_QPA_PLATFORM: offscreen
 

.gitignore

@@ -7,3 +7,4 @@ build
 *.spec
 uv.lock
 .venv
+.benchmarks

bench/conftest.py

@@ -0,0 +1,10 @@
+"""Reuse existing test fixtures/helpers for collagraph benchmarks."""
+
+from tests.conftest import (  # noqa: F401
+    CustomElement,
+    CustomElementRenderer,
+    TrackingRenderer,
+    cleanup,
+    parse_source,
+    process_events,
+)

bench/test_component_mount.py

@@ -0,0 +1,66 @@
+"""
+Benchmarks for mounting a chain of nested components.
+
+Each level is its own ComponentFragment, so this stresses per-component
+mount overhead: the weak()-wrapped ref/bind watchers set up in
+ComponentFragment.create(), and the root-element lookup at the end of
+ComponentFragment.mount() (self.first()).
+"""
+
+import pytest
+
+from collagraph import Collagraph, EventLoopType
+from collagraph.renderers import DictRenderer
+
+DEPTHS = [10, 50, 200]
+
+
+def _build_nested_component(parse_source, depth):
+    Leaf, _ = parse_source(
+        """
+        <leaf />
+
+        <script>
+        import collagraph as cg
+
+        class Leaf(cg.Component):
+            pass
+        </script>
+        """
+    )
+
+    prev = Leaf
+    prev_name = "Leaf"
+    for i in range(depth):
+        name = f"Wrapper{i}"
+        prev, _ = parse_source(
+            f"""
+            <div>
+              <{prev_name} />
+            </div>
+
+            <script>
+            import collagraph as cg
+
+            class {name}(cg.Component):
+                pass
+            </script>
+            """,
+            namespace={prev_name: prev},
+        )
+        prev_name = name
+
+    return prev
+
+
+@pytest.mark.timeout(timeout=0)
+@pytest.mark.benchmark(group="mount_nested_components")
+@pytest.mark.parametrize("depth", DEPTHS, ids=[str(d) for d in DEPTHS])
+def test_mount_nested_components(benchmark, parse_source, depth):
+    App = _build_nested_component(parse_source, depth)
+
+    def mount():
+        gui = Collagraph(renderer=DictRenderer(), event_loop_type=EventLoopType.SYNC)
+        gui.render(App, {"type": "root"})
+
+    benchmark(mount)

bench/test_creation.py

@@ -0,0 +1,71 @@
+"""
+Benchmarks for the cost of mounting a fragment tree from scratch.
+
+Mounting stresses Fragment.create()/mount(), and for elements with a
+dynamic bind, also the watcher-setup path (`_watch_bind`) which relies
+on the `weak()` decorator for every callback registration.
+"""
+
+import pytest
+from observ import reactive
+
+from collagraph import Collagraph, EventLoopType
+from collagraph.renderers import DictRenderer
+
+SIZES = [10, 100, 1_000]
+SIZE_IDS = ["10", "100", "1k"]
+
+
+@pytest.mark.timeout(timeout=0)
+@pytest.mark.benchmark(group="mount_plain_elements")
+@pytest.mark.parametrize("n", SIZES, ids=SIZE_IDS)
+def test_mount_plain_elements(benchmark, parse_source, n):
+    children = "\n".join("        <item />" for _ in range(n))
+    App, _ = parse_source(
+        f"""
+        <root>
+{children}
+        </root>
+
+        <script>
+        import collagraph as cg
+
+        class App(cg.Component):
+            pass
+        </script>
+        """
+    )
+
+    def mount():
+        gui = Collagraph(renderer=DictRenderer(), event_loop_type=EventLoopType.SYNC)
+        gui.render(App, {"type": "root"})
+
+    benchmark(mount)
+
+
+@pytest.mark.timeout(timeout=0)
+@pytest.mark.benchmark(group="mount_bound_elements")
+@pytest.mark.parametrize("n", SIZES, ids=SIZE_IDS)
+def test_mount_bound_elements(benchmark, parse_source, n):
+    children = "\n".join(f'        <item :value="v{i}" />' for i in range(n))
+    App, _ = parse_source(
+        f"""
+        <root>
+{children}
+        </root>
+
+        <script>
+        import collagraph as cg
+
+        class App(cg.Component):
+            pass
+        </script>
+        """
+    )
+    state = reactive({f"v{i}": i for i in range(n)})
+
+    def mount():
+        gui = Collagraph(renderer=DictRenderer(), event_loop_type=EventLoopType.SYNC)
+        gui.render(App, {"type": "root"}, state=state)
+
+    benchmark(mount)

bench/test_keyed_list.py

@@ -0,0 +1,81 @@
+"""
+Benchmarks for keyed v-for reconciliation.
+
+Stresses ListFragment's keyed reconciliation path: the LIS computation
+and Fragment.anchor() lookups used to reposition/insert fragments.
+
+Each round replaces the whole `items` list in one reactive assignment
+(the idiomatic way to reorder/replace a collection), rather than doing
+many individual in-place mutations, which would each trigger their own
+full reconciliation pass and dominate the measurement with unrelated
+per-mutation dependency-tracking cost.
+"""
+
+import gc
+import random
+
+import pytest
+from observ import reactive
+
+from collagraph import Collagraph, EventLoopType
+from collagraph.renderers import DictRenderer
+
+SIZES = [10, 100, 1_000]
+SIZE_IDS = ["10", "100", "1k"]
+PATTERNS = ["append", "prepend", "reverse", "shuffle"]
+
+
+def _new_items(items, pattern):
+    if pattern == "append":
+        return [*items, {"id": len(items), "text": "x"}]
+    if pattern == "prepend":
+        return [{"id": len(items), "text": "x"}, *items]
+    if pattern == "reverse":
+        return list(reversed(items))
+    if pattern == "shuffle":
+        shuffled = list(items)
+        random.Random(0).shuffle(shuffled)
+        return shuffled
+    raise ValueError(pattern)
+
+
+def _apply(gui, state, items):
+    # `gui` is unused but must be kept as a live argument: it holds the
+    # only strong reference to the mounted fragment tree, so passing it
+    # through keeps that tree alive for the duration of the timed call.
+    # GC is disabled during the timed call so that collection pauses
+    # (triggered by the allocation burst) don't dominate the variance.
+    gc.disable()
+    try:
+        state["items"] = items
+    finally:
+        gc.enable()
+
+
+@pytest.mark.timeout(timeout=0)
+@pytest.mark.benchmark(group="keyed_list_reconcile")
+@pytest.mark.parametrize("n", SIZES, ids=SIZE_IDS)
+@pytest.mark.parametrize("pattern", PATTERNS)
+def test_keyed_list_reconcile(benchmark, parse_source, pattern, n):
+    App, _ = parse_source(
+        """
+        <node v-for="item in items" :key="item['id']" :text="item['text']" />
+
+        <script>
+        import collagraph as cg
+
+        class App(cg.Component):
+            pass
+        </script>
+        """
+    )
+
+    def setup():
+        initial = [{"id": i, "text": str(i)} for i in range(n)]
+        state = reactive({"items": list(initial)})
+        gui = Collagraph(renderer=DictRenderer(), event_loop_type=EventLoopType.SYNC)
+        gui.render(App, {"type": "root"}, state=state)
+        new_items = _new_items(initial, pattern)
+        return (gui, state, new_items), {}
+
+    benchmark.pedantic(_apply, setup=setup, warmup_rounds=1, rounds=30)