feat(benchmark): add cache behavior suite

.github/workflows/cache-benchmark.yml

@@ -0,0 +1,51 @@
+name: Cache benchmark validation
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/cache-benchmark.yml
+      - benchmarks/cache/**
+      - crates/**
+      - Cargo.*
+  push:
+    branches:
+      - main
+    paths:
+      - .github/workflows/cache-benchmark.yml
+      - benchmarks/cache/**
+      - crates/**
+      - Cargo.*
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  cache-benchmark-validation:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+      - name: Generate workload
+        run: python3 benchmarks/cache/scripts/generate_workload.py --self-check
+      - name: Validate workload
+        run: python3 benchmarks/cache/scripts/validate_workload.py benchmarks/cache/workloads/default
+      - name: Build dlin
+        run: cargo build -p dlin --locked
+      - name: Validate cache behavior
+        run: >
+          python3 benchmarks/cache/scripts/run_benchmarks.py
+          --workload benchmarks/cache/workloads/default
+          --binary target/debug/dlin
+          --results-dir benchmarks/cache/results/ci
+          --summary-file "$GITHUB_STEP_SUMMARY"
+          --skip-timing

benchmarks/cache/.gitignore

@@ -0,0 +1,4 @@
+workloads/
+results/
+__pycache__/
+*.py[cod]

benchmarks/cache/README.md

@@ -0,0 +1,118 @@
+# dlin cache benchmark
+
+This suite measures dlin's three persistent caches without changing production
+code or comparing dlin with another tool. It is intentionally separate from
+`benchmarks/column-lineage`, whose responsibility is external-tool correctness
+and comparison.
+
+## Setup
+
+From the repository root, build the release binary and install hyperfine:
+
+```sh
+cargo build --release --locked
+cargo install hyperfine --version 1.19.0 --locked
+```
+
+Python standard library is sufficient; if an isolated environment is desired,
+use `uv` (do not install packages with `pip`). Generate and validate a fixture:
+
+```sh
+python3 benchmarks/cache/scripts/generate_workload.py \
+  --output benchmarks/cache/workloads/default --profile small --self-check
+python3 benchmarks/cache/scripts/validate_workload.py \
+  benchmarks/cache/workloads/default
+```
+
+The `medium` profile is an explicit larger workload:
+
+```sh
+python3 benchmarks/cache/scripts/generate_workload.py \
+  --output benchmarks/cache/workloads/medium --profile medium --self-check
+```
+
+The small profile has 64 models and medium has 512. Generation refuses to
+overwrite a non-empty directory. To replace a workload previously generated
+by this suite, pass `--force`; the marker file `workload_metadata.json` is
+required as a safety check:
+
+```sh
+python3 benchmarks/cache/scripts/generate_workload.py \
+  --output benchmarks/cache/workloads/default --profile small --self-check --force
+```
+
+## Benchmark and validation
+
+Run probes and hyperfine timings with the release binary (the default binary
+path is `target/release/dlin`):
+
+```sh
+python3 benchmarks/cache/scripts/run_benchmarks.py \
+  --workload benchmarks/cache/workloads/default \
+  --binary target/release/dlin --runs 3 --warmup 1
+```
+
+Pass `--summary-file <PATH>` to write the concise semantic-validation report
+as Markdown (the runner also prints the same report to its log). CI passes
+`$GITHUB_STEP_SUMMARY` so the probe results are visible in the job summary;
+timing values are intentionally not included there.
+
+Use `--skip-timing` for a fast probe-only check, or pass a debug binary
+explicitly for functional development checks. Results are written under the
+ignored `benchmarks/cache/results/` directory:
+
+* `run_metadata.json` records git HEAD, binary version and SHA-256, platform,
+  input sizes, commands, cache metadata, probe status, and timing paths.
+* `hyperfine/*.json` contains raw timing output.
+* `cache/<scenario>/` contains the observed persistent cache files.
+* `invalidation/` contains isolated SQL-project copies and functional
+  invalidation metadata.
+
+Each scenario uses its own cache directory and observes
+`extraction_cache.json`, `manifest_graph_cache.json`, and
+`column_lineage_cache.json` as applicable. The runner also verifies dlin's
+generated cache-directory `.gitignore` content. `persistent-cold` means a forced
+miss using `--refresh-cache`; `persistent-warm` means reuse after a preparation
+run. These labels distinguish persistent cache state only: the OS/filesystem
+cache is not flushed. `--no-cache` is the no-persistent-I/O probe.
+
+The runner checks semantic JSON equivalence between no-cache, persistent-cold,
+and persistent-warm output. It also records cache size, SHA-256, and mtime in
+nanoseconds and asserts that observed cache files are unchanged by the warm
+probe and timed warm runs. This is combined benchmark evidence, not direct
+proof of an internal cache hit; direct hit guarantees belong in production
+unit/integration tests. Timing is informational and has no pass/fail threshold.
+
+For SQL and manifest scenarios, timing uses the small-output `summary -o json`
+command while semantic probes use `graph -o json`. This keeps graph rendering
+out of cache timings while comparing the observable DAG. The column scenario
+uses the same compiled-SQL column query for both.
+
+The runner also performs three SQL invalidation baselines on isolated copies:
+a size-changing single-file ref edit, a macro body edit that adds a rendered
+dependency, and a `vars.yml` edit that changes the final model ref. Each must
+produce an equivalent cached/no-cache result and a changed graph/cache state;
+these are functional checks, not timing thresholds. The generated SQL project
+uses `vars.yml` without duplicating `vars` in `dbt_project.yml`.
+
+The SQL and manifest scenarios use the small-output `summary` command to
+measure model-level DAG construction without timing a large graph renderer.
+The manifest scenario does not measure column-lineage or MCP typed-`Manifest`
+replacement. The column scenario separately exercises a compiled-SQL column
+query.
+
+## Design context
+
+The SQL extraction workload includes macros and `ref()`/`source()` calls so
+Minijinja extraction is exercised. Existing measurements show Minijinja is the
+dominant model-lineage cost, so this suite informs cache changes without
+assuming a more elaborate incremental graph. A future semantic SQL cache must
+hash the exact effective macro-prefix bytes passed to rendering; it should not
+silently replace that input with an order-independent macro-set hash.
+
+The generator is deterministic and writes no production cache. Workloads,
+results, and generated caches are ignored by git. Keep timing comparisons
+local and reproducible; benchmark thresholds are deliberately not CI gates.
+The generated SQL project contains only SQL-mode inputs (no target manifest),
+while the manifest project contains only `target/manifest.json`; this keeps
+the model-level cache scenarios isolated from freshness and filesystem scans.

benchmarks/cache/scripts/generate_workload.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+"""Generate deterministic, dlin-only cache benchmark fixtures."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import shutil
+import tempfile
+from pathlib import Path
+
+
+SUITE_ROOT = Path(__file__).resolve().parents[1]
+PROFILES = {"small": 64, "medium": 512}
+
+
+def manifest_node(
+    project: str,
+    name: str,
+    depends_on: list[str],
+    path: str,
+    sql: str,
+) -> dict:
+    return {
+        "unique_id": f"model.{project}.{name}",
+        "name": name,
+        "resource_type": "model",
+        "depends_on": {"nodes": depends_on},
+        "config": {"materialized": "view"},
+        "description": f"Cache benchmark model {name}",
+        "path": path,
+        "original_file_path": path,
+        "columns": {"id": {"name": "id"}, "amount": {"name": "amount"}},
+        "compiled_code": sql,
+        "database": "analytics",
+        "schema": "main",
+    }
+
+
+def write_workload(root: Path, count: int) -> None:
+    project = "cache_benchmark"
+    sql_root = root / "sql_project"
+    manifest_root = root / "manifest_project"
+    (sql_root / "models").mkdir(parents=True, exist_ok=True)
+    (manifest_root / "target").mkdir(parents=True, exist_ok=True)
+
+    dbt_project = (
+        "name: cache_benchmark\n"
+        "version: '1.0'\n"
+        "config-version: 2\n"
+        "model-paths: [models]\n"
+        "macro-paths: [macros]\n"
+    )
+    macro = """{% macro benchmark_label(value) %}{{ value }}{% endmacro %}\n"""
+    (sql_root / "dbt_project.yml").write_text(dbt_project, encoding="utf-8")
+    (sql_root / "macros").mkdir(exist_ok=True)
+    (sql_root / "macros" / "benchmark.sql").write_text(macro, encoding="utf-8")
+    parent_name = "orders" if count == 1 else f"orders_{count - 2:04d}"
+    (sql_root / "vars.yml").write_text(
+        f"vars:\n  benchmark_parent: {parent_name}\n", encoding="utf-8"
+    )
+
+    source_id = f"source.{project}.raw.orders"
+    source = {
+        "unique_id": source_id,
+        "name": "orders",
+        "source_name": "raw",
+        "resource_type": "source",
+        "description": "Benchmark source",
+        "path": "models/schema.yml",
+        "original_file_path": "models/schema.yml",
+        "columns": {"id": {"name": "id"}, "amount": {"name": "amount"}},
+        "database": "raw",
+        "schema": "main",
+        "identifier": "orders",
+    }
+    nodes: dict[str, dict] = {}
+    previous = source_id
+    for index in range(count):
+        name = "orders" if index == 0 else f"orders_{index:04d}"
+        path = f"models/{name}.sql"
+        relation = (
+            '"raw"."orders"'
+            if index == 0
+            else f'"main"."{("orders" if index == 1 else f"orders_{index - 1:04d}")}"'
+        )
+        if index == count - 1 and index > 0:
+            sql = (
+                f"select {{{{ benchmark_label('id') }}}}, amount + {index} as amount "
+                "from {{ ref(var('benchmark_parent')) }}"
+            )
+        elif index == 0:
+            sql = "select id, amount from {{ source('raw', 'orders') }}"
+        else:
+            previous_name = "orders" if index == 1 else f"orders_{index - 1:04d}"
+            sql = (
+                f"select {{{{ benchmark_label('id') }}}}, amount + {index} as amount "
+                f"from {{{{ ref('{previous_name}') }}}}"
+            )
+        # The generated manifest contains compiled SQL, while SQL mode exercises
+        # the Jinja source/ref extraction path above.
+        compiled = (
+            f"select id, amount{f' + {index}' if index else ''} as amount "
+            f"from {relation}"
+        )
+        node = manifest_node(project, name, [previous], path, compiled)
+        nodes[node["unique_id"]] = node
+        previous = node["unique_id"]
+        (sql_root / path).parent.mkdir(parents=True, exist_ok=True)
+        (sql_root / path).write_text(sql + "\n", encoding="utf-8")
+
+    manifest = {
+        "metadata": {
+            "project_name": project,
+            "dbt_version": "1.12.0",
+            "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12/manifest.json",
+            "adapter_type": "duckdb",
+        },
+        "nodes": nodes,
+        "sources": {source_id: source},
+        "exposures": {},
+    }
+    payload = json.dumps(manifest, indent=2, sort_keys=True) + "\n"
+    (manifest_root / "target" / "manifest.json").write_text(payload, encoding="utf-8")
+
+    metadata = {
+        "profile": next(
+            (name for name, size in PROFILES.items() if size == count), "custom"
+        ),
+        "model_count": count,
+    }
+    metadata["files"] = {}
+    for file in sorted(root.rglob("*")):
+        if file.is_file() and file.name != "workload_metadata.json":
+            metadata["files"][file.relative_to(root).as_posix()] = hashlib.sha256(
+                file.read_bytes()
+            ).hexdigest()
+    metadata_path = root / "workload_metadata.json"
+    metadata_path.write_text(
+        json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+
+
+def tree_digest(root: Path) -> str:
+    digest = hashlib.sha256()
+    for file in sorted(root.rglob("*")):
+        if file.is_file():
+            digest.update(file.relative_to(root).as_posix().encode())
+            digest.update(file.read_bytes())
+    return digest.hexdigest()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, default=SUITE_ROOT / "workloads/default")
+    parser.add_argument("--profile", choices=sorted(PROFILES), default="small")
+    parser.add_argument(
+        "--self-check",
+        action="store_true",
+        help="verify two independent generations are byte-identical",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="replace an existing generated workload containing workload_metadata.json",
+    )
+    args = parser.parse_args()
+    if args.self_check:
+        with tempfile.TemporaryDirectory() as first, tempfile.TemporaryDirectory() as second:
+            write_workload(Path(first), PROFILES[args.profile])
+            write_workload(Path(second), PROFILES[args.profile])
+            if tree_digest(Path(first)) != tree_digest(Path(second)):
+                parser.error("workload generation is not deterministic")
+    args.output = args.output.resolve()
+    if args.output.exists():
+        if not args.output.is_dir():
+            parser.error(f"output exists but is not a directory: {args.output}")
+        if any(args.output.iterdir()):
+            if not args.force or not (args.output / "workload_metadata.json").is_file():
+                parser.error("output must be empty; use --force only for a previously generated workload")
+            shutil.rmtree(args.output)
+    args.output.mkdir(parents=True, exist_ok=True)
+    write_workload(args.output, PROFILES[args.profile])
+    print(f"generated {args.profile} workload ({PROFILES[args.profile]} models) at {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())