benchmark: merge verify_results into the run check level

OutlyingWest · OutlyingWest · commit e835a5dd2334 · 2026-07-22T18:20:22.000+02:00
diff --git a/jumper_extension/adapters/ai_reviewer/benchmark/checks.py b/jumper_extension/adapters/ai_reviewer/benchmark/checks.py
@@ -13,16 +13,14 @@
 from jumper_extension.adapters.ai_reviewer.language import (
     RUN,
     VALIDATE_SYNTAX,
-    VERIFY_RESULTS,
     LanguageAdapter,
 )
 
 logger = logging.getLogger("extension")
 
-_NAMES = ("validate_syntax", "verify_results", "run")
+_NAMES = ("validate_syntax", "run")
 _CAPABILITY = {
     "validate_syntax": VALIDATE_SYNTAX,
-    "verify_results": VERIFY_RESULTS,
     "run": RUN,
 }
 
@@ -37,15 +35,13 @@ class Decision:
 @dataclass
 class CheckPlan:
     validate_syntax: Decision
-    verify_results: Decision
     run: Decision
 
 
 def all_active() -> CheckPlan:
     """The default plan: every step on. Used when no plan is injected."""
     return CheckPlan(
         validate_syntax=Decision(True),
-        verify_results=Decision(True),
         run=Decision(True),
     )
 
@@ -65,13 +61,6 @@ def resolve_checks(
 
     decisions = {name: _decide(name, enabled[name], adapter) for name in _NAMES}
 
-    # Verifying results means fingerprinting what an execution produced; with no
-    # timed run there is nothing to fingerprint, so it cannot stand on its own.
-    if decisions["verify_results"].active and not decisions["run"].active:
-        decisions["verify_results"] = Decision(
-            False, "requires the timed run, which is not running"
-        )
-
     _log_plan(adapter, decisions)
     return CheckPlan(**decisions)
 
diff --git a/jumper_extension/adapters/ai_reviewer/benchmark/runner.py b/jumper_extension/adapters/ai_reviewer/benchmark/runner.py
@@ -17,7 +17,6 @@
 import pandas as pd
 
 from jumper_extension.adapters.ai_reviewer.benchmark import fingerprint
-from jumper_extension.adapters.ai_reviewer.benchmark.checks import CheckPlan, all_active
 from jumper_extension.adapters.ai_reviewer.benchmark.models import FAILED, OK, TIMEOUT, RunOutcome
 from jumper_extension.adapters.ai_reviewer.context.collector import ContextCollector
 from jumper_extension.adapters.ai_reviewer.language import (
@@ -45,7 +44,6 @@ def __init__(
         level: str = "process",
         work_dir: str | None = None,
         adapter: LanguageAdapter | None = None,
-        checks: CheckPlan | None = None,
     ):
         self.prefix_cells = prefix_cells
         self.interval = interval
@@ -54,8 +52,6 @@ def __init__(
         # The target cell's language decides how a replay is built and launched;
         # defaults to Python so direct constructors keep their old behaviour.
         self.adapter = adapter or get_adapter("python")
-        # Defaults to every step on; controls whether results are fingerprinted.
-        self.checks = checks or all_active()
 
     @property
     def work_dir(self) -> str:
@@ -65,9 +61,9 @@ def run_once(self, code: str, tag: str, timeout: float | None = None) -> RunOutc
         """Replay the prefix plus *code* once, timing the last cell."""
         session_path = os.path.join(self._work_dir, f"{tag}_session.zip")
         fingerprint_path = os.path.join(self._work_dir, f"{tag}_fingerprint.json")
-        # Only fingerprint outputs when results are being verified; otherwise the
-        # replay captures nothing and correctness comes back UNVERIFIED.
-        output_names = self.adapter.output_names(code) if self.checks.verify_results.active else []
+        # A timed run always fingerprints its outputs: verification rides along
+        # with the replay (the two are one check level) and is cheap beside it.
+        output_names = self.adapter.output_names(code)
         artifact = self.adapter.render_replay(
             ReplayRequest(
                 prefix_cells=self.prefix_cells,
diff --git a/jumper_extension/adapters/ai_reviewer/language/__init__.py b/jumper_extension/adapters/ai_reviewer/language/__init__.py
@@ -7,7 +7,6 @@
 from jumper_extension.adapters.ai_reviewer.language.base import (
     RUN,
     VALIDATE_SYNTAX,
-    VERIFY_RESULTS,
     CapabilityNotSupported,
     LanguageAdapter,
     ReplayArtifact,
@@ -33,7 +32,6 @@
 __all__ = [
     "RUN",
     "VALIDATE_SYNTAX",
-    "VERIFY_RESULTS",
     "CapabilityNotSupported",
     "LanguageAdapter",
     "ReplayArtifact",
diff --git a/jumper_extension/adapters/ai_reviewer/language/base.py b/jumper_extension/adapters/ai_reviewer/language/base.py
@@ -14,11 +14,12 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 
-# The three benchmark steps an adapter may implement, matching the configurable
+# The two benchmark steps an adapter may implement, matching the configurable
 # check levels (ai.benchmark.checks). A step whose capability is absent is
-# skipped with a warning unless the user turned it off deliberately.
+# skipped with a warning unless the user turned it off deliberately. RUN covers
+# the timed replay and the result fingerprinting together: there is nothing to
+# fingerprint without an execution, and capturing it is cheap next to the run.
 VALIDATE_SYNTAX = "validate_syntax"
-VERIFY_RESULTS = "verify_results"
 RUN = "run"
 
 
@@ -68,7 +69,7 @@ class LanguageAdapter(ABC):
 
     Concrete adapters set ``language`` (matched case-insensitively against the
     ``language`` recorded on each cell) and ``caps`` (the subset of
-    ``VALIDATE_SYNTAX`` / ``VERIFY_RESULTS`` / ``RUN`` they actually implement).
+    ``VALIDATE_SYNTAX`` / ``RUN`` they actually implement).
     """
     language: str = ""
     caps: frozenset[str] = frozenset()
diff --git a/jumper_extension/adapters/ai_reviewer/language/python_adapter.py b/jumper_extension/adapters/ai_reviewer/language/python_adapter.py
@@ -13,7 +13,6 @@
 from jumper_extension.adapters.ai_reviewer.language.base import (
     RUN,
     VALIDATE_SYNTAX,
-    VERIFY_RESULTS,
     LanguageAdapter,
     ReplayArtifact,
     ReplayRequest,
@@ -24,7 +23,7 @@
 class PythonAdapter(LanguageAdapter):
     """Python cells: full support for every benchmark step."""
     language = "python"
-    caps = frozenset({VALIDATE_SYNTAX, VERIFY_RESULTS, RUN})
+    caps = frozenset({VALIDATE_SYNTAX, RUN})
 
     def validate_syntax(self, code: str) -> SyntaxResult:
         try:
diff --git a/jumper_extension/adapters/ai_reviewer/reviewer.py b/jumper_extension/adapters/ai_reviewer/reviewer.py
@@ -153,7 +153,6 @@ def build_orchestrator(self, state, fix_fn, target_index: int):
             interval=defaults.interval,
             level=state["level"],
             adapter=adapter,
-            checks=checks,
         )
         logger.info(
             f"[JUmPER]: benchmarking {len(state['suggestions'])} suggestion(s) against cell "
diff --git a/jumper_extension/config/ai/default.yaml b/jumper_extension/config/ai/default.yaml
@@ -29,8 +29,7 @@ benchmark:
   # --skip-check / --check.
   checks:
     validate_syntax: true   # static parse gate before any replay
-    verify_results: true    # fingerprint + compare results vs baseline (needs run)
-    run: true               # the timed replay itself
+    run: true               # timed replay + fingerprint/compare results vs baseline
 
 context:
   strategy: faster
diff --git a/jumper_extension/config/models.py b/jumper_extension/config/models.py
@@ -129,10 +129,9 @@ class AIBenchmarkChecksConfig(BaseModel):
     """
     # Static parse check gating a suggestion before any replay.
     validate_syntax: bool = True
-    # Fingerprint + compare a variant's results against the baseline's. Needs
-    # ``run`` (there is nothing to fingerprint without an execution).
-    verify_results: bool = True
-    # The timed replay itself - the measurement the benchmark exists for.
+    # The timed replay, plus fingerprinting and comparing each variant's results
+    # against the baseline's. The two are one step: there is nothing to
+    # fingerprint without an execution, and capturing it is cheap next to the run.
     run: bool = True
 
 
diff --git a/jumper_extension/core/parsers.py b/jumper_extension/core/parsers.py
@@ -126,7 +126,7 @@ def build_ai_review_parser() -> argparse.ArgumentParser:
         metavar="N",
         help="Repair rounds before a failing suggestion is reported as failed",
     )
-    _CHECK_NAMES = ("validate_syntax", "verify_results", "run")
+    _CHECK_NAMES = ("validate_syntax", "run")
     parser.add_argument(
         "--check",
         action="append",
diff --git a/jumper_extension/core/service.py b/jumper_extension/core/service.py
@@ -1000,7 +1000,7 @@ def _check_overrides(args) -> dict:
     skip = getattr(args, "skip_check", None)
     overrides: dict = {}
     if check:
-        for name in ("validate_syntax", "verify_results", "run"):
+        for name in ("validate_syntax", "run"):
             overrides[name] = name in check
     for name in skip or []:
         overrides[name] = False
diff --git a/tests/ai_reviewer/test_benchmark_checks.py b/tests/ai_reviewer/test_benchmark_checks.py
@@ -7,7 +7,6 @@
 from jumper_extension.adapters.ai_reviewer.language import (
     RUN,
     VALIDATE_SYNTAX,
-    VERIFY_RESULTS,
     LanguageAdapter,
     ReplayArtifact,
     SyntaxResult,
@@ -30,7 +29,7 @@ def render_replay(self, request):
         return ReplayArtifact(script_path="", command=[])
 
 
-_ALL_CAPS = {VALIDATE_SYNTAX, VERIFY_RESULTS, RUN}
+_ALL_CAPS = {VALIDATE_SYNTAX, RUN}
 
 
 def _config(**overrides):
@@ -41,15 +40,13 @@ def test_all_active_turns_every_step_on():
     plan = all_active()
 
     assert plan.validate_syntax.active
-    assert plan.verify_results.active
     assert plan.run.active
 
 
 def test_config_enabled_and_adapter_capable_runs_every_step():
     plan = resolve_checks(_Adapter("python", _ALL_CAPS), _config(), overrides=None)
 
     assert plan.validate_syntax.active
-    assert plan.verify_results.active
     assert plan.run.active
 
 
@@ -67,7 +64,7 @@ def test_step_disabled_in_config_is_skipped_without_warning(caplog):
 
 
 def test_enabled_step_without_capability_is_skipped_with_warning(caplog):
-    # R that can validate syntax but cannot run yet: run + verify degrade.
+    # R that can validate syntax but cannot run yet: the run step degrades.
     adapter = _Adapter("r", {VALIDATE_SYNTAX})
     with caplog.at_level(logging.WARNING, logger="extension"):
         plan = resolve_checks(adapter, _config(), overrides=None)
@@ -78,17 +75,17 @@ def test_enabled_step_without_capability_is_skipped_with_warning(caplog):
     assert any("benchmark run skipped" in r.message for r in caplog.records)
 
 
-def test_verify_results_needs_the_timed_run():
-    # run turned off deliberately -> verify cannot stand on its own.
+def test_run_can_be_turned_off_leaving_only_the_syntax_gate():
+    # The syntax gate stands on its own; disabling run is silent (deliberate).
     plan = resolve_checks(
         _Adapter("python", _ALL_CAPS),
         _config(run=False),
         overrides=None,
     )
 
+    assert plan.validate_syntax.active
     assert not plan.run.active
-    assert not plan.verify_results.active
-    assert "requires the timed run" in plan.verify_results.reason
+    assert plan.run.reason == ""
 
 
 def test_every_check_status_is_logged_at_debug_even_when_off(caplog):
@@ -101,16 +98,15 @@ def test_every_check_status_is_logged_at_debug_even_when_off(caplog):
 
     debug = [r.message for r in caplog.records if r.levelno == logging.DEBUG]
     assert any("check validate_syntax for 'python': skipped (disabled)" in m for m in debug)
-    assert any("check verify_results for 'python': active" in m for m in debug)
     assert any("check run for 'python': active" in m for m in debug)
 
 
 def test_overrides_win_over_config():
     plan = resolve_checks(
         _Adapter("python", _ALL_CAPS),
         _config(),
-        overrides={"verify_results": False},
+        overrides={"run": False},
     )
 
     assert plan.validate_syntax.active
-    assert not plan.verify_results.active
+    assert not plan.run.active
diff --git a/tests/ai_reviewer/test_language_registry.py b/tests/ai_reviewer/test_language_registry.py
@@ -3,7 +3,6 @@
 from jumper_extension.adapters.ai_reviewer.language import (
     RUN,
     VALIDATE_SYNTAX,
-    VERIFY_RESULTS,
     CapabilityNotSupported,
     FallbackAdapter,
     LanguageAdapter,
@@ -19,7 +18,7 @@
 
 class _StubAdapter(LanguageAdapter):
     language = "stub"
-    caps = frozenset({VALIDATE_SYNTAX, VERIFY_RESULTS})
+    caps = frozenset({VALIDATE_SYNTAX})
 
     def validate_syntax(self, code):
         return SyntaxResult(ok=True)
@@ -85,7 +84,6 @@ def test_supports_reflects_declared_caps():
     adapter = _StubAdapter()
 
     assert adapter.supports(VALIDATE_SYNTAX)
-    assert adapter.supports(VERIFY_RESULTS)
     assert not adapter.supports(RUN)
 
 
diff --git a/tests/ai_reviewer/test_r_adapter.py b/tests/ai_reviewer/test_r_adapter.py
@@ -22,12 +22,11 @@ def test_r_language_resolves_to_the_r_adapter():
 
 
 def test_run_is_not_claimed_yet():
-    # 5a exposes no timed run; that is Phase 5b.
-    from jumper_extension.adapters.ai_reviewer.language import RUN, VERIFY_RESULTS
+    # 5a exposes no timed run (and so no result verification); that is Phase 5b.
+    from jumper_extension.adapters.ai_reviewer.language import RUN
 
     adapter = get_adapter("r")
     assert RUN not in adapter.caps
-    assert VERIFY_RESULTS not in adapter.caps
 
 
 def test_capability_drops_when_rscript_is_missing():

Original file line number	Diff line number	Diff line change
`@@ -153,7 +153,6 @@ def build_orchestrator(self, state, fix_fn, target_index: int):`
`153`	`153`	`interval=defaults.interval,`
`154`	`154`	`level=state["level"],`
`155`	`155`	`adapter=adapter,`
`156`		`- checks=checks,`
`157`	`156`	`)`
`158`	`157`	`logger.info(`
`159`	`158`	`f"[JUmPER]: benchmarking {len(state['suggestions'])} suggestion(s) against cell "`
Original file line number	Diff line number	Diff line change
`@@ -126,7 +126,7 @@ def build_ai_review_parser() -> argparse.ArgumentParser:`
`126`	`126`	`metavar="N",`
`127`	`127`	`help="Repair rounds before a failing suggestion is reported as failed",`
`128`	`128`	`)`
`129`		`- _CHECK_NAMES = ("validate_syntax", "verify_results", "run")`
	`129`	`+ _CHECK_NAMES = ("validate_syntax", "run")`
`130`	`130`	`parser.add_argument(`
`131`	`131`	`"--check",`
`132`	`132`	`action="append",`