benchmark: add a standalone syntax-only check mode

Author: OutlyingWest

jumper_extension/adapters/ai_reviewer/benchmark/orchestrator.py

@@ -7,6 +7,7 @@
 """
 import logging
 from concurrent.futures import Future, ThreadPoolExecutor, wait, FIRST_COMPLETED
+from functools import partial
 from typing import Callable
 
 from jumper_extension.adapters.ai_reviewer.benchmark import fingerprint
@@ -68,20 +69,30 @@ def __init__(
         self._position: dict[str, int] = {}
 
     def run(self, baseline_code: str, variants: list[tuple[str, str]]) -> dict:
-        """Benchmark *baseline_code*, then each ``(label, code)`` variant.
-
-        Returns ``{label: BenchmarkResult}``; the baseline is under
-        ``BASELINE_LABEL``. An empty dict means the baseline itself would not
-        run, which leaves nothing to compare against.
+        """Score each ``(label, code)`` variant the best the active checks allow.
+
+        With the timed run on, this measures *baseline_code* and every variant
+        and compares their results, returning ``{label: BenchmarkResult}`` with
+        the baseline under ``BASELINE_LABEL``. With the run turned off but
+        ``validate_syntax`` still on, it falls back to a syntax-only pass that
+        still repairs broken suggestions - just without timing or comparison,
+        and without a baseline entry. With neither active there is nothing to
+        do. An empty dict also means the baseline itself would not run, leaving
+        nothing to compare against.
         """
-        if not self.checks.run.active:
-            # No timed replay: the only thing left worth measuring is gone, so
-            # there is nothing to compare. resolve_checks already warned why.
-            logger.info(
-                "[JUmPER]: benchmark timed run is off; nothing to measure or compare."
-            )
-            return {}
+        if self.checks.run.active:
+            return self._run_timed(baseline_code, variants)
+        if self.checks.validate_syntax.active:
+            return self._run_syntax_only(variants)
+        # Both the run and the syntax gate are off: resolve_checks already
+        # explained why, and there is genuinely nothing left to do here.
+        logger.info(
+            "[JUmPER]: benchmark has no active checks; nothing to measure or verify."
+        )
+        return {}
 
+    def _run_timed(self, baseline_code: str, variants: list[tuple[str, str]]) -> dict:
+        """The full benchmark: measure the baseline, then time and verify each variant."""
         self.progress = BenchmarkProgress(len(variants), self.runs)
         self._position = {label: index for index, (label, _) in enumerate(variants, start=1)}
 
@@ -111,74 +122,63 @@ def run(self, baseline_code: str, variants: list[tuple[str, str]]) -> dict:
         timeout = self._timeout_from(baseline_runs, duration)
 
         results = {BASELINE_LABEL: baseline}
-        results.update(self._benchmark_variants(variants, timeout, duration, prints))
+        results.update(
+            self._drive(
+                variants,
+                partial(
+                    self._process_timed,
+                    timeout=timeout,
+                    baseline_duration=duration,
+                    baseline_prints=prints,
+                ),
+            )
+        )
         self.progress.finished()
         return results
 
-    def _benchmark_variants(
-        self,
-        variants: list[tuple[str, str]],
-        timeout: float,
-        baseline_duration: float,
-        baseline_prints: dict,
-    ) -> dict:
+    def _run_syntax_only(self, variants: list[tuple[str, str]]) -> dict:
+        """Check each suggestion parses, repairing what does not - no timing.
+
+        The timed run is off, so there is nothing to measure or fingerprint, but
+        a broken suggestion can still be caught and handed to the same repair
+        loop. Each surviving variant is reported OK with ``UNVERIFIED``
+        correctness and its valid (possibly repaired) code in ``final_code``;
+        ones that never parse are reported FAILED.
+        """
+        logger.info(
+            f"[JUmPER]: benchmark: validating the syntax of {len(variants)} "
+            "suggestion(s), with the timed run off."
+        )
+        self.progress = BenchmarkProgress(len(variants), self.runs)
+        self._position = {label: index for index, (label, _) in enumerate(variants, start=1)}
+
+        results = self._drive(variants, self._process_syntax)
+
+        valid = sum(1 for result in results.values() if result.ok)
+        logger.info(
+            f"[JUmPER]: benchmark: syntax validated - {valid} valid, "
+            f"{len(results) - valid} unfixable."
+        )
+        return results
+
+    def _drive(self, variants: list[tuple[str, str]], process: Callable) -> dict:
+        """Run *process* over each variant, draining repairs the shared way.
+
+        Every mode differs only in what it does with a fresh candidate; the
+        repair round-trip - fold a returned fix back into the queue, or settle a
+        candidate whose repair produced nothing - is identical, so it lives here
+        once. *process* is called as ``process(results, candidate, pending,
+        repairing, pool)`` and is responsible for settling the candidate or
+        submitting it for repair.
+        """
         results: dict = {}
         pending = [_Candidate(label, code, self.fix_attempts) for label, code in variants]
         repairing: dict[Future, _Candidate] = {}
 
         with ThreadPoolExecutor(max_workers=max(1, len(pending))) as pool:
             while pending or repairing:
                 if pending:
-                    candidate = pending.pop(0)
-                    position = self._position[candidate.label]
-
-                    # Syntax is checkable for free, so broken code never costs a
-                    # replay - whether it came from the model or from a repair.
-                    if not self._syntax_ok(candidate):
-                        self._reject(results, candidate, position, pool, repairing)
-                        continue
-
-                    outcome = self._measure(
-                        candidate.code,
-                        candidate.label,
-                        timeout,
-                        on_run=lambda index: self.progress.variant_run(position, index),
-                    )
-                    outstanding = len(pending) + len(repairing)
-                    if isinstance(outcome, list):
-                        verdict = self._verdict(
-                            candidate, outcome, baseline_duration, baseline_prints
-                        )
-                        candidate.last_verdict = verdict
-                        candidate.last_code = candidate.code
-
-                        # A wrong answer is worth repairing too: it costs the
-                        # review more than a crash, since nothing else catches it.
-                        if verdict.correctness == fingerprint.DIFFERS:
-                            candidate.error = fingerprint.describe_divergence(
-                                baseline_prints,
-                                outcome[-1].fingerprints,
-                                verdict.differing_names,
-                            )
-                            if self._submit_fix(candidate, pool, repairing):
-                                self.progress.variant_diverged(
-                                    position,
-                                    verdict.differing_names,
-                                    candidate.attempts,
-                                    self.fix_attempts,
-                                )
-                                continue
-
-                        self._settle(results, candidate, verdict)
-                        self.progress.variant_done(
-                            position,
-                            _summarize(verdict),
-                            _walls(outcome),
-                            outstanding=outstanding,
-                        )
-                    else:
-                        candidate.error = outcome.error
-                        self._reject(results, candidate, position, pool, repairing)
+                    process(results, pending.pop(0), pending, repairing, pool)
                     continue
 
                 done, _ = wait(list(repairing), return_when=FIRST_COMPLETED)
@@ -193,6 +193,89 @@ def _benchmark_variants(
                     pending.append(candidate)
         return results
 
+    def _process_timed(
+        self,
+        results: dict,
+        candidate: _Candidate,
+        pending: list,
+        repairing: dict,
+        pool: ThreadPoolExecutor,
+        *,
+        timeout: float,
+        baseline_duration: float,
+        baseline_prints: dict,
+    ) -> None:
+        """Time one candidate against the baseline, repairing a crash or a divergence."""
+        position = self._position[candidate.label]
+
+        # Syntax is checkable for free, so broken code never costs a replay -
+        # whether it came from the model or from a repair.
+        if not self._syntax_ok(candidate):
+            self._reject(results, candidate, position, pool, repairing)
+            return
+
+        outcome = self._measure(
+            candidate.code,
+            candidate.label,
+            timeout,
+            on_run=lambda index: self.progress.variant_run(position, index),
+        )
+        outstanding = len(pending) + len(repairing)
+        if isinstance(outcome, list):
+            verdict = self._verdict(candidate, outcome, baseline_duration, baseline_prints)
+            candidate.last_verdict = verdict
+            candidate.last_code = candidate.code
+
+            # A wrong answer is worth repairing too: it costs the review more
+            # than a crash, since nothing else catches it.
+            if verdict.correctness == fingerprint.DIFFERS:
+                candidate.error = fingerprint.describe_divergence(
+                    baseline_prints,
+                    outcome[-1].fingerprints,
+                    verdict.differing_names,
+                )
+                if self._submit_fix(candidate, pool, repairing):
+                    self.progress.variant_diverged(
+                        position,
+                        verdict.differing_names,
+                        candidate.attempts,
+                        self.fix_attempts,
+                    )
+                    return
+
+            self._settle(results, candidate, verdict)
+            self.progress.variant_done(
+                position,
+                _summarize(verdict),
+                _walls(outcome),
+                outstanding=outstanding,
+            )
+        else:
+            candidate.error = outcome.error
+            self._reject(results, candidate, position, pool, repairing)
+
+    def _process_syntax(
+        self,
+        results: dict,
+        candidate: _Candidate,
+        pending: list,
+        repairing: dict,
+        pool: ThreadPoolExecutor,
+    ) -> None:
+        """Parse one candidate, repairing it if it does not - never running it."""
+        position = self._position[candidate.label]
+        if self._syntax_ok(candidate):
+            self.final_code[candidate.label] = candidate.code
+            results[candidate.label] = BenchmarkResult(
+                label=candidate.label,
+                status=OK,
+                attempts=candidate.attempts,
+                correctness=fingerprint.UNVERIFIED,
+            )
+            self.progress.variant_validated(position, candidate.attempts)
+        else:
+            self._reject(results, candidate, position, pool, repairing)
+
     def _reject(
         self,
         results: dict,

jumper_extension/adapters/ai_reviewer/benchmark/progress.py

@@ -61,6 +61,14 @@ def variant_done(self, position: int, summary: str, walls: list[float], outstand
             f"{self._remaining(outstanding, prefix='; ', suffix=' left')}"
         )
 
+    def variant_validated(self, position: int, attempts: int) -> None:
+        """A suggestion parsed (syntax-only mode: no timing to report)."""
+        repaired = f" (repaired {attempts - 1}x)" if attempts > 1 else ""
+        logger.info(
+            f"[JUmPER]: benchmark: option {position}/{self.total_variants} "
+            f"syntax valid{repaired}"
+        )
+
     def variant_failed(self, position: int, error: str, attempt: int, attempts: int) -> None:
         logger.info(
             f"[JUmPER]: benchmark: option {position}/{self.total_variants} failed "

jumper_extension/adapters/ai_reviewer/ui/review_display.py

@@ -75,6 +75,19 @@ def verdict_of(state: OptimizationState, index: int) -> dict | None:
             "notes": [],
         }
 
+    # No timed run (e.g. --check validate_syntax): nothing was measured or
+    # compared, so report only that the code parses, plus any repair it took.
+    if result.duration_s is None:
+        notes = []
+        if result.attempts > 1:
+            notes.append(f"repaired after {result.attempts - 1} failed attempt(s)")
+        return {
+            "headline": "Syntax valid",
+            "tone": "warn",
+            "detail": "not timed",
+            "notes": notes,
+        }
+
     notes = []
     tone = "good"
     if result.correctness == fingerprint.DIFFERS:

tests/ai_reviewer/test_benchmark_orchestrator.py

@@ -1,12 +1,18 @@
 from unittest.mock import Mock
 
 from jumper_extension.adapters.ai_reviewer.benchmark import fingerprint
+from jumper_extension.adapters.ai_reviewer.benchmark.checks import CheckPlan, Decision
 from jumper_extension.adapters.ai_reviewer.benchmark.models import FAILED, OK, RunOutcome
 from jumper_extension.adapters.ai_reviewer.benchmark.orchestrator import (
     BASELINE_LABEL,
     BenchmarkOrchestrator,
 )
 
+
+def _syntax_only_checks() -> CheckPlan:
+    """run off, validate_syntax on - the standalone syntax-check plan."""
+    return CheckPlan(validate_syntax=Decision(True), run=Decision(False))
+
 _PRINTS = {"y": {"kind": "scalar", "value": 42.0}}
 
 
@@ -183,6 +189,64 @@ def test_an_unrepairable_divergence_keeps_the_measurement_it_did_get():
     assert results["1"].correctness == fingerprint.DIFFERS
 
 
+def test_syntax_only_mode_validates_without_running_anything():
+    log = []
+    orchestrator = _orchestrator(
+        lambda code: 1.0,
+        log=log,
+        checks=_syntax_only_checks(),
+    )
+
+    results = orchestrator.run("base", [("1", "y = 1")])
+
+    assert log == []  # not even the baseline was replayed
+    assert BASELINE_LABEL not in results  # nothing to compare against
+    assert results["1"].status == OK
+    assert results["1"].correctness == fingerprint.UNVERIFIED
+    assert results["1"].duration_s is None
+    assert orchestrator.final_code["1"] == "y = 1"
+
+
+def test_syntax_only_mode_repairs_a_broken_suggestion_without_running_it():
+    log = []
+    orchestrator = _orchestrator(
+        lambda code: 1.0,
+        fix_fn=lambda code, error, label: "y = 1",
+        log=log,
+        checks=_syntax_only_checks(),
+    )
+
+    results = orchestrator.run("base", [("1", "def broken(:")])
+
+    assert log == []  # the fix was checked statically, never executed
+    assert results["1"].status == OK
+    assert results["1"].attempts == 2
+    assert orchestrator.final_code["1"] == "y = 1"
+
+
+def test_syntax_only_mode_gives_up_on_an_unfixable_suggestion():
+    orchestrator = _orchestrator(
+        lambda code: 1.0,
+        fix_fn=lambda code, error, label: "def still_broken(:",
+        fix_attempts=2,
+        checks=_syntax_only_checks(),
+    )
+
+    results = orchestrator.run("base", [("1", "def broken(:")])
+
+    assert results["1"].status == FAILED
+    assert "1" not in orchestrator.final_code
+
+
+def test_nothing_runs_when_every_check_is_off():
+    orchestrator = _orchestrator(
+        lambda code: 1.0,
+        checks=CheckPlan(validate_syntax=Decision(False), run=Decision(False)),
+    )
+
+    assert orchestrator.run("base", [("1", "y = 1")]) == {}
+
+
 def test_timeout_budget_covers_the_prefix_and_room_to_be_slow():
     orchestrator = _orchestrator(lambda code: 4.0, timeout_factor=10.0)