Harden evaluation harnesses

- isolate nested uv template invocations from the root virtualenv
- checkpoint eval steps so chained commands start from a clean tree
- handle byte output from timed-out subprocesses
- ignore XML formatting tails in duplicate-registration detection
- prune dependency dirs while walking generated projects
- fail skill eval runs on process errors and baseline skill leaks
- add fast unit tests for both harnesses

Author: MrTango

CHANGES.md

@@ -3,7 +3,12 @@
 ## 7.0.0b15 (unreleased)
 
 
-- Nothing changed yet.
+- Allow the plonecli skill to start the development server when the user
+  explicitly requests it.
+  [MrTango]
+
+- Harden scaffolding evaluation and remove false template Git warnings.
+  [MrTango]
 
 
 ## 7.0.0b14 (2026-08-13)

evals/scaffolding/EVALUATION.md

@@ -1,7 +1,9 @@
 # Scaffolding evaluation findings
 
-> Historical baseline from before the fixes. The current full report passes
-> 245/245 cases; see the ignored `results/report.md` generated on 2026-08-13.
+> Historical baseline from before the fixes. A full verification run passed
+> 245/245 cases with no warnings on 2026-08-13. The ignored
+> `results/report.md` is mutable and may instead contain the latest quick or
+> CI-validation run.
 
 ## Scope
 
@@ -61,7 +63,24 @@ Update the assertion and pytest target to the generated `src/<package>/tests/` l
 - Feature generation inside these nested, `--no-git` workspaces reports the outer plonecli repository as dirty. Git cleanliness checks should be scoped to the detected generated project rather than walking into an unrelated parent repository.
 - Keep the generated TOML/XML/Python validators as CI checks. They found failures that successful Copier exit codes did not detect.
 
-## Test receipts
+## Resolution
+
+All findings above have been addressed:
+
+- free-text TOML values use serialization filters;
+- standalone Zope projects are detected correctly;
+- chained `create` → `setup` refreshes project context;
+- theme variants reject conflicting overlays;
+- the Barceloneta integration test uses the generated package test path;
+- context hooks use the current in-place API;
+- the deprecated command-alias dependency was removed;
+- Git checks are scoped to the generated project;
+- subtemplate validation tasks use Copier's `_copier_operation` value and no
+  longer report files generated earlier in the same copy as pre-existing
+  changes;
+- generated TOML/XML/Python validation runs in CI.
+
+## Baseline test receipts
 
 - Root unit suite: **209 passed, 16 skipped**.
 - Copier-template unit suite: **386 passed, 2 integration tests deselected**.

evals/scaffolding/run_evals.py

@@ -948,20 +948,32 @@ def build_cases(quick: bool) -> tuple[list[Case], dict[str, int]]:
     return cases, planned
 
 
+def _subprocess_text(value: str | bytes | None) -> str:
+    """Normalize subprocess output, including TimeoutExpired byte payloads."""
+    if value is None:
+        return ""
+    if isinstance(value, bytes):
+        return value.decode(errors="replace")
+    return value
+
+
 def run_command(
     step: Step, env: dict[str, str], timeout_seconds: int
 ) -> dict[str, Any]:
-    command = (
-        ["uv", *step.args[1:]]
-        if step.args and step.args[0] == "__uv__"
-        else [*CLI, *step.args]
-    )
+    is_direct_uv = bool(step.args and step.args[0] == "__uv__")
+    command = ["uv", *step.args[1:]] if is_direct_uv else [*CLI, *step.args]
+    command_env = env
+    if is_direct_uv:
+        # The template checkout has its own uv project. Do not leak the root
+        # project's active environment into that nested invocation.
+        command_env = dict(env)
+        command_env.pop("VIRTUAL_ENV", None)
     started = time.monotonic()
     try:
         completed = subprocess.run(
             command,
             cwd=step.cwd,
-            env=env,
+            env=command_env,
             text=True,
             capture_output=True,
             stdin=subprocess.DEVNULL,
@@ -973,8 +985,10 @@ def run_command(
         timed_out = False
     except subprocess.TimeoutExpired as exc:
         exit_code = 124
-        stdout = exc.stdout or ""
-        stderr = (exc.stderr or "") + f"\nTimed out after {timeout_seconds}s\n"
+        stdout = _subprocess_text(exc.stdout)
+        stderr = (
+            _subprocess_text(exc.stderr) + f"\nTimed out after {timeout_seconds}s\n"
+        )
         timed_out = True
     return {
         "command": command,
@@ -987,6 +1001,41 @@ def run_command(
     }
 
 
+def _checkpoint_project(project: Path, step_number: int) -> None:
+    """Commit an intermediate eval step so the next command starts clean."""
+    status = subprocess.run(
+        ["git", "status", "--porcelain"],
+        cwd=project,
+        check=True,
+        capture_output=True,
+        text=True,
+        stdin=subprocess.DEVNULL,
+    )
+    if not status.stdout.strip():
+        return
+    subprocess.run(
+        ["git", "add", "-A"],
+        cwd=project,
+        check=True,
+        stdin=subprocess.DEVNULL,
+    )
+    subprocess.run(
+        [
+            "git",
+            "-c",
+            "user.name=Evaluation Runner",
+            "-c",
+            "user.email=eval@example.invalid",
+            "commit",
+            "-qm",
+            f"Evaluation step {step_number}",
+        ],
+        cwd=project,
+        check=True,
+        stdin=subprocess.DEVNULL,
+    )
+
+
 def execute_case(
     case: Case, env: dict[str, str], timeout_seconds: int
 ) -> dict[str, Any]:
@@ -1048,7 +1097,7 @@ def execute_case(
     commands = []
     errors = []
     observations: list[str] = []
-    for step in case.steps:
+    for step_number, step in enumerate(case.steps, 1):
         step.cwd.mkdir(parents=True, exist_ok=True)
         result = run_command(step, env, timeout_seconds)
         commands.append(result)
@@ -1062,6 +1111,8 @@ def execute_case(
         if result["exit_code"] != 0:
             errors.append(f"command {len(commands)} exited {result['exit_code']}")
             break
+        if case.project and step_number < len(case.steps):
+            _checkpoint_project(case.project, step_number)
 
     validation: dict[str, list[str]] = {}
     if case.project and case.project.exists():

evals/scaffolding/validators.py

@@ -2,24 +2,28 @@
 
 from __future__ import annotations
 
+import os
 import tomllib
 import xml.etree.ElementTree as ET
 from collections import Counter
+from collections.abc import Iterable
 from pathlib import Path
 
 IGNORED_PARTS = {".git", ".venv", "node_modules", "__pycache__"}
 XML_SUFFIXES = {".xml", ".zcml"}
 
 
 def _files(root: Path):
-    for path in root.rglob("*"):
-        if path.is_file() and not (set(path.parts) & IGNORED_PARTS):
-            yield path
+    """Yield project files while pruning generated dependency directories."""
+    for directory, dirnames, filenames in os.walk(root):
+        dirnames[:] = [name for name in dirnames if name not in IGNORED_PARTS]
+        base = Path(directory)
+        yield from (base / name for name in filenames)
 
 
-def validate_toml(root: Path) -> list[str]:
+def validate_toml(root: Path, files: Iterable[Path] | None = None) -> list[str]:
     errors: list[str] = []
-    for path in _files(root):
+    for path in files if files is not None else _files(root):
         if path.suffix != ".toml":
             continue
         try:
@@ -30,9 +34,9 @@ def validate_toml(root: Path) -> list[str]:
     return errors
 
 
-def validate_xml(root: Path) -> list[str]:
+def validate_xml(root: Path, files: Iterable[Path] | None = None) -> list[str]:
     errors: list[str] = []
-    for path in _files(root):
+    for path in files if files is not None else _files(root):
         if path.suffix not in XML_SUFFIXES:
             continue
         try:
@@ -42,9 +46,9 @@ def validate_xml(root: Path) -> list[str]:
     return errors
 
 
-def validate_python(root: Path) -> list[str]:
+def validate_python(root: Path, files: Iterable[Path] | None = None) -> list[str]:
     errors: list[str] = []
-    for path in _files(root):
+    for path in files if files is not None else _files(root):
         if path.suffix != ".py":
             continue
         try:
@@ -55,25 +59,47 @@ def validate_python(root: Path) -> list[str]:
     return errors
 
 
-def detect_duplicate_xml_registrations(root: Path) -> list[str]:
+def _element_identity(element: ET.Element, cache: dict[int, tuple]) -> tuple:
+    """Build a hashable subtree identity once, excluding formatting tails."""
+    key = id(element)
+    if key not in cache:
+        cache[key] = (
+            element.tag,
+            tuple(sorted(element.attrib.items())),
+            element.text,
+            tuple(_element_identity(child, cache) for child in element),
+        )
+    return cache[key]
+
+
+def detect_duplicate_xml_registrations(
+    root: Path, files: Iterable[Path] | None = None
+) -> list[str]:
     """Find exact repeated direct-child registrations in generated XML.
 
     Exact element identity is deliberately conservative: it catches hooks that
     append the same registration twice without treating similar, valid
     registrations as duplicates.
     """
     errors: list[str] = []
-    for path in _files(root):
+    for path in files if files is not None else _files(root):
         if path.suffix not in XML_SUFFIXES:
             continue
         try:
             tree = ET.parse(path)
         except (OSError, ET.ParseError):
             continue
+        cache: dict[int, tuple] = {}
         for parent in tree.iter():
-            serialized = [ET.tostring(child, encoding="unicode") for child in parent]
-            for element, count in Counter(serialized).items():
+            children_by_identity = {
+                _element_identity(child, cache): child for child in parent
+            }
+            counts = Counter(_element_identity(child, cache) for child in parent)
+            for identity, count in counts.items():
                 if count > 1:
+                    element = ET.tostring(
+                        children_by_identity[identity], encoding="unicode"
+                    )
                     preview = " ".join(element.split())[:160]
                     errors.append(
                         f"duplicate XML registration ({count}x) "
@@ -84,9 +110,10 @@ def detect_duplicate_xml_registrations(root: Path) -> list[str]:
 
 def validate_project(root: Path) -> dict[str, list[str]]:
     """Run all deterministic, install-free generated-project checks."""
+    files = tuple(_files(root))
     return {
-        "toml": validate_toml(root),
-        "xml": validate_xml(root),
-        "python": validate_python(root),
-        "duplicate_xml": detect_duplicate_xml_registrations(root),
+        "toml": validate_toml(root, files),
+        "xml": validate_xml(root, files),
+        "python": validate_python(root, files),
+        "duplicate_xml": detect_duplicate_xml_registrations(root, files),
     }

evals/skill/README.md

@@ -41,11 +41,11 @@ python evals/skill/run_evals.py --mode both --cases restapi-implicit,upgrade-ste
 python evals/skill/run_evals.py --model haiku          # cheaper smoke run
 ```
 
-Requires the `claude` CLI logged in. Runs bill real model usage — a full
-`--mode both` sweep is ~16 agent runs. Sandboxes and transcripts land in
+Requires the `claude` CLI logged in. Runs bill real model usage. Sandboxes and transcripts land in
 `<tmpdir>/plonecli-skill-evals/<timestamp>/` (outside the repo on purpose:
 a sandbox inside this repo lets the baseline agent *find* the skill by
-searching the project); `results.json` there summarizes. Each run prints
+searching the project); `results.json` there summarizes. A full `--mode both`
+sweep is 24 agent runs. Each run prints
 `skills fired: [...]` — in `noskill` mode it must be `none`, anything else
 means a skill leaked into the baseline.