test(harness): validate Hermes and Antigravity runtimes (#450)

solomonneas · claude · web-flow · commit a29ec4e1b209 · 2026-07-22T18:43:25.000-04:00
* test(harness): validate hermes and antigravity runtimes via version probe Replace the antigravity installation proxy (bare command_candidates availability on a gui surface that could never execute) with direct runtime evidence: gui/desktop fixtures that declare a CLI-companion binary block now run the same sandboxed --version probe, and the observed/nonzero receipts carry a parsed version line and a platform field. Hermes already used the cli path; both surfaces now have tests pinning the acceptance contract: - binary available: version probe records vendor version + platform - binary missing: externally_blocked with exact reason binary_not_found - a present home/config directory alone never counts as conformance Safety guards are unchanged: external_only desktop/gui surfaces still short-circuit, gui fixtures without a binary block stay limited to availability-only evidence, the exact [--version] gate still blocks unsafe args before spawn, and the sandbox keeps minimal env, temp HOME, bounded output, and redaction. Closes #343 Co-authored-by: Claude <noreply@anthropic.com> * fix(harness): align probe candidates and version parsing per Greptile review P1: a desktop/GUI install whose CLI companion is present under a declared availability candidate instead of the primary binary command produced a contradictory receipt (availability available, version probe binary_not_found). The version probe now resolves whichever declared candidate is present, so both signals describe the same runtime command. P2: the parsed version was the first nonblank output line, so a warning or banner printed before the version was reported as version evidence. The receipt now reports the first line carrying a dotted version number and null when no such line exists. Co-authored-by: Claude <noreply@anthropic.com> --------- Co-authored-by: Claude <noreply@anthropic.com>
diff --git a/docs/research/fixtures/harness-contract.v1/antigravity.json b/docs/research/fixtures/harness-contract.v1/antigravity.json
@@ -1,6 +1,7 @@
 {
   "schema": "harness-contract.v1",
   "harness": {"id": "antigravity", "surface": "gui"},
+  "binary": {"command": "agy", "version_args": ["--version"]},
   "availability": {"command_candidates": ["agy", "antigravity"]},
   "capabilities": [
     {"id": "instructions", "claim": "Skills codelab exists, but no local binary was available.", "provenance": "documented", "support_state": "externally_blocked", "implementation_layers": ["unknown"], "evidence": [{"kind": "vendor_documentation", "reference": "https://codelabs.developers.google.com/getting-started-with-antigravity-skills"}], "tested_version": null, "platform": "linux", "scope": "externally blocked"},
diff --git a/tests/test_harness_conformance_probe.py b/tests/test_harness_conformance_probe.py
@@ -399,6 +399,197 @@ def test_antigravity_is_gui_surface_with_availability_only_candidates(probe, sch
     assert result["version_probe"]["state"] == "skipped"
 
 
+def _fake_version_process(exit_code: int = 0) -> mock.Mock:
+    process = mock.Mock()
+    process.poll.return_value = exit_code
+    process.pid = 6161
+    return process
+
+
+def test_hermes_version_probe_records_version_and_platform_receipt(probe, schema, fixtures) -> None:
+    fixture = next(item for item in fixtures if item["harness"]["id"] == "hermes")
+    with (
+        mock.patch.object(probe.shutil, "which", return_value="/fake/bin/hermes"),
+        mock.patch.object(probe, "_popen_probe_process", return_value=_fake_version_process()),
+        mock.patch.object(
+            probe,
+            "_collect_bounded_output",
+            return_value=(b"hermes 0.3.1\n", False, None),
+        ),
+    ):
+        result = probe.probe_fixture(fixture, schema, run_version=True, timeout_seconds=1.0)
+    version_probe = result["version_probe"]
+    assert version_probe["state"] == "observed"
+    assert version_probe["command"] == "hermes"
+    assert version_probe["version"] == "hermes 0.3.1"
+    assert version_probe["platform"] == sys.platform
+    assert "hermes 0.3.1" in version_probe["output"]
+
+
+def test_hermes_missing_binary_stays_externally_blocked_binary_not_found(probe, schema, fixtures) -> None:
+    fixture = next(item for item in fixtures if item["harness"]["id"] == "hermes")
+    with mock.patch.object(probe.shutil, "which", return_value=None):
+        result = probe.probe_fixture(fixture, schema, run_version=True, timeout_seconds=1.0)
+    assert result["availability"]["state"] == "externally_blocked"
+    assert result["availability"]["reason"] == "binary_not_found"
+    assert result["version_probe"]["state"] == "externally_blocked"
+    assert result["version_probe"]["reason"] == "binary_not_found"
+
+
+def test_hermes_home_directory_only_never_counts_as_runtime_conformance(
+    probe, schema, fixtures, monkeypatch, tmp_path: Path
+) -> None:
+    fixture = next(item for item in fixtures if item["harness"]["id"] == "hermes")
+    home = tmp_path / "home"
+    (home / ".hermes").mkdir(parents=True)
+    (home / ".config" / "hermes").mkdir(parents=True)
+    monkeypatch.setenv("HOME", str(home))
+    with mock.patch.object(probe.shutil, "which", return_value=None):
+        result = probe.probe_fixture(fixture, schema, run_version=True, timeout_seconds=1.0)
+    for section in (result["availability"], result["version_probe"]):
+        assert section["state"] == "externally_blocked"
+        assert section["reason"] == "binary_not_found"
+
+
+def test_antigravity_version_probe_records_version_and_platform_receipt(probe, schema, fixtures) -> None:
+    fixture = next(item for item in fixtures if item["harness"]["id"] == "antigravity")
+    assert fixture["binary"] == {"command": "agy", "version_args": ["--version"]}
+    with (
+        mock.patch.object(
+            probe.shutil,
+            "which",
+            side_effect=lambda name: "/fake/bin/agy" if name == "agy" else None,
+        ),
+        mock.patch.object(probe, "_popen_probe_process", return_value=_fake_version_process()),
+        mock.patch.object(
+            probe,
+            "_collect_bounded_output",
+            return_value=(b"agy 1.4.0\n", False, None),
+        ),
+    ):
+        result = probe.probe_fixture(fixture, schema, run_version=True, timeout_seconds=1.0)
+    version_probe = result["version_probe"]
+    assert version_probe["state"] == "observed"
+    assert version_probe["command"] == "agy"
+    assert version_probe["version"] == "agy 1.4.0"
+    assert version_probe["platform"] == sys.platform
+    assert "agy 1.4.0" in version_probe["output"]
+
+
+def test_antigravity_version_probe_falls_back_to_declared_availability_candidates(probe, schema, fixtures) -> None:
+    # Greptile P1 on PR 450: an install with only the `antigravity` launcher
+    # must not report availability "available" alongside a blocked version
+    # probe; the probe resolves whichever declared candidate is present.
+    fixture = next(item for item in fixtures if item["harness"]["id"] == "antigravity")
+    with (
+        mock.patch.object(
+            probe.shutil,
+            "which",
+            side_effect=lambda name: "/fake/bin/antigravity" if name == "antigravity" else None,
+        ),
+        mock.patch.object(probe, "_popen_probe_process", return_value=_fake_version_process()),
+        mock.patch.object(
+            probe,
+            "_collect_bounded_output",
+            return_value=(b"antigravity 2.0.1\n", False, None),
+        ),
+    ):
+        result = probe.probe_fixture(fixture, schema, run_version=True, timeout_seconds=1.0)
+    assert result["availability"]["state"] == "available"
+    version_probe = result["version_probe"]
+    assert version_probe["state"] == "observed"
+    assert version_probe["command"] == "antigravity"
+    assert version_probe["version"] == "antigravity 2.0.1"
+    assert version_probe["platform"] == sys.platform
+
+
+def test_version_receipt_skips_banner_lines_without_version_numbers(probe, schema, fixtures) -> None:
+    # Greptile P2 on PR 450: a warning or banner printed before the version
+    # must not be reported as the version evidence.
+    fixture = next(item for item in fixtures if item["harness"]["id"] == "hermes")
+    bannered_output = b"WARNING: deprecated config key detected\nBuild channel: stable\nhermes 0.3.1\n"
+    with (
+        mock.patch.object(probe.shutil, "which", return_value="/fake/bin/hermes"),
+        mock.patch.object(probe, "_popen_probe_process", return_value=_fake_version_process()),
+        mock.patch.object(
+            probe,
+            "_collect_bounded_output",
+            return_value=(bannered_output, False, None),
+        ),
+    ):
+        result = probe.probe_fixture(fixture, schema, run_version=True, timeout_seconds=1.0)
+    version_probe = result["version_probe"]
+    assert version_probe["state"] == "observed"
+    assert version_probe["version"] == "hermes 0.3.1"
+
+
+def test_version_receipt_reports_none_when_no_version_line_present(probe, schema, fixtures) -> None:
+    fixture = next(item for item in fixtures if item["harness"]["id"] == "hermes")
+    with (
+        mock.patch.object(probe.shutil, "which", return_value="/fake/bin/hermes"),
+        mock.patch.object(probe, "_popen_probe_process", return_value=_fake_version_process()),
+        mock.patch.object(
+            probe,
+            "_collect_bounded_output",
+            return_value=(b"build channel: stable\n", False, None),
+        ),
+    ):
+        result = probe.probe_fixture(fixture, schema, run_version=True, timeout_seconds=1.0)
+    version_probe = result["version_probe"]
+    assert version_probe["state"] == "observed"
+    assert version_probe["version"] is None
+
+
+def test_antigravity_missing_binary_stays_externally_blocked_binary_not_found(probe, schema, fixtures) -> None:
+    fixture = next(item for item in fixtures if item["harness"]["id"] == "antigravity")
+    with mock.patch.object(probe.shutil, "which", return_value=None):
+        result = probe.probe_fixture(fixture, schema, run_version=True, timeout_seconds=1.0)
+    assert result["availability"]["state"] == "externally_blocked"
+    assert result["availability"]["reason"] == "binary_not_found"
+    assert result["version_probe"]["state"] == "externally_blocked"
+    assert result["version_probe"]["reason"] == "binary_not_found"
+
+
+def test_antigravity_home_directory_only_never_counts_as_runtime_conformance(
+    probe, schema, fixtures, monkeypatch, tmp_path: Path
+) -> None:
+    fixture = next(item for item in fixtures if item["harness"]["id"] == "antigravity")
+    home = tmp_path / "home"
+    (home / ".antigravity").mkdir(parents=True)
+    (home / ".config" / "antigravity").mkdir(parents=True)
+    monkeypatch.setenv("HOME", str(home))
+    with mock.patch.object(probe.shutil, "which", return_value=None):
+        result = probe.probe_fixture(fixture, schema, run_version=True, timeout_seconds=1.0)
+    for section in (result["availability"], result["version_probe"]):
+        assert section["state"] == "externally_blocked"
+        assert section["reason"] == "binary_not_found"
+
+
+def test_antigravity_version_args_gate_still_blocks_unsafe_args(probe, fixtures) -> None:
+    fixture = json.loads(json.dumps(next(item for item in fixtures if item["harness"]["id"] == "antigravity")))
+    fixture["binary"]["version_args"] = ["--full"]
+    with mock.patch.object(probe, "_popen_probe_process") as popen:
+        result = probe.run_version_probe(fixture, timeout_seconds=1.0)
+    assert result["state"] == "externally_blocked"
+    assert result["reason"] == "unsafe_version_arguments"
+    popen.assert_not_called()
+
+
+def test_gui_surface_without_binary_declaration_remains_version_limited(probe, schema) -> None:
+    fixture = {
+        "schema": "harness-contract.v1",
+        "harness": {"id": "gui-without-binary", "surface": "gui"},
+        "availability": {"command_candidates": ["example-gui"]},
+        "capabilities": _minimal_capabilities(),
+        "deep_probes": _minimal_deep_probes(),
+    }
+    with mock.patch.object(probe, "_popen_probe_process") as popen:
+        result = probe.probe_fixture(fixture, schema, run_version=True, timeout_seconds=1.0)
+    assert result["version_probe"]["state"] == "externally_blocked"
+    assert result["version_probe"]["reason"] == "version_execution_limited_to_cli_surface"
+    popen.assert_not_called()
+
+
 def test_deep_probes_are_returned_not_executed(probe, schema, fixtures) -> None:
     result = probe.probe_fixture(fixtures[0], schema, run_version=False, timeout_seconds=1.0)
     assert result["deep_probes"] == fixtures[0]["deep_probes"]
diff --git a/tools/harness_conformance_probe.py b/tools/harness_conformance_probe.py
@@ -48,6 +48,7 @@
 AUTHORIZATION_HEADER = re.compile(r"(?i)(Authorization\s*:\s*).+")
 BEARER_TOKEN = re.compile(r"(?i)\bBearer\s+([A-Za-z0-9._~+/=-]+)\b")
 JSON_QUOTED_CREDENTIAL = re.compile(r'(?i)("(?:api[_-]?key|token|secret|password)"\s*:\s*")[^"]+(")')
+VERSION_LINE = re.compile(r"\d+\.\d+(?:\.\d+)?(?:[-+~][0-9A-Za-z][0-9A-Za-z.-]*)?")
 OUTPUT_CAP_BYTES = 65536
 DEFAULT_TIMEOUT_SECONDS = 10.0
 
@@ -83,6 +84,21 @@ def redact_text(value: str, *home_dirs: str | None) -> str:
     return redacted
 
 
+def _extract_version_line(output: str) -> str | None:
+    """Return the first output line carrying a dotted version number.
+
+    Vendors sometimes print warnings or banners before the version on
+    ``--version``; a bare first-nonblank-line parse would report that banner as
+    the version. Requiring a dotted numeric component skips banner lines while
+    still accepting common formats (``0.3.1``, ``1.4.0-rc.1``, ``2.0+build5``).
+    """
+    for line in output.splitlines():
+        stripped = line.strip()
+        if stripped and VERSION_LINE.search(stripped):
+            return stripped
+    return None
+
+
 def load_schema(schema_path: Path) -> dict[str, Any]:
     with schema_path.open(encoding="utf-8") as handle:
         return json.load(handle)
@@ -369,7 +385,15 @@ def _collect_bounded_output(
 
 
 def run_version_probe(fixture: dict[str, Any], *, timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS) -> dict[str, Any]:
-    """Execute only ``<bare-command> --version`` inside an isolated sandbox."""
+    """Execute only ``<bare-command> --version`` inside an isolated sandbox.
+
+    CLI surfaces execute their declared ``binary.command``. Desktop/GUI surfaces
+    may declare a CLI-companion ``binary`` block (for example Antigravity's
+    ``agy``); when present, the same sandboxed version probe runs so runtime
+    conformance comes from direct vendor-binary evidence rather than an
+    installation proxy. Non-CLI fixtures without a ``binary`` block stay
+    limited to availability-only evidence.
+    """
     timeout_seconds = _positive_finite_timeout(timeout_seconds)
     harness = fixture.get("harness", {})
     harness_id = harness.get("id", "unknown")
@@ -382,16 +406,18 @@ def run_version_probe(fixture: dict[str, Any], *, timeout_seconds: float = DEFAU
             "reason": "desktop_or_gui_surface",
         }
 
-    if surface != "cli":
+    binary = fixture.get("binary", {})
+    if not isinstance(binary, dict):
+        binary = {}
+    command = binary.get("command")
+    version_args = binary.get("version_args")
+    if surface != "cli" and not isinstance(command, str):
         return {
             "harness_id": harness_id,
             "state": "externally_blocked",
             "reason": "version_execution_limited_to_cli_surface",
         }
 
-    binary = fixture.get("binary", {})
-    command = binary.get("command")
-    version_args = binary.get("version_args")
     if not isinstance(command, str) or not _is_safe_command_name(command):
         return {
             "harness_id": harness_id,
@@ -408,6 +434,20 @@ def run_version_probe(fixture: dict[str, Any], *, timeout_seconds: float = DEFAU
         }
 
     executable = shutil.which(command)
+    probed_command = command
+    if executable is None and surface != "cli":
+        # A desktop/GUI install may ship its CLI companion under one of the
+        # declared availability candidates instead of the primary binary
+        # command. Probe whichever candidate is present so the availability
+        # signal and the version receipt never disagree.
+        for candidate in _command_candidates(fixture):
+            if candidate == command or not _is_safe_command_name(candidate):
+                continue
+            resolved = shutil.which(candidate)
+            if resolved is not None:
+                executable = resolved
+                probed_command = candidate
+                break
     if executable is None:
         return {
             "harness_id": harness_id,
@@ -458,30 +498,33 @@ def run_version_probe(fixture: dict[str, Any], *, timeout_seconds: float = DEFAU
                 "harness_id": harness_id,
                 "state": "externally_blocked",
                 "reason": "TimeoutExpired",
-                "command": command,
+                "command": probed_command,
             }
         if overflow:
             return {
                 "harness_id": harness_id,
                 "state": "externally_blocked",
                 "reason": "output_overflow",
-                "command": command,
+                "command": probed_command,
             }
 
         output = redact_text(output_bytes.decode("utf-8", errors="replace"), str(home_dir))
         if return_code == 0:
             return {
                 "harness_id": harness_id,
                 "state": "observed",
-                "command": command,
+                "command": probed_command,
                 "exit_code": return_code,
+                "platform": sys.platform,
+                "version": _extract_version_line(output),
                 "output": output,
             }
         return {
             "harness_id": harness_id,
             "state": "nonzero_exit",
-            "command": command,
+            "command": probed_command,
             "exit_code": return_code,
+            "platform": sys.platform,
             "output": output,
         }
 

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"schema": "harness-contract.v1",`
`3`	`3`	`"harness": {"id": "antigravity", "surface": "gui"},`
	`4`	`+ "binary": {"command": "agy", "version_args": ["--version"]},`
`4`	`5`	`"availability": {"command_candidates": ["agy", "antigravity"]},`
`5`	`6`	`"capabilities": [`
`6`	`7`	`{"id": "instructions", "claim": "Skills codelab exists, but no local binary was available.", "provenance": "documented", "support_state": "externally_blocked", "implementation_layers": ["unknown"], "evidence": [{"kind": "vendor_documentation", "reference": "https://codelabs.developers.google.com/getting-started-with-antigravity-skills"}], "tested_version": null, "platform": "linux", "scope": "externally blocked"},`