Skip to content

Commit 40b3a33

Browse files
committed
Fail closed on missing example goldens
1 parent 0745b1c commit 40b3a33

3 files changed

Lines changed: 37 additions & 1 deletion

File tree

CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
88
## [Unreleased]
99

1010
### Changed
11+
- Normal example verifier runs now fail closed when a golden output fixture is
12+
missing; only explicit `--update-golden` runs write fixture files.
1113
- Documentation now scopes the low-recursion implementation claim to the
1214
compiler stages that are actually stack-based today and tracks fully
1315
iterative backend emission as follow-up work.

scripts/verify_examples_common.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,12 +148,16 @@ def verify_example(
148148
fixture_path = fixtures_dir / f"{example.stem}.out"
149149
result.output_path = str(fixture_path)
150150

151-
if update_golden or not fixture_path.exists():
151+
if update_golden:
152152
fixture_path.parent.mkdir(parents=True, exist_ok=True)
153153
fixture_path.write_text(actual, encoding="utf-8")
154154
result.output_match = True
155155
return result
156156

157+
if not fixture_path.exists():
158+
result.error = f"missing golden fixture: {fixture_path}"
159+
return result
160+
157161
expected = fixture_path.read_text(encoding="utf-8")
158162
if actual == expected:
159163
result.output_match = True

test/test_examples_e2e.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import subprocess
66
import sys
77
import json
8+
import shutil
89
from pathlib import Path
910

1011
import pytest
@@ -52,3 +53,32 @@ def test_examples_end_to_end_outputs_match_goldens(tmp_path: Path) -> None:
5253
assert item["run_ok"] is True
5354
assert item["output_match"] is True
5455
assert item["error"] == ""
56+
57+
58+
@pytest.mark.skipif(not has_zig(), reason="zig not installed")
59+
def test_examples_verifier_fails_when_golden_fixture_is_missing(tmp_path: Path) -> None:
60+
examples_dir = tmp_path / "examples"
61+
fixtures_dir = tmp_path / "fixtures"
62+
examples_dir.mkdir()
63+
fixtures_dir.mkdir()
64+
shutil.copy(PROJECT_ROOT / "examples" / "001_hello.a7", examples_dir / "001_hello.a7")
65+
66+
result = subprocess.run(
67+
[
68+
sys.executable,
69+
str(VERIFY_SCRIPT),
70+
"--examples-dir",
71+
str(examples_dir),
72+
"--fixtures-dir",
73+
str(fixtures_dir),
74+
],
75+
cwd=PROJECT_ROOT,
76+
capture_output=True,
77+
text=True,
78+
timeout=180,
79+
)
80+
81+
combined = (result.stdout or "") + (result.stderr or "")
82+
assert result.returncode == 1, combined
83+
assert "missing golden fixture" in combined
84+
assert not (fixtures_dir / "001_hello.out").exists()

0 commit comments

Comments
 (0)