Skip to content

Commit 7fb5c23

Browse files
ryan-williamsclaude
andcommitted
Add precision-aware test comparison and gen-expected workflow
- Add `scripts/merge_expected.py`: Merges Ubuntu and macOS CSVs, finding the longest decimal representation that both platforms would round to. Precision is encoded in the string representation. - Add `.github/workflows/gen-expected.yml`: Runs tests in GEN_VALS mode on both Ubuntu and macOS, then merges results into single `expected.csv` files per test. - Add `ExpectedHistory` type that loads CSVs as strings and compares with precision derived from the expected value string length. - Update test `check()` function: - Gen mode: writes platform-specific CSV (linux.csv / macos.csv) - Test mode: compares against merged `expected.csv` with precision- aware comparison Tests will fail until gen-expected workflow runs and creates the `expected.csv` files. Trigger with: push to `gen-expected` branch or manual workflow_dispatch. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent f4c2cf1 commit 7fb5c23

4 files changed

Lines changed: 444 additions & 20 deletions

File tree

.github/workflows/gen-expected.yml

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
name: Generate Expected Values
2+
on:
3+
workflow_dispatch:
4+
push:
5+
branches: [ gen-expected ]
6+
7+
env:
8+
RUST_BACKTRACE: 1
9+
RUST_LOG: info
10+
11+
jobs:
12+
gen-ubuntu:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v4
16+
- uses: Swatinem/rust-cache@v2
17+
18+
- name: Generate expected values
19+
run: GEN_VALS=1 cargo test || true
20+
21+
- uses: actions/upload-artifact@v4
22+
with:
23+
name: ubuntu-testdata
24+
path: testdata/*/linux.csv
25+
26+
gen-macos:
27+
runs-on: macos-latest
28+
steps:
29+
- uses: actions/checkout@v4
30+
- uses: Swatinem/rust-cache@v2
31+
32+
- name: Generate expected values
33+
run: GEN_VALS=1 cargo test || true
34+
35+
- uses: actions/upload-artifact@v4
36+
with:
37+
name: macos-testdata
38+
path: testdata/*/macos.csv
39+
40+
merge:
41+
needs: [gen-ubuntu, gen-macos]
42+
runs-on: ubuntu-latest
43+
steps:
44+
- uses: actions/checkout@v4
45+
46+
- uses: actions/download-artifact@v4
47+
with:
48+
name: ubuntu-testdata
49+
path: artifacts/ubuntu
50+
51+
- uses: actions/download-artifact@v4
52+
with:
53+
name: macos-testdata
54+
path: artifacts/macos
55+
56+
- name: Show artifacts
57+
run: find artifacts -type f | head -20
58+
59+
- uses: actions/setup-python@v5
60+
with:
61+
python-version: '3.12'
62+
63+
- name: Merge expected values
64+
run: |
65+
python scripts/merge_expected.py --batch \
66+
artifacts/ubuntu \
67+
artifacts/macos \
68+
testdata
69+
70+
- name: Show merged files
71+
run: |
72+
echo "=== Merged expected.csv files ==="
73+
find testdata -name 'expected.csv' | while read f; do
74+
echo "--- $f ---"
75+
head -3 "$f"
76+
echo "..."
77+
done
78+
79+
- name: Commit merged testdata
80+
run: |
81+
git config user.name "github-actions[bot]"
82+
git config user.email "github-actions[bot]@users.noreply.github.com"
83+
84+
# Remove old platform-specific files
85+
git rm -f testdata/*/linux.csv testdata/*/macos.csv 2>/dev/null || true
86+
87+
git add testdata/*/expected.csv
88+
git status
89+
90+
if git diff --staged --quiet; then
91+
echo "No changes to commit"
92+
else
93+
git commit -m "Regenerate merged expected values from Ubuntu + macOS"
94+
git push
95+
fi

scripts/merge_expected.py

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Merge platform-specific expected value CSVs into a single file.
4+
5+
For each numeric value, finds the longest decimal representation that both
6+
platforms would round to, preserving only the precision where they agree.
7+
"""
8+
9+
import csv
10+
import sys
11+
from decimal import Decimal, ROUND_HALF_EVEN
12+
from pathlib import Path
13+
14+
15+
def round_to_digits(value: str, digits: int) -> str:
16+
"""Round a numeric string to specified number of significant digits after decimal."""
17+
if digits <= 0:
18+
raise ValueError(f"digits must be positive, got {digits}")
19+
20+
d = Decimal(value)
21+
sign = '-' if d < 0 else ''
22+
d = abs(d)
23+
24+
# Find position of decimal point and first significant digit
25+
str_val = str(d)
26+
if '.' in str_val:
27+
int_part, frac_part = str_val.split('.')
28+
else:
29+
int_part, frac_part = str_val, ''
30+
31+
# For values like 0.00123, we need to count leading zeros in fraction
32+
if int_part == '0':
33+
leading_zeros = len(frac_part) - len(frac_part.lstrip('0'))
34+
# Round to `digits` significant figures
35+
quantize_exp = Decimal(10) ** (-(leading_zeros + digits))
36+
else:
37+
# For values >= 1, round to `digits` decimal places
38+
quantize_exp = Decimal(10) ** (-digits)
39+
40+
rounded = d.quantize(quantize_exp, rounding=ROUND_HALF_EVEN)
41+
return sign + str(rounded)
42+
43+
44+
def find_common_precision(a: str, b: str, min_sig_figs: int = 6) -> str:
45+
"""
46+
Find the longest decimal string that both values round to.
47+
48+
Returns the longest string S such that:
49+
- round(a, precision_of(S)) == S
50+
- round(b, precision_of(S)) == S
51+
52+
Args:
53+
a: First numeric string
54+
b: Second numeric string
55+
min_sig_figs: Minimum significant figures to keep (default 6)
56+
57+
Returns:
58+
The merged value string
59+
"""
60+
# Handle non-numeric values (headers, etc.)
61+
try:
62+
Decimal(a)
63+
Decimal(b)
64+
except:
65+
if a != b:
66+
raise ValueError(f"Non-numeric values don't match: {a!r} vs {b!r}")
67+
return a
68+
69+
# If they're exactly equal, return as-is
70+
if a == b:
71+
return a
72+
73+
# Find the decimal places in each
74+
def decimal_places(s: str) -> int:
75+
if '.' not in s:
76+
return 0
77+
return len(s.split('.')[1])
78+
79+
max_places = max(decimal_places(a), decimal_places(b))
80+
81+
# Try progressively fewer decimal places until both round to the same value
82+
for places in range(max_places, 0, -1):
83+
rounded_a = round_to_digits(a, places)
84+
rounded_b = round_to_digits(b, places)
85+
if rounded_a == rounded_b:
86+
# Verify we have enough significant figures
87+
sig_figs = len(rounded_a.replace('-', '').replace('.', '').lstrip('0'))
88+
if sig_figs >= min_sig_figs:
89+
return rounded_a
90+
91+
# If we can't find agreement, raise an error
92+
raise ValueError(
93+
f"Values diverge too much to merge with {min_sig_figs} sig figs:\n"
94+
f" a = {a}\n"
95+
f" b = {b}"
96+
)
97+
98+
99+
def merge_csvs(csv_a_path: Path, csv_b_path: Path, output_path: Path, skip_first_row_merge: bool = True):
100+
"""
101+
Merge two CSVs, finding common precision for each numeric cell.
102+
103+
Args:
104+
csv_a_path: First CSV (e.g., from Ubuntu)
105+
csv_b_path: Second CSV (e.g., from macOS)
106+
output_path: Where to write merged CSV
107+
skip_first_row_merge: If True, data row 0 (after header) is passed through
108+
without merging (for fixed initial values)
109+
"""
110+
with open(csv_a_path) as f_a, open(csv_b_path) as f_b:
111+
reader_a = csv.reader(f_a)
112+
reader_b = csv.reader(f_b)
113+
114+
rows_a = list(reader_a)
115+
rows_b = list(reader_b)
116+
117+
# Strict shape validation
118+
if len(rows_a) != len(rows_b):
119+
raise ValueError(
120+
f"Row count mismatch: {csv_a_path} has {len(rows_a)} rows, "
121+
f"{csv_b_path} has {len(rows_b)} rows"
122+
)
123+
124+
if not rows_a:
125+
raise ValueError("CSVs are empty")
126+
127+
# Validate header
128+
header_a, header_b = rows_a[0], rows_b[0]
129+
if header_a != header_b:
130+
raise ValueError(
131+
f"Header mismatch:\n"
132+
f" {csv_a_path}: {header_a}\n"
133+
f" {csv_b_path}: {header_b}"
134+
)
135+
136+
merged_rows = [header_a] # Keep header as-is
137+
138+
for row_idx, (row_a, row_b) in enumerate(zip(rows_a[1:], rows_b[1:]), start=1):
139+
if len(row_a) != len(row_b):
140+
raise ValueError(
141+
f"Column count mismatch at row {row_idx}: "
142+
f"{len(row_a)} vs {len(row_b)}"
143+
)
144+
145+
if len(row_a) != len(header_a):
146+
raise ValueError(
147+
f"Row {row_idx} has {len(row_a)} columns, header has {len(header_a)}"
148+
)
149+
150+
if skip_first_row_merge and row_idx == 1:
151+
# First data row: initial values, pass through from first file
152+
merged_rows.append(row_a)
153+
continue
154+
155+
merged_row = []
156+
for col_idx, (val_a, val_b) in enumerate(zip(row_a, row_b)):
157+
try:
158+
merged_val = find_common_precision(val_a, val_b)
159+
merged_row.append(merged_val)
160+
except ValueError as e:
161+
raise ValueError(
162+
f"Merge failed at row {row_idx}, column {col_idx} ({header_a[col_idx]}):\n{e}"
163+
) from e
164+
165+
merged_rows.append(merged_row)
166+
167+
# Write output
168+
output_path.parent.mkdir(parents=True, exist_ok=True)
169+
with open(output_path, 'w', newline='') as f:
170+
writer = csv.writer(f)
171+
writer.writerows(merged_rows)
172+
173+
print(f"Merged {len(merged_rows) - 1} data rows -> {output_path}")
174+
175+
176+
def main():
177+
if len(sys.argv) < 4:
178+
print(f"Usage: {sys.argv[0]} <csv_a> <csv_b> <output>", file=sys.stderr)
179+
print(f" {sys.argv[0]} --batch <ubuntu_dir> <macos_dir> <output_dir>", file=sys.stderr)
180+
sys.exit(1)
181+
182+
if sys.argv[1] == '--batch':
183+
if len(sys.argv) != 5:
184+
print("--batch requires: <ubuntu_dir> <macos_dir> <output_dir>", file=sys.stderr)
185+
sys.exit(1)
186+
187+
ubuntu_dir = Path(sys.argv[2])
188+
macos_dir = Path(sys.argv[3])
189+
output_dir = Path(sys.argv[4])
190+
191+
# Find all test directories
192+
ubuntu_csvs = sorted(ubuntu_dir.glob('*/linux.csv'))
193+
194+
for ubuntu_csv in ubuntu_csvs:
195+
test_name = ubuntu_csv.parent.name
196+
macos_csv = macos_dir / test_name / 'macos.csv'
197+
198+
if not macos_csv.exists():
199+
print(f"Warning: No macOS CSV for {test_name}, skipping", file=sys.stderr)
200+
continue
201+
202+
output_csv = output_dir / test_name / 'expected.csv'
203+
merge_csvs(ubuntu_csv, macos_csv, output_csv)
204+
else:
205+
csv_a = Path(sys.argv[1])
206+
csv_b = Path(sys.argv[2])
207+
output = Path(sys.argv[3])
208+
merge_csvs(csv_a, csv_b, output)
209+
210+
211+
if __name__ == '__main__':
212+
main()

0 commit comments

Comments
 (0)