|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | +Merge platform-specific expected value CSVs into a single file. |
| 4 | +
|
| 5 | +For each numeric value, finds the longest decimal representation that both |
| 6 | +platforms would round to, preserving only the precision where they agree. |
| 7 | +""" |
| 8 | + |
| 9 | +import csv |
| 10 | +import sys |
| 11 | +from decimal import Decimal, ROUND_HALF_EVEN |
| 12 | +from pathlib import Path |
| 13 | + |
| 14 | + |
| 15 | +def round_to_digits(value: str, digits: int) -> str: |
| 16 | + """Round a numeric string to specified number of significant digits after decimal.""" |
| 17 | + if digits <= 0: |
| 18 | + raise ValueError(f"digits must be positive, got {digits}") |
| 19 | + |
| 20 | + d = Decimal(value) |
| 21 | + sign = '-' if d < 0 else '' |
| 22 | + d = abs(d) |
| 23 | + |
| 24 | + # Find position of decimal point and first significant digit |
| 25 | + str_val = str(d) |
| 26 | + if '.' in str_val: |
| 27 | + int_part, frac_part = str_val.split('.') |
| 28 | + else: |
| 29 | + int_part, frac_part = str_val, '' |
| 30 | + |
| 31 | + # For values like 0.00123, we need to count leading zeros in fraction |
| 32 | + if int_part == '0': |
| 33 | + leading_zeros = len(frac_part) - len(frac_part.lstrip('0')) |
| 34 | + # Round to `digits` significant figures |
| 35 | + quantize_exp = Decimal(10) ** (-(leading_zeros + digits)) |
| 36 | + else: |
| 37 | + # For values >= 1, round to `digits` decimal places |
| 38 | + quantize_exp = Decimal(10) ** (-digits) |
| 39 | + |
| 40 | + rounded = d.quantize(quantize_exp, rounding=ROUND_HALF_EVEN) |
| 41 | + return sign + str(rounded) |
| 42 | + |
| 43 | + |
| 44 | +def find_common_precision(a: str, b: str, min_sig_figs: int = 6) -> str: |
| 45 | + """ |
| 46 | + Find the longest decimal string that both values round to. |
| 47 | +
|
| 48 | + Returns the longest string S such that: |
| 49 | + - round(a, precision_of(S)) == S |
| 50 | + - round(b, precision_of(S)) == S |
| 51 | +
|
| 52 | + Args: |
| 53 | + a: First numeric string |
| 54 | + b: Second numeric string |
| 55 | + min_sig_figs: Minimum significant figures to keep (default 6) |
| 56 | +
|
| 57 | + Returns: |
| 58 | + The merged value string |
| 59 | + """ |
| 60 | + # Handle non-numeric values (headers, etc.) |
| 61 | + try: |
| 62 | + Decimal(a) |
| 63 | + Decimal(b) |
| 64 | + except: |
| 65 | + if a != b: |
| 66 | + raise ValueError(f"Non-numeric values don't match: {a!r} vs {b!r}") |
| 67 | + return a |
| 68 | + |
| 69 | + # If they're exactly equal, return as-is |
| 70 | + if a == b: |
| 71 | + return a |
| 72 | + |
| 73 | + # Find the decimal places in each |
| 74 | + def decimal_places(s: str) -> int: |
| 75 | + if '.' not in s: |
| 76 | + return 0 |
| 77 | + return len(s.split('.')[1]) |
| 78 | + |
| 79 | + max_places = max(decimal_places(a), decimal_places(b)) |
| 80 | + |
| 81 | + # Try progressively fewer decimal places until both round to the same value |
| 82 | + for places in range(max_places, 0, -1): |
| 83 | + rounded_a = round_to_digits(a, places) |
| 84 | + rounded_b = round_to_digits(b, places) |
| 85 | + if rounded_a == rounded_b: |
| 86 | + # Verify we have enough significant figures |
| 87 | + sig_figs = len(rounded_a.replace('-', '').replace('.', '').lstrip('0')) |
| 88 | + if sig_figs >= min_sig_figs: |
| 89 | + return rounded_a |
| 90 | + |
| 91 | + # If we can't find agreement, raise an error |
| 92 | + raise ValueError( |
| 93 | + f"Values diverge too much to merge with {min_sig_figs} sig figs:\n" |
| 94 | + f" a = {a}\n" |
| 95 | + f" b = {b}" |
| 96 | + ) |
| 97 | + |
| 98 | + |
| 99 | +def merge_csvs(csv_a_path: Path, csv_b_path: Path, output_path: Path, skip_first_row_merge: bool = True): |
| 100 | + """ |
| 101 | + Merge two CSVs, finding common precision for each numeric cell. |
| 102 | +
|
| 103 | + Args: |
| 104 | + csv_a_path: First CSV (e.g., from Ubuntu) |
| 105 | + csv_b_path: Second CSV (e.g., from macOS) |
| 106 | + output_path: Where to write merged CSV |
| 107 | + skip_first_row_merge: If True, data row 0 (after header) is passed through |
| 108 | + without merging (for fixed initial values) |
| 109 | + """ |
| 110 | + with open(csv_a_path) as f_a, open(csv_b_path) as f_b: |
| 111 | + reader_a = csv.reader(f_a) |
| 112 | + reader_b = csv.reader(f_b) |
| 113 | + |
| 114 | + rows_a = list(reader_a) |
| 115 | + rows_b = list(reader_b) |
| 116 | + |
| 117 | + # Strict shape validation |
| 118 | + if len(rows_a) != len(rows_b): |
| 119 | + raise ValueError( |
| 120 | + f"Row count mismatch: {csv_a_path} has {len(rows_a)} rows, " |
| 121 | + f"{csv_b_path} has {len(rows_b)} rows" |
| 122 | + ) |
| 123 | + |
| 124 | + if not rows_a: |
| 125 | + raise ValueError("CSVs are empty") |
| 126 | + |
| 127 | + # Validate header |
| 128 | + header_a, header_b = rows_a[0], rows_b[0] |
| 129 | + if header_a != header_b: |
| 130 | + raise ValueError( |
| 131 | + f"Header mismatch:\n" |
| 132 | + f" {csv_a_path}: {header_a}\n" |
| 133 | + f" {csv_b_path}: {header_b}" |
| 134 | + ) |
| 135 | + |
| 136 | + merged_rows = [header_a] # Keep header as-is |
| 137 | + |
| 138 | + for row_idx, (row_a, row_b) in enumerate(zip(rows_a[1:], rows_b[1:]), start=1): |
| 139 | + if len(row_a) != len(row_b): |
| 140 | + raise ValueError( |
| 141 | + f"Column count mismatch at row {row_idx}: " |
| 142 | + f"{len(row_a)} vs {len(row_b)}" |
| 143 | + ) |
| 144 | + |
| 145 | + if len(row_a) != len(header_a): |
| 146 | + raise ValueError( |
| 147 | + f"Row {row_idx} has {len(row_a)} columns, header has {len(header_a)}" |
| 148 | + ) |
| 149 | + |
| 150 | + if skip_first_row_merge and row_idx == 1: |
| 151 | + # First data row: initial values, pass through from first file |
| 152 | + merged_rows.append(row_a) |
| 153 | + continue |
| 154 | + |
| 155 | + merged_row = [] |
| 156 | + for col_idx, (val_a, val_b) in enumerate(zip(row_a, row_b)): |
| 157 | + try: |
| 158 | + merged_val = find_common_precision(val_a, val_b) |
| 159 | + merged_row.append(merged_val) |
| 160 | + except ValueError as e: |
| 161 | + raise ValueError( |
| 162 | + f"Merge failed at row {row_idx}, column {col_idx} ({header_a[col_idx]}):\n{e}" |
| 163 | + ) from e |
| 164 | + |
| 165 | + merged_rows.append(merged_row) |
| 166 | + |
| 167 | + # Write output |
| 168 | + output_path.parent.mkdir(parents=True, exist_ok=True) |
| 169 | + with open(output_path, 'w', newline='') as f: |
| 170 | + writer = csv.writer(f) |
| 171 | + writer.writerows(merged_rows) |
| 172 | + |
| 173 | + print(f"Merged {len(merged_rows) - 1} data rows -> {output_path}") |
| 174 | + |
| 175 | + |
| 176 | +def main(): |
| 177 | + if len(sys.argv) < 4: |
| 178 | + print(f"Usage: {sys.argv[0]} <csv_a> <csv_b> <output>", file=sys.stderr) |
| 179 | + print(f" {sys.argv[0]} --batch <ubuntu_dir> <macos_dir> <output_dir>", file=sys.stderr) |
| 180 | + sys.exit(1) |
| 181 | + |
| 182 | + if sys.argv[1] == '--batch': |
| 183 | + if len(sys.argv) != 5: |
| 184 | + print("--batch requires: <ubuntu_dir> <macos_dir> <output_dir>", file=sys.stderr) |
| 185 | + sys.exit(1) |
| 186 | + |
| 187 | + ubuntu_dir = Path(sys.argv[2]) |
| 188 | + macos_dir = Path(sys.argv[3]) |
| 189 | + output_dir = Path(sys.argv[4]) |
| 190 | + |
| 191 | + # Find all test directories |
| 192 | + ubuntu_csvs = sorted(ubuntu_dir.glob('*/linux.csv')) |
| 193 | + |
| 194 | + for ubuntu_csv in ubuntu_csvs: |
| 195 | + test_name = ubuntu_csv.parent.name |
| 196 | + macos_csv = macos_dir / test_name / 'macos.csv' |
| 197 | + |
| 198 | + if not macos_csv.exists(): |
| 199 | + print(f"Warning: No macOS CSV for {test_name}, skipping", file=sys.stderr) |
| 200 | + continue |
| 201 | + |
| 202 | + output_csv = output_dir / test_name / 'expected.csv' |
| 203 | + merge_csvs(ubuntu_csv, macos_csv, output_csv) |
| 204 | + else: |
| 205 | + csv_a = Path(sys.argv[1]) |
| 206 | + csv_b = Path(sys.argv[2]) |
| 207 | + output = Path(sys.argv[3]) |
| 208 | + merge_csvs(csv_a, csv_b, output) |
| 209 | + |
| 210 | + |
| 211 | +if __name__ == '__main__': |
| 212 | + main() |
0 commit comments