reconcile: identical column names are one column set when the row counts differ

Spec: ACE-128

Sandeep: "Identical column names read as a column difference. Both sides return exactly ['department',
'pending_items'], yet columns grades defect. Because the row counts differ (13 vs 759), value-based
pairing fails, and the new 'compare by values, never by name' rule then reports identical names as
different columns." Two vectors of different length are never equal, so with differing counts every
column read unpaired: a fact about the counts, not the columns. The comparator now reports no pairs
and no extras when the counts differ, and the items verb compares columns by data only when a values
comparison ran (same counts, and a pairing or an unmatched golden column); otherwise it falls back to
names, so identical names are one column set and the rows check alone carries the difference.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Author: sandeep-agami

packages/agami-core/src/semantic_model/comparator.py

@@ -642,6 +642,11 @@ def _column_pairs(
     reports nothing rather than failing a score that already ran."""
     if match not in ("exact", "values") or not golden.rows or not generated.rows:
         return (), ()
+    if len(golden.rows) != len(generated.rows):
+        # Pairing is by value vectors, and two vectors of different length are never equal, so
+        # every column would read unpaired: not a fact about the columns, only about the counts,
+        # which the score already reports. Nothing is claimed here.
+        return (), ()
     try:
         pairing, _unmatched = match_columns(
             golden.columns, golden.rows, generated.columns, generated.rows,

plugins/agami/scripts/reconcile.py

@@ -1713,13 +1713,17 @@ def add(key, state, yours=None, agami=None, note=None, yours_hi=None, agami_hi=N
         ac = list(((rec.get("recorded") or {}).get("columns")) or [])
         pairs = [tuple(p) for p in (result_set.get("column_pairs") or []) if isinstance(p, (list, tuple)) and len(p) == 2]
         if yc or ac:
-            if pairs or "unmatched_generated_columns" in result_set:
+            same_count = result_set.get("golden_row_count") == result_set.get("generated_row_count")
+            values_compared = same_count and (bool(pairs) or bool(result_set.get("unmatched_golden_columns")))
+            if values_compared:
                 # Columns are compared by the values they carry, never by name: the comparator says
                 # which of yours paired with which of agami's, and a renamed column is the same column.
-                only_yours = [c for c in (result_set.get("unmatched_golden_columns") or []) if c in yc] or [c for c in yc if c not in {p[0] for p in pairs}]
+                only_yours = [c for c in (result_set.get("unmatched_golden_columns") or []) if c in yc]
                 only_agami = list(result_set.get("unmatched_generated_columns") or [])
                 renamed = [f"{a} → {b}" for a, b in pairs if a != b]
-            else:  # an older score file: names are all there is
+            else:
+                # No values comparison ran (the row counts differ, or an older score file): names are
+                # all there is. Identical names are one column set; the rows check carries the counts.
                 only_yours, only_agami, renamed = _only(yc, ac), _only(ac, yc), []
             if not only_yours and not only_agami:
                 col_state = "held"

tests/test_comparator_column_pairs.py

@@ -27,3 +27,11 @@ def test_a_scalar_pair_serialises_and_other_levels_report_nothing():
     assert list(map(list, d["column_pairs"])) == [["revenue", "total"]] and d["unmatched_generated_columns"] == ()
     shape = compare_result_sets(ExecResult(columns=["a"], rows=[(1,)]), ExecResult(columns=["b"], rows=[(2,)]), match="shape")
     assert shape.column_pairs == () and shape.unmatched_generated_columns == ()
+
+
+def test_different_row_counts_report_no_pairs_and_no_extras():
+    golden = ExecResult(columns=["department", "pending_items"], rows=[("a", 1), ("b", 2)])
+    generated = ExecResult(columns=["department", "pending_items"], rows=[("a", 1), ("b", 2), ("c", 3)])
+    score = compare_result_sets(golden, generated, match="values")
+    assert score.accuracy == 0.0 and score.column_pairs == () and score.unmatched_generated_columns == () and score.unmatched_golden_columns == ()
+

tests/test_reconcile_result_and_fix.py

@@ -125,3 +125,18 @@ def test_the_change_text_the_prefill_and_the_fix_come_from_one_source(tmp_path):
     defect = items[2]
     assert defect["fix"] == "query" and defect["prefill"]["fix"] == "value orders.status='Delivered'"
 
+
+def test_identical_column_names_with_different_row_counts_are_one_column_set(tmp_path):
+    """Values cannot pair when the row counts differ, so the columns fall back to names: identical
+    names read as the same columns, and only the rows check carries the difference."""
+    rec = dict(TABLE_DIFF, row=1, statement_recorded={"columns": ["department", "pending_items"], "row_count": 13},
+               recorded={"columns": ["department", "pending_items"], "rows": []},
+               comparison={"result_set": {"accuracy": 0.0, "reason": "the answer key has 13 rows and the generated result has 759",
+                                          "unmatched_golden_columns": [], "column_pairs": [], "unmatched_generated_columns": [],
+                                          "golden_row_count": 13, "generated_row_count": 759}})
+    item = reconcile.report_items(_run(tmp_path, [rec]))[0]
+    rows = {r["key"]: r for r in item["diff"]}
+    assert rows["rows"]["state"] == "defect" and rows["rows"]["yours"] == "13 rows" and rows["rows"]["agami"] == "759 rows"
+    assert rows["columns"]["state"] == "held" and not rows["columns"].get("yours_hi") and not rows["columns"].get("agami_hi")
+    assert item["result"]["data"] == "differs" and "columns" not in item["result"]["differs_in"]
+