A statement that may not answer its question is never kept as an example

CHANGELOG.md

@@ -103,6 +103,9 @@ below corresponds to one such version.
   example that mentions a table or column a statement reads, and the reconcile ledger puts those
   words beside every part that fell short, with a flag when two of them name different values for
   one column. Quoted, never graded. (ACE-119)
+- Reconcile Phase 1.5g reads a person's question beside their statement and writes
+  `question_fit.json`; the ledger's `question_fit` part withholds a doubtful row from the keep-offer,
+  and a fit that was never checked is an open part rather than a silent pass. (ACE-120)
 
 ### Fixed
 

plugins/agami/scripts/reconcile.py

@@ -621,6 +621,32 @@ def _first_number(rows, key: str) -> float | None:
         return None
 
 
+def _grade_question_fit(fit: Any, ran: bool) -> list[dict]:
+    """The one part graded by reading rather than measuring: the skill's Phase 1.5g judgment of
+    whether the statement answers the question it came with, written to `question_fit.json`. It can
+    withhold a row from the keep-offer and never proves anything about the semantic model. Expected
+    for every statement row after a run that succeeded, so a check that was never made is an open
+    part and not a silent pass."""
+    got, why = _usable(fit, "fit")
+    if got is None:
+        if not ran:
+            return []
+        return [_part("question_fit", UNRESOLVED, evidence={"file": "question_fit.json", "problem": why},
+                      note=f"question_fit.json {why}, so the fit of the statement to its question was not checked")]
+    word, reason = got.get("fit"), got.get("reason")
+    if word == "no_question":
+        return []
+    if word == "plausible":
+        return [_part("question_fit", CONFIRMED, evidence={"fit": word, "reason": reason},
+                      note="the statement plausibly answers the question, by reading; a judgment, not a measurement")]
+    if word == "doubtful":
+        return [_part("question_fit", UNRESOLVED, evidence={"fit": word, "reason": reason},
+                      note=f"the statement may not answer the question: {reason or 'no reason was given'}; "
+                           "reword the question or the statement and re-run this row")]
+    return [_part("question_fit", UNRESOLVED, evidence={"fit": word},
+                  note=f"question_fit.json carries an unknown fit {word!r}, so the fit was not checked")]
+
+
 def _grade_run(run: dict | None) -> list[dict]:
     if run is None:
         return [_part("runs", UNRESOLVED, note="no run record was found for the statement")]
@@ -1150,8 +1176,10 @@ def ledger(row_dir: Path, *, with_claims: bool = False) -> dict:
     # Optional: the semantic model's own words about what the statement reads. Absent, the ledger
     # grades exactly as it would have; present, they ride on the parts that fell short.
     mentions = _load_json(row_dir / "mentions.json")
+    fit = _load_json(row_dir / "question_fit.json")
 
     rows = _grade_run(run)
+    rows.extend(_grade_question_fit(fit, isinstance(run, dict) and run.get("status") == "ok"))
     # After a run that succeeded, every input the later steps write is expected. One that is absent,
     # empty, or an error object is a part of the statement that was NOT checked, said as such: the
     # alternative, grading only what is there, makes a crashed verb read as a clean statement.
@@ -1268,6 +1296,12 @@ def findings(run_dir: Path) -> dict:
         # grade, and bars nothing.
         own = [p for p in parts if p["part"] not in _CLAIM_PARTS]
         clean = bool(own) and all(p["verdict"] in (CONFIRMED, NOTED) for p in own)
+        # A `no_question` fit removes the `question_fit` part; against a row that carries a
+        # question, that is a contradiction and not a pass. The cross-check lives here because
+        # this is where the row record and the ledger meet.
+        if record.get("question") and record.get("statement") and graded is not None \
+                and not any(p["part"] == "question_fit" for p in parts):
+            clean = False
         if record.get("status") == "mismatch" and clean and record.get("question"):
             key = f"example:{_fold(record['question'])}"
             entry = grouped.setdefault(key, {"key": key, "kind": "example", "evidence": []})

plugins/agami/shared/evidence-row.md

@@ -91,7 +91,7 @@ always had stays, with the same meaning: `label`, `question`, `expected`, `actua
 | Status | When |
 |---|---|
 | `match` | the numbers agree within tolerance, and every graded part is `confirmed` (or there was no statement to grade) |
-| `match_unverified` | the numbers agree, but a part of the person's statement is not `confirmed`. Never offered in Phase 3e: a match nobody could verify may be luck |
+| `match_unverified` | the numbers agree, but a part of the person's statement is not `confirmed`. Never offered in Phase 3e: a match nobody could verify may be luck. A doubtful `question_fit` (Phase 1.5g: the statement may not answer its question) is such a part, so a sound statement paired with the wrong question is never kept as an example |
 | `mismatch` | the numbers differ and the person's statement has no `query_defect`, so the AI is the likelier culprit |
 | `expected_doubtful` | the numbers differ and the person's statement has a `query_defect`, so the expected value itself is in doubt. Kept out of the mismatch tally |
 | `error` | the row could not run; `sql` and `recorded` are `null`, as they always were |

plugins/agami/shared/part-ledger.md

@@ -34,16 +34,17 @@ downstream: a join that could not be graded leaves the fan-out check on its aggr
 | `literal:<t>.<c>=<v>` | `filter-values.judge.json` | the judge's grade, as it stands; a `model_gap` is of kind `description`, the column's list of values being stale. Every grade carries `declared` (`populated`, `empty`, `absent`), and a grade the warehouse decided over an undeclared column says so in its note |
 | `values_declared:<t>.<c>` | `filter-values.judge.json` `columns` | one per filtered column. `populated` → confirmed; `absent` or `empty` with the distinct probe `listed` (under 26 values) → model_gap of kind `description`, the same finding family as a stale list; `overflow` → noted, no list is expected of a wide column; `empty` → noted; `failed` or `not_run` → unresolved; a sensitive column → noted |
 | `dropped_rows:<a>-<b>` | `<join id>.dropped_rows.csv` | noted, never a grade: `<dropped> of <total> <left> rows have no <right> partner`, counted over the whole table before the statement's own filters, and naming the other side as not counted (an inner join drops from both); a probe planned but not run → noted, nothing claimed; no probe planned → no part |
+| `question_fit` | `question_fit.json`, the skill's Phase 1.5g reading of whether the statement answers its question | `plausible` → confirmed, by reading, and the note says so; `doubtful` → unresolved with the reason, so the row grades `match_unverified` at best and never reaches the keep-offer; `no_question` → no part, and only for a statement that came with no question: against a row that carries one it is a contradiction, and the findings verb and the report page refuse to keep such a row; absent after a run that succeeded → unresolved, the fit was not checked. The one part graded by judgment: it can withhold a row and never proves anything about the semantic model |
 | `predicates`, `date_window` | `claims.json`, only with `--with-claims` | `agrees` → confirmed; `differs` → unresolved, with both sides named; `unknown` → unresolved, except a `date_window` that is `null` on both sides when `unreadable` says both statements parsed and `temporal_predicates` is zero on both sides, which is confirmed (neither writes a date filter, so there is nothing to disagree about); a count above zero is a window written in a shape the reader does not fold, and stays open. A difference is reported, never judged here |
 
 **The verdict is the weakest part:** `query_defect` outranks `unresolved`, which outranks
 `model_gap`, which outranks `confirmed`. The counts travel with it so a reader sees what else was there.
 
 **An input that is not there is a part that was not checked.** After a run whose `run.json` says
-`ok`, the ledger expects `statement-prepare.json`, `statement-receipt.json`, `join-probes.json` and
-`filter-values.judge.json`. One that is absent, zero bytes, one JSON error line from a verb that
-exited non-zero, or JSON of another shape becomes one open part, `fan_out:*`, `receipt:*`, `join:*` or
-`literal:*`, whose evidence names the file and the problem. A verb that could not read the statement
+`ok`, the ledger expects `statement-prepare.json`, `statement-receipt.json`, `join-probes.json`,
+`filter-values.judge.json` and `question_fit.json`. One that is absent, zero bytes, one JSON error line from a verb that
+exited non-zero, or JSON of another shape becomes one open part, `fan_out:*`, `receipt:*`, `join:*`,
+`literal:*` or `question_fit`, whose evidence names the file and the problem. A verb that could not read the statement
 (`unreadable` set) opens `join:*` or `literal:*` the same way. Grading only what happened to be there
 would make a crashed verb read as a clean statement.
 
@@ -69,6 +70,7 @@ and writes `ledger.json` beside the inputs.
 | `<literal id>.exists.csv`, `<literal id>.exists_folded.csv` | the tier | one value's row count, and the folded near miss, run only when `exists` returned 0 |
 | `filter-values.judge.json` | `sm filter-values judge` | one grade per typed value |
 | `claims.json` | `sm claims` | the diff against the AI's own statement, once both exist |
+| `question_fit.json` | the skill, Phase 1.5g | `{"fit": "plausible" \| "doubtful" \| "no_question", "reason": "<one sentence, or null>"}`: whether the statement plausibly answers the question it came with, decided by reading |
 | `mentions.json` | `sm mentions --sql-file` | every description, caveat, glossary line, narrative paragraph and prompt example that mentions a table or column the statement reads, with a `values_named_differ` flag when two mentions about one column name different quoted values. Optional: absent, the ledger grades as before |
 | `receipt.json` | `sm receipt` | the receipt of the AI's statement |
 | `ledger.json` | `reconcile.py ledger` | the graded parts |

plugins/agami/shared/statement-check.md

@@ -64,6 +64,13 @@ Work in the row's directory, `<artifacts_dir>/local/reconcile/<ts>/rows/<n>/`. W
    its exit, rule and kind, so every execution and every refusal in this phase is written down. The
    AI's own run logs as `agami-query` Phase 5 always has.
 
+10. **Does the statement answer the question?** For every statement row, write `question_fit.json`;
+   when the row carries a question, read the two side by side first: `{"fit": "plausible" | "doubtful" | "no_question",
+   "reason": "<one sentence, or null>"}`. Doubtful when the grain differs, the measure differs, a
+   filter is present the question never asked for or absent when it did, or the time window differs.
+   `no_question` for a statement that came alone. This is the one step here that judges by reading;
+   the ledger turns a doubtful fit into an open part, and a missing file into one too.
+
 Then `python3 "$AGAMI_PLUGIN_ROOT/scripts/reconcile.py" ledger --row-dir .` grades what was found,
 and again with `--with-claims` once `sm claims` has compared the two statements.
 

plugins/agami/skills/agami-reconcile/SKILL.md

@@ -109,6 +109,7 @@ For every row that carries a `statement`. Skip this phase for a row that does no
 - **1.5d — Probes.** `sm join-probes` and `sm filter-values plan` emit SQL; write each emitted probe to its own `.sql` file and run it through the same tier by path, each to the CSV `part-ledger.md` names with a `.run.json` beside it, then `sm filter-values judge`. A probe the tier refuses or fails leaves an empty CSV; leave it there, the ledger reads it as a probe that failed. A file the ledger expects and does not find, or finds empty, is a part it grades `unresolved`, never clean.
 - **1.5e — The ledger.** `python3 "$AGAMI_PLUGIN_ROOT/scripts/reconcile.py" ledger --row-dir rows/<n>` writes `ledger.json`: one grade per part, `confirmed`, `model_gap`, `query_defect` or `unresolved`, and the weakest grade as the row's `ledger_verdict`; a part may also be `noted`, a fact the run states and never judges, which never decides the verdict. A part reaches `model_gap` only by measurement; the statement asserting something is never the evidence for it.
 - **1.5f — Its result is the expected value.** For a row that came with no number, `expected` is the single cell `statement.csv` returned (its text folded to a number by `reconcile.py diff`, which reads `$4.2M` and `47,238,221.00` alike), or the table's shape when it returned several rows. For a row that came with a tile number too, run `reconcile.py diff` between the tile and the statement's own result: a disagreement means the statement is not the tile's statement, or the data moved; flag the row in Phase 3b.5 and keep the tile's number as `expected`.
+- **1.5g — Does the statement answer the question?** For every statement row, write `question_fit.json`; when the row carries a question, read the two side by side first, before anything is compared. Doubtful when the grain differs (a count of items for a question about orders), the measure differs (revenue for a question about a count), a filter is present the question never asked for or absent when it did, or the time window differs. Write `question_fit.json` in the row directory: `{"fit": "plausible" | "doubtful" | "no_question", "reason": "<one sentence, or null>"}`, with `no_question` only for a statement that came alone: against a row that carries a question it is a contradiction, and the findings verb refuses to keep such a row. This is a judgment made by reading, the one part of the ledger that is; it can withhold a row from the keep-offer and never proves anything about the semantic model. A doubtful row grades `match_unverified` at best, shows in Phase 3b.5 with the reason, and the person settles it by rewording the question or the statement and re-running that row. Phase 3e keeps a `match` row as a worked example, which teaches the AI a question-to-SQL pairing, and a sound statement paired with the wrong question is the most harmful thing that step could keep.
 
 ---
 
@@ -317,6 +318,7 @@ Only when a row carried a statement, and only for the parts that did not grade `
 
 **What couldn't be checked**
 - Q3 Revenue, fan_out: SUM(total): the pre-flight could not bind this aggregate to one table: a column inside the aggregate could not be attributed to one table
+- Orders placed, question_fit: the statement may not answer the question: the question asks how many orders were placed and the statement counts order items; reword the question or the statement and re-run this row
 
 **What this run noticed**
 - Open items, dropped_rows: items-users: 3 of 8,345 items rows have no users partner and are dropped by this inner join; counted over the whole table, before the statement's own filters

tests/test_reconcile_flawed_inputs_e2e.py

@@ -113,8 +113,10 @@ def store(tmp_path_factory):
 # --- the chain, as the skill's Phase 1.5 walks it ----------------------------------------
 
 
-def grade_statement(store: dict, n: int, sql: str) -> dict:
-    """Phase 1.5 in code: run, receipt, probes, judge, ledger. Returns the ledger."""
+def grade_statement(store: dict, n: int, sql: str, *, fit: str = "plausible",
+                    fit_reason: str | None = None) -> dict:
+    """Phase 1.5 in code: run, receipt, probes, judge, the fit judgment, ledger. Returns the ledger.
+    `fit` stands in for the AI's Phase 1.5g reading of the question beside the statement."""
     db, root = store["db"], store["root"]
     row_dir = store["run"] / "rows" / str(n)
     row_dir.mkdir(parents=True, exist_ok=True)
@@ -170,6 +172,8 @@ def grade_statement(store: dict, n: int, sql: str) -> dict:
     (row_dir / "filter-values.judge.json").write_text(json.dumps(_sm(
         "filter-values", "judge", str(root), "--plan", str(row_dir / "filter-values.plan.json"),
         "--results", str(row_dir))))
+    # 1.5g: whether the statement answers its question, a judgment the skill writes down.
+    (row_dir / "question_fit.json").write_text(json.dumps({"fit": fit, "reason": fit_reason}))
     return reconcile.ledger(row_dir)
 
 
@@ -454,6 +458,32 @@ def test_15_a_join_that_really_drops_rows_says_how_many_and_which_side_it_did_no
     assert noted["evidence"]["total"] > noted["evidence"]["dropped"] > 0, noted["evidence"]
     assert f"{noted['evidence']['dropped']} of {noted['evidence']['total']} orders rows have no payments partner" in noted["note"]
     assert "rows of payments with no orders partner were not counted" in noted["note"]
+def test_14_a_sound_statement_paired_with_the_wrong_question_is_never_kept(store):
+    """The statement is right and every measured part holds, but it counts order items while the
+    question asks about orders. The skill's reading writes a doubtful fit; the ledger holds the row
+    open; a matching number becomes `match_unverified` and never reaches the keep-offer."""
+    sql = ("SELECT COUNT(*) AS n FROM order_items oi JOIN orders o ON o.id = oi.order_id "
+           "WHERE o.status != 'cancelled'")
+    ledger = grade_statement(store, 14, sql, fit="doubtful",
+                             fit_reason="the question asks how many orders were placed; the statement counts order items")
+    parts = _parts(ledger)
+    assert parts["fan_out:COUNT(*)"]["verdict"] == "confirmed"
+    assert parts["question_fit"]["verdict"] == "unresolved" and "counts order items" in parts["question_fit"]["note"]
+    expected = scalar((store["run"] / "rows" / "14" / "statement.csv").read_text())
+    actual = ask_agami(store, 14, sql)
+    diff, ledger = compare(store, 14, expected, actual)
+    rec = record(store, 14, "How many orders were placed?", sql, expected, diff, ledger)
+    assert diff["match"] is True and rec["status"] == "match_unverified"
+    # The fit is the ONLY open part, so it alone is what withheld the row.
+    open_parts = {p for p, r in _parts(ledger).items() if r["verdict"] not in ("confirmed", "noted")}
+    assert open_parts == {"question_fit"}, open_parts
+    # The same statement read as a plausible fit reaches match.
+    twin = grade_statement(store, 16, sql, fit="plausible")
+    expected = scalar((store["run"] / "rows" / "16" / "statement.csv").read_text())
+    actual = ask_agami(store, 16, sql)
+    diff, twin = compare(store, 16, expected, actual)
+    rec = record(store, 16, "How many order items were placed?", sql, expected, diff, twin)
+    assert rec["status"] == "match"
 
 
 def test_8_the_profile_was_never_written_to(store):

tests/test_reconcile_learning_loop_skill.py

@@ -171,6 +171,17 @@ def test_the_summary_gains_a_second_line_and_the_statements_get_their_own_table(
     assert "which is right is the person's call, never the ledger's" in statements
     assert "| `mentions.json` |" in REFERENCES["part-ledger.md"]
     assert "mentions" in REFERENCES["statement-check.md"]
+    # 1.5g: the one judgment in the ledger, named as such, with its four doubt signals and its file.
+    fit = PHASE_1_5.split("**1.5g")[1]
+    for signal in ("grain differs", "measure differs", "a filter is present the question never asked for", "time window differs"):
+        assert signal in fit, signal
+    assert "`question_fit.json`" in fit and "a judgment made by reading" in fit
+    assert "never proves anything about the semantic model" in fit
+    assert "question_fit:" in statements and "reword the question or the statement and re-run this row" in statements
+    assert "For every statement row, write `question_fit.json`" in PHASE_1_5
+    assert "only for a statement that came alone" in PHASE_1_5
+    assert "| `question_fit` |" in REFERENCES["part-ledger.md"]
+    assert "question_fit.json" in REFERENCES["statement-check.md"] and "question_fit" in REFERENCES["evidence-row.md"]
 
 
 def test_hard_rule_3_gains_the_findings_file_and_keeps_every_pin():