diff --git a/CHANGELOG.md b/CHANGELOG.md index 35bfb3df..1a45103d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,6 +103,9 @@ below corresponds to one such version. example that mentions a table or column a statement reads, and the reconcile ledger puts those words beside every part that fell short, with a flag when two of them name different values for one column. Quoted, never graded. (ACE-119) +- Reconcile Phase 1.5g reads a person's question beside their statement and writes + `question_fit.json`; the ledger's `question_fit` part withholds a doubtful row from the keep-offer, + and a fit that was never checked is an open part rather than a silent pass. (ACE-120) ### Fixed diff --git a/plugins/agami/scripts/reconcile.py b/plugins/agami/scripts/reconcile.py index bb43b3cc..fa3c91d8 100644 --- a/plugins/agami/scripts/reconcile.py +++ b/plugins/agami/scripts/reconcile.py @@ -621,6 +621,32 @@ def _first_number(rows, key: str) -> float | None: return None +def _grade_question_fit(fit: Any, ran: bool) -> list[dict]: + """The one part graded by reading rather than measuring: the skill's Phase 1.5g judgment of + whether the statement answers the question it came with, written to `question_fit.json`. It can + withhold a row from the keep-offer and never proves anything about the semantic model. Expected + for every statement row after a run that succeeded, so a check that was never made is an open + part and not a silent pass.""" + got, why = _usable(fit, "fit") + if got is None: + if not ran: + return [] + return [_part("question_fit", UNRESOLVED, evidence={"file": "question_fit.json", "problem": why}, + note=f"question_fit.json {why}, so the fit of the statement to its question was not checked")] + word, reason = got.get("fit"), got.get("reason") + if word == "no_question": + return [] + if word == "plausible": + return [_part("question_fit", CONFIRMED, evidence={"fit": word, "reason": reason}, + note="the statement plausibly answers the question, by reading; a judgment, not a measurement")] + if word == "doubtful": + return [_part("question_fit", UNRESOLVED, evidence={"fit": word, "reason": reason}, + note=f"the statement may not answer the question: {reason or 'no reason was given'}; " + "reword the question or the statement and re-run this row")] + return [_part("question_fit", UNRESOLVED, evidence={"fit": word}, + note=f"question_fit.json carries an unknown fit {word!r}, so the fit was not checked")] + + def _grade_run(run: dict | None) -> list[dict]: if run is None: return [_part("runs", UNRESOLVED, note="no run record was found for the statement")] @@ -1150,8 +1176,10 @@ def ledger(row_dir: Path, *, with_claims: bool = False) -> dict: # Optional: the semantic model's own words about what the statement reads. Absent, the ledger # grades exactly as it would have; present, they ride on the parts that fell short. mentions = _load_json(row_dir / "mentions.json") + fit = _load_json(row_dir / "question_fit.json") rows = _grade_run(run) + rows.extend(_grade_question_fit(fit, isinstance(run, dict) and run.get("status") == "ok")) # After a run that succeeded, every input the later steps write is expected. One that is absent, # empty, or an error object is a part of the statement that was NOT checked, said as such: the # alternative, grading only what is there, makes a crashed verb read as a clean statement. @@ -1268,6 +1296,12 @@ def findings(run_dir: Path) -> dict: # grade, and bars nothing. own = [p for p in parts if p["part"] not in _CLAIM_PARTS] clean = bool(own) and all(p["verdict"] in (CONFIRMED, NOTED) for p in own) + # A `no_question` fit removes the `question_fit` part; against a row that carries a + # question, that is a contradiction and not a pass. The cross-check lives here because + # this is where the row record and the ledger meet. + if record.get("question") and record.get("statement") and graded is not None \ + and not any(p["part"] == "question_fit" for p in parts): + clean = False if record.get("status") == "mismatch" and clean and record.get("question"): key = f"example:{_fold(record['question'])}" entry = grouped.setdefault(key, {"key": key, "kind": "example", "evidence": []}) diff --git a/plugins/agami/shared/evidence-row.md b/plugins/agami/shared/evidence-row.md index 9b71b61b..056b8b11 100644 --- a/plugins/agami/shared/evidence-row.md +++ b/plugins/agami/shared/evidence-row.md @@ -91,7 +91,7 @@ always had stays, with the same meaning: `label`, `question`, `expected`, `actua | Status | When | |---|---| | `match` | the numbers agree within tolerance, and every graded part is `confirmed` (or there was no statement to grade) | -| `match_unverified` | the numbers agree, but a part of the person's statement is not `confirmed`. Never offered in Phase 3e: a match nobody could verify may be luck | +| `match_unverified` | the numbers agree, but a part of the person's statement is not `confirmed`. Never offered in Phase 3e: a match nobody could verify may be luck. A doubtful `question_fit` (Phase 1.5g: the statement may not answer its question) is such a part, so a sound statement paired with the wrong question is never kept as an example | | `mismatch` | the numbers differ and the person's statement has no `query_defect`, so the AI is the likelier culprit | | `expected_doubtful` | the numbers differ and the person's statement has a `query_defect`, so the expected value itself is in doubt. Kept out of the mismatch tally | | `error` | the row could not run; `sql` and `recorded` are `null`, as they always were | diff --git a/plugins/agami/shared/part-ledger.md b/plugins/agami/shared/part-ledger.md index 3879b289..7670055a 100644 --- a/plugins/agami/shared/part-ledger.md +++ b/plugins/agami/shared/part-ledger.md @@ -34,16 +34,17 @@ downstream: a join that could not be graded leaves the fan-out check on its aggr | `literal:.=` | `filter-values.judge.json` | the judge's grade, as it stands; a `model_gap` is of kind `description`, the column's list of values being stale. Every grade carries `declared` (`populated`, `empty`, `absent`), and a grade the warehouse decided over an undeclared column says so in its note | | `values_declared:.` | `filter-values.judge.json` `columns` | one per filtered column. `populated` → confirmed; `absent` or `empty` with the distinct probe `listed` (under 26 values) → model_gap of kind `description`, the same finding family as a stale list; `overflow` → noted, no list is expected of a wide column; `empty` → noted; `failed` or `not_run` → unresolved; a sensitive column → noted | | `dropped_rows:-` | `.dropped_rows.csv` | noted, never a grade: ` of rows have no partner`, counted over the whole table before the statement's own filters, and naming the other side as not counted (an inner join drops from both); a probe planned but not run → noted, nothing claimed; no probe planned → no part | +| `question_fit` | `question_fit.json`, the skill's Phase 1.5g reading of whether the statement answers its question | `plausible` → confirmed, by reading, and the note says so; `doubtful` → unresolved with the reason, so the row grades `match_unverified` at best and never reaches the keep-offer; `no_question` → no part, and only for a statement that came with no question: against a row that carries one it is a contradiction, and the findings verb and the report page refuse to keep such a row; absent after a run that succeeded → unresolved, the fit was not checked. The one part graded by judgment: it can withhold a row and never proves anything about the semantic model | | `predicates`, `date_window` | `claims.json`, only with `--with-claims` | `agrees` → confirmed; `differs` → unresolved, with both sides named; `unknown` → unresolved, except a `date_window` that is `null` on both sides when `unreadable` says both statements parsed and `temporal_predicates` is zero on both sides, which is confirmed (neither writes a date filter, so there is nothing to disagree about); a count above zero is a window written in a shape the reader does not fold, and stays open. A difference is reported, never judged here | **The verdict is the weakest part:** `query_defect` outranks `unresolved`, which outranks `model_gap`, which outranks `confirmed`. The counts travel with it so a reader sees what else was there. **An input that is not there is a part that was not checked.** After a run whose `run.json` says -`ok`, the ledger expects `statement-prepare.json`, `statement-receipt.json`, `join-probes.json` and -`filter-values.judge.json`. One that is absent, zero bytes, one JSON error line from a verb that -exited non-zero, or JSON of another shape becomes one open part, `fan_out:*`, `receipt:*`, `join:*` or -`literal:*`, whose evidence names the file and the problem. A verb that could not read the statement +`ok`, the ledger expects `statement-prepare.json`, `statement-receipt.json`, `join-probes.json`, +`filter-values.judge.json` and `question_fit.json`. One that is absent, zero bytes, one JSON error line from a verb that +exited non-zero, or JSON of another shape becomes one open part, `fan_out:*`, `receipt:*`, `join:*`, +`literal:*` or `question_fit`, whose evidence names the file and the problem. A verb that could not read the statement (`unreadable` set) opens `join:*` or `literal:*` the same way. Grading only what happened to be there would make a crashed verb read as a clean statement. @@ -69,6 +70,7 @@ and writes `ledger.json` beside the inputs. | `.exists.csv`, `.exists_folded.csv` | the tier | one value's row count, and the folded near miss, run only when `exists` returned 0 | | `filter-values.judge.json` | `sm filter-values judge` | one grade per typed value | | `claims.json` | `sm claims` | the diff against the AI's own statement, once both exist | +| `question_fit.json` | the skill, Phase 1.5g | `{"fit": "plausible" \| "doubtful" \| "no_question", "reason": ""}`: whether the statement plausibly answers the question it came with, decided by reading | | `mentions.json` | `sm mentions --sql-file` | every description, caveat, glossary line, narrative paragraph and prompt example that mentions a table or column the statement reads, with a `values_named_differ` flag when two mentions about one column name different quoted values. Optional: absent, the ledger grades as before | | `receipt.json` | `sm receipt` | the receipt of the AI's statement | | `ledger.json` | `reconcile.py ledger` | the graded parts | diff --git a/plugins/agami/shared/statement-check.md b/plugins/agami/shared/statement-check.md index 5bbf3a1d..b4b124f7 100644 --- a/plugins/agami/shared/statement-check.md +++ b/plugins/agami/shared/statement-check.md @@ -64,6 +64,13 @@ Work in the row's directory, `/local/reconcile//rows//`. W its exit, rule and kind, so every execution and every refusal in this phase is written down. The AI's own run logs as `agami-query` Phase 5 always has. +10. **Does the statement answer the question?** For every statement row, write `question_fit.json`; + when the row carries a question, read the two side by side first: `{"fit": "plausible" | "doubtful" | "no_question", + "reason": ""}`. Doubtful when the grain differs, the measure differs, a + filter is present the question never asked for or absent when it did, or the time window differs. + `no_question` for a statement that came alone. This is the one step here that judges by reading; + the ledger turns a doubtful fit into an open part, and a missing file into one too. + Then `python3 "$AGAMI_PLUGIN_ROOT/scripts/reconcile.py" ledger --row-dir .` grades what was found, and again with `--with-claims` once `sm claims` has compared the two statements. diff --git a/plugins/agami/skills/agami-reconcile/SKILL.md b/plugins/agami/skills/agami-reconcile/SKILL.md index bd198a42..22564f90 100644 --- a/plugins/agami/skills/agami-reconcile/SKILL.md +++ b/plugins/agami/skills/agami-reconcile/SKILL.md @@ -109,6 +109,7 @@ For every row that carries a `statement`. Skip this phase for a row that does no - **1.5d — Probes.** `sm join-probes` and `sm filter-values plan` emit SQL; write each emitted probe to its own `.sql` file and run it through the same tier by path, each to the CSV `part-ledger.md` names with a `.run.json` beside it, then `sm filter-values judge`. A probe the tier refuses or fails leaves an empty CSV; leave it there, the ledger reads it as a probe that failed. A file the ledger expects and does not find, or finds empty, is a part it grades `unresolved`, never clean. - **1.5e — The ledger.** `python3 "$AGAMI_PLUGIN_ROOT/scripts/reconcile.py" ledger --row-dir rows/` writes `ledger.json`: one grade per part, `confirmed`, `model_gap`, `query_defect` or `unresolved`, and the weakest grade as the row's `ledger_verdict`; a part may also be `noted`, a fact the run states and never judges, which never decides the verdict. A part reaches `model_gap` only by measurement; the statement asserting something is never the evidence for it. - **1.5f — Its result is the expected value.** For a row that came with no number, `expected` is the single cell `statement.csv` returned (its text folded to a number by `reconcile.py diff`, which reads `$4.2M` and `47,238,221.00` alike), or the table's shape when it returned several rows. For a row that came with a tile number too, run `reconcile.py diff` between the tile and the statement's own result: a disagreement means the statement is not the tile's statement, or the data moved; flag the row in Phase 3b.5 and keep the tile's number as `expected`. +- **1.5g — Does the statement answer the question?** For every statement row, write `question_fit.json`; when the row carries a question, read the two side by side first, before anything is compared. Doubtful when the grain differs (a count of items for a question about orders), the measure differs (revenue for a question about a count), a filter is present the question never asked for or absent when it did, or the time window differs. Write `question_fit.json` in the row directory: `{"fit": "plausible" | "doubtful" | "no_question", "reason": ""}`, with `no_question` only for a statement that came alone: against a row that carries a question it is a contradiction, and the findings verb refuses to keep such a row. This is a judgment made by reading, the one part of the ledger that is; it can withhold a row from the keep-offer and never proves anything about the semantic model. A doubtful row grades `match_unverified` at best, shows in Phase 3b.5 with the reason, and the person settles it by rewording the question or the statement and re-running that row. Phase 3e keeps a `match` row as a worked example, which teaches the AI a question-to-SQL pairing, and a sound statement paired with the wrong question is the most harmful thing that step could keep. --- @@ -317,6 +318,7 @@ Only when a row carried a statement, and only for the parts that did not grade ` **What couldn't be checked** - Q3 Revenue, fan_out: SUM(total): the pre-flight could not bind this aggregate to one table: a column inside the aggregate could not be attributed to one table +- Orders placed, question_fit: the statement may not answer the question: the question asks how many orders were placed and the statement counts order items; reword the question or the statement and re-run this row **What this run noticed** - Open items, dropped_rows: items-users: 3 of 8,345 items rows have no users partner and are dropped by this inner join; counted over the whole table, before the statement's own filters diff --git a/tests/test_reconcile_flawed_inputs_e2e.py b/tests/test_reconcile_flawed_inputs_e2e.py index ba074c32..a4598ebf 100644 --- a/tests/test_reconcile_flawed_inputs_e2e.py +++ b/tests/test_reconcile_flawed_inputs_e2e.py @@ -113,8 +113,10 @@ def store(tmp_path_factory): # --- the chain, as the skill's Phase 1.5 walks it ---------------------------------------- -def grade_statement(store: dict, n: int, sql: str) -> dict: - """Phase 1.5 in code: run, receipt, probes, judge, ledger. Returns the ledger.""" +def grade_statement(store: dict, n: int, sql: str, *, fit: str = "plausible", + fit_reason: str | None = None) -> dict: + """Phase 1.5 in code: run, receipt, probes, judge, the fit judgment, ledger. Returns the ledger. + `fit` stands in for the AI's Phase 1.5g reading of the question beside the statement.""" db, root = store["db"], store["root"] row_dir = store["run"] / "rows" / str(n) row_dir.mkdir(parents=True, exist_ok=True) @@ -170,6 +172,8 @@ def grade_statement(store: dict, n: int, sql: str) -> dict: (row_dir / "filter-values.judge.json").write_text(json.dumps(_sm( "filter-values", "judge", str(root), "--plan", str(row_dir / "filter-values.plan.json"), "--results", str(row_dir)))) + # 1.5g: whether the statement answers its question, a judgment the skill writes down. + (row_dir / "question_fit.json").write_text(json.dumps({"fit": fit, "reason": fit_reason})) return reconcile.ledger(row_dir) @@ -454,6 +458,32 @@ def test_15_a_join_that_really_drops_rows_says_how_many_and_which_side_it_did_no assert noted["evidence"]["total"] > noted["evidence"]["dropped"] > 0, noted["evidence"] assert f"{noted['evidence']['dropped']} of {noted['evidence']['total']} orders rows have no payments partner" in noted["note"] assert "rows of payments with no orders partner were not counted" in noted["note"] +def test_14_a_sound_statement_paired_with_the_wrong_question_is_never_kept(store): + """The statement is right and every measured part holds, but it counts order items while the + question asks about orders. The skill's reading writes a doubtful fit; the ledger holds the row + open; a matching number becomes `match_unverified` and never reaches the keep-offer.""" + sql = ("SELECT COUNT(*) AS n FROM order_items oi JOIN orders o ON o.id = oi.order_id " + "WHERE o.status != 'cancelled'") + ledger = grade_statement(store, 14, sql, fit="doubtful", + fit_reason="the question asks how many orders were placed; the statement counts order items") + parts = _parts(ledger) + assert parts["fan_out:COUNT(*)"]["verdict"] == "confirmed" + assert parts["question_fit"]["verdict"] == "unresolved" and "counts order items" in parts["question_fit"]["note"] + expected = scalar((store["run"] / "rows" / "14" / "statement.csv").read_text()) + actual = ask_agami(store, 14, sql) + diff, ledger = compare(store, 14, expected, actual) + rec = record(store, 14, "How many orders were placed?", sql, expected, diff, ledger) + assert diff["match"] is True and rec["status"] == "match_unverified" + # The fit is the ONLY open part, so it alone is what withheld the row. + open_parts = {p for p, r in _parts(ledger).items() if r["verdict"] not in ("confirmed", "noted")} + assert open_parts == {"question_fit"}, open_parts + # The same statement read as a plausible fit reaches match. + twin = grade_statement(store, 16, sql, fit="plausible") + expected = scalar((store["run"] / "rows" / "16" / "statement.csv").read_text()) + actual = ask_agami(store, 16, sql) + diff, twin = compare(store, 16, expected, actual) + rec = record(store, 16, "How many order items were placed?", sql, expected, diff, twin) + assert rec["status"] == "match" def test_8_the_profile_was_never_written_to(store): diff --git a/tests/test_reconcile_learning_loop_skill.py b/tests/test_reconcile_learning_loop_skill.py index c17a6c6d..e22e3fb6 100644 --- a/tests/test_reconcile_learning_loop_skill.py +++ b/tests/test_reconcile_learning_loop_skill.py @@ -171,6 +171,17 @@ def test_the_summary_gains_a_second_line_and_the_statements_get_their_own_table( assert "which is right is the person's call, never the ledger's" in statements assert "| `mentions.json` |" in REFERENCES["part-ledger.md"] assert "mentions" in REFERENCES["statement-check.md"] + # 1.5g: the one judgment in the ledger, named as such, with its four doubt signals and its file. + fit = PHASE_1_5.split("**1.5g")[1] + for signal in ("grain differs", "measure differs", "a filter is present the question never asked for", "time window differs"): + assert signal in fit, signal + assert "`question_fit.json`" in fit and "a judgment made by reading" in fit + assert "never proves anything about the semantic model" in fit + assert "question_fit:" in statements and "reword the question or the statement and re-run this row" in statements + assert "For every statement row, write `question_fit.json`" in PHASE_1_5 + assert "only for a statement that came alone" in PHASE_1_5 + assert "| `question_fit` |" in REFERENCES["part-ledger.md"] + assert "question_fit.json" in REFERENCES["statement-check.md"] and "question_fit" in REFERENCES["evidence-row.md"] def test_hard_rule_3_gains_the_findings_file_and_keeps_every_pin(): diff --git a/tests/test_reconcile_ledger.py b/tests/test_reconcile_ledger.py index 1b71dc46..e69cf8c7 100644 --- a/tests/test_reconcile_ledger.py +++ b/tests/test_reconcile_ledger.py @@ -84,6 +84,7 @@ def _complete(row_dir: Path) -> None: _write(row_dir, "statement-receipt.json", _receipt()) _write(row_dir, "join-probes.json", _no_joins()) _write(row_dir, "filter-values.judge.json", _no_literals()) + _write(row_dir, "question_fit.json", {"fit": "plausible", "reason": None}) def _join_probe(a: str, ac: str, b: str, bc: str, *, declared_between: bool, matches: bool, @@ -145,7 +146,7 @@ def test_a_statement_that_ran_clean_is_confirmed_on_every_part(tmp_path): assert _parts(result)["metric:total"]["evidence"] == {"metric": "revenue"} assert result["verdict"] == "confirmed" assert {row["verdict"] for row in result["rows"]} == {"confirmed"} - assert set(_parts(result)) == {"runs", "scope", "fan_out:SUM(total)", "aggregation:SUM(total)", + assert set(_parts(result)) == {"runs", "scope", "question_fit", "fan_out:SUM(total)", "aggregation:SUM(total)", "default_filter:orders:orders.deleted_at IS NULL", "metric:total"} @@ -572,7 +573,7 @@ def test_a_run_that_failed_expects_no_later_files(tmp_path): def test_a_complete_clean_row_has_no_open_part(tmp_path): _complete(tmp_path) result = ledger(tmp_path) - assert result["verdict"] == "confirmed" and set(_parts(result)) == {"runs", "scope"} + assert result["verdict"] == "confirmed" and set(_parts(result)) == {"runs", "scope", "question_fit"} def test_an_output_column_the_receipt_could_not_settle_is_open_not_a_gap(tmp_path): @@ -966,3 +967,53 @@ def test_words_that_could_not_be_read_are_noted_not_silently_missing(tmp_path): assert "prose:*" not in _parts(ledger(tmp_path)) (tmp_path / "mentions.json").unlink() assert "prose:*" not in _parts(ledger(tmp_path)) +# --- question fit: the one part graded by reading ----------------------------------------------- + + +def test_a_plausible_fit_is_confirmed_and_says_it_was_read_not_measured(tmp_path): + _complete(tmp_path) + row = _parts(ledger(tmp_path))["question_fit"] + assert row["verdict"] == "confirmed" and "by reading" in row["note"] + + +def test_a_doubtful_fit_holds_the_row_open_with_the_reason(tmp_path): + _complete(tmp_path) + _write(tmp_path, "question_fit.json", {"fit": "doubtful", + "reason": "the question asks about orders and the statement counts items"}) + result = ledger(tmp_path) + row = _parts(result)["question_fit"] + assert row["verdict"] == "unresolved" and "counts items" in row["note"] and "re-run this row" in row["note"] + assert result["verdict"] == "unresolved" + # Which is what keeps a matching number away from the keep-offer. + assert reconcile.row_status(True, result["verdict"]) == "match_unverified" + + +def test_a_statement_that_came_with_no_question_has_no_fit_to_grade(tmp_path): + _complete(tmp_path) + _write(tmp_path, "question_fit.json", {"fit": "no_question", "reason": None}) + assert "question_fit" not in _parts(ledger(tmp_path)) + + +def test_a_fit_that_was_never_checked_is_an_open_part_after_a_successful_run_only(tmp_path): + _complete(tmp_path) + (tmp_path / "question_fit.json").unlink() + row = _parts(ledger(tmp_path))["question_fit"] + assert row["verdict"] == "unresolved" and "was not checked" in row["note"] + _write(tmp_path, "question_fit.json", {"fit": "maybe"}) + assert _parts(ledger(tmp_path))["question_fit"]["verdict"] == "unresolved" + failed = tmp_path / "failed" + _write(failed, "run.json", {"status": "failed", "rule": None, "kind": "timeout", "detail": None}) + assert "question_fit" not in _parts(ledger(failed)) + + +def test_a_no_question_fit_against_a_row_that_carries_a_question_is_never_an_example(tmp_path): + """`no_question` removes the part; declared against a row that carries a question it is a + contradiction, and the findings verb refuses to treat the row as a statement that held.""" + run = _run_dir(tmp_path, [{"row": 1, "question": "How many orders?", "statement": "s", "expected": 1, + "status": "mismatch"}]) + d = run / "rows" / "1" + _complete(d) + _write(d, "question_fit.json", {"fit": "no_question", "reason": None}) + assert findings(run)["findings"] == [] + _write(d, "question_fit.json", {"fit": "plausible", "reason": None}) + assert [f["kind"] for f in findings(run)["findings"]] == ["example"]