diff --git a/CHANGELOG.md b/CHANGELOG.md index a7191c45..e515d223 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,14 @@ below corresponds to one such version. ### Added +- **Agami's answer may be several statements, and the reconcile page shows every one.** The cold + client's reply was read as one string under `sql`; a list, or several statements in one string, + lost everything but the first object or read as unreadable. The generator now keeps every + statement in order (`statements`), answers with the last, and the prompt says so; the ask door + writes `statements` beside `sql`; the row record carries `agami_statements`; the report card's + SQL block lists them numbered with the last marked "compared", and the rows check notes that + agami ran N queries. Only the last is run and graded. (ACE-135) + - **An eighth claim, `outputs`, says what a statement selects.** `sm claims` and the golden run compared tables, filters, date window, group keys, join keys, ordering and limit, and never the projection, so two statements selecting different expressions could still read as the same diff --git a/packages/agami-core/src/semantic_model/golden_run.py b/packages/agami-core/src/semantic_model/golden_run.py index 1499ba72..25300be6 100644 --- a/packages/agami-core/src/semantic_model/golden_run.py +++ b/packages/agami-core/src/semantic_model/golden_run.py @@ -109,6 +109,10 @@ class GeneratedSql: sql: str error: Optional[str] + # Every statement the generator wrote, in order, when it wrote more than one; `sql` is the last + # of them, the one whose result answers the question. Empty when the generator did not say (an + # injected generator built before this field existed), which readers treat as `(sql,)`. + statements: tuple[str, ...] = () class SqlGenerator(Protocol): @@ -616,6 +620,7 @@ class GenerationContext(NamedTuple): {question} Reply with a single JSON object and no other text: {{"sql": ""}} +If answering takes more than one query, put them in order in a list under sql; the last must be the statement whose result answers the question. """ @@ -711,6 +716,54 @@ def _first_json_object(text: str) -> Optional[dict[str, Any]]: return None +def _split_statements(text: str) -> list[str]: + """The top-level statements in a reply string: split on `;` outside quotes and comments, each + piece stripped, empty pieces dropped. Text splitting and nothing else: no statement is parsed and + regenerated here (ACE-093), so what comes out is what the model wrote, cut at its semicolons.""" + pieces: list[str] = [] + buf: list[str] = [] + quote: Optional[str] = None + i, n = 0, len(text) + while i < n: + ch = text[i] + if quote is not None: + buf.append(ch) + if ch == quote: + if i + 1 < n and text[i + 1] == quote: # a doubled quote inside the literal + buf.append(text[i + 1]) + i += 2 + continue + quote = None + i += 1 + continue + if ch in ("'", '"', "`"): + quote = ch + buf.append(ch) + i += 1 + continue + if text.startswith("--", i): + j = text.find("\n", i) + j = n if j == -1 else j + buf.append(text[i:j]) + i = j + continue + if text.startswith("/*", i): + j = text.find("*/", i + 2) + j = n if j == -1 else j + 2 + buf.append(text[i:j]) + i = j + continue + if ch == ";": + pieces.append("".join(buf)) + buf = [] + i += 1 + continue + buf.append(ch) + i += 1 + pieces.append("".join(buf)) + return [piece.strip() for piece in pieces if piece.strip()] + + def _child_env() -> dict[str, str]: """The environment the child is given: the allowlist, and only the names that are actually set.""" return {key: os.environ[key] for key in _CHILD_ENV_KEYS if key in os.environ} @@ -834,10 +887,21 @@ def _spawn(prompt: str, argv: list[str], timeout_s: float, *, system_prompt: str if completed.returncode != 0: return GeneratedSql(sql="", error=_GENERATION_EXITED) answer = _first_json_object(completed.stdout) - sql = answer.get("sql") if answer else None - if not isinstance(sql, str) or not sql.strip(): + raw = answer.get("sql") if answer else None + # One statement, or several: a list under `sql`, or one string cut at its top-level semicolons. + # Every statement is kept in order and the LAST is the answer, which is what the prompt asked + # for; a list carrying anything that is not a statement is unreadable as a whole. + if isinstance(raw, list): + statements = [item.strip() for item in raw if isinstance(item, str) and item.strip()] + if len(statements) != len(raw): + statements = [] + elif isinstance(raw, str) and raw.strip(): + statements = _split_statements(raw) + else: + statements = [] + if not statements: return GeneratedSql(sql="", error=_GENERATION_UNREADABLE) - return GeneratedSql(sql=sql.strip(), error=None) + return GeneratedSql(sql=statements[-1], error=None, statements=tuple(statements)) __all__ = [ diff --git a/plugins/agami/scripts/reconcile.py b/plugins/agami/scripts/reconcile.py index 4decd1fd..1d51e23c 100644 --- a/plugins/agami/scripts/reconcile.py +++ b/plugins/agami/scripts/reconcile.py @@ -1680,6 +1680,13 @@ def _receipt_metrics(receipt: Any) -> dict[str, str]: return out +def _agami_steps(rec: dict) -> list[str]: + """Every statement agami wrote for this row when there was more than one, in order, the last + being the one run and compared; empty for the usual single statement.""" + steps = [s for s in (rec.get("agami_statements") or []) if isinstance(s, str) and s.strip()] + return steps if len(steps) > 1 else [] + + def _diff_rows(rec: dict, agami_receipt: Any) -> tuple[list[dict], list[str]]: rows: list[dict] = [] words: list[str] = [] @@ -1711,9 +1718,11 @@ def add(key, state, yours=None, agami=None, note=None, yours_hi=None, agami_hi=N runs = parts.get("runs") if runs and runs["verdict"] != CONFIRMED: yours_text = _PART_WORDS["runs"].get(_STATE[runs["verdict"]], yours_text) + steps = _agami_steps(rec) + steps_note = f"agami ran {len(steps)} queries; the last one's result is compared" if steps else None if result_set: same_rows = result_set.get("golden_row_count") == result_set.get("generated_row_count") - add("rows", "held" if same_rows else "defect", yours_text, agami_text) + add("rows", "held" if same_rows else "defect", yours_text, agami_text, note=steps_note) yc = list(((rec.get("statement_recorded") or {}).get("columns")) or []) ac = list(((rec.get("recorded") or {}).get("columns")) or []) pairs = [tuple(p) for p in (result_set.get("column_pairs") or []) if isinstance(p, (list, tuple)) and len(p) == 2] @@ -1785,7 +1794,7 @@ def add(key, state, yours=None, agami=None, note=None, yours_hi=None, agami_hi=N note = None if state == "defect" and isinstance(delta, (int, float)) and not isinstance(delta, bool): note = f"agami is {delta * 100:+.1f}% from your number" # `delta_pct` is a signed fraction (2d) - add("answer", state, yours_text, agami_text, note=note or rec.get("error"), + add("answer", state, yours_text, agami_text, note=note or rec.get("error") or steps_note, yours_hi=[yours_text] if state == "defect" and yours_text else None, agami_hi=[agami_text] if state == "defect" and agami_text else None) @@ -2272,6 +2281,7 @@ def report_items(run_dir: Path) -> list[dict]: "sentence": _sentence(rec, diff) + (" " + clause if clause else ""), "words": words, "disagreement": None, "change": list(change), "todo": list(todo), "sql_yours": rec.get("statement") or None, "sql_agami": rec.get("sql") or None, + "sql_agami_steps": _agami_steps(rec), "report_path": rec.get("report_path"), }) return items diff --git a/plugins/agami/scripts/render_reconcile_report.py b/plugins/agami/scripts/render_reconcile_report.py index 2fe04490..d588f8bf 100644 --- a/plugins/agami/scripts/render_reconcile_report.py +++ b/plugins/agami/scripts/render_reconcile_report.py @@ -41,7 +41,7 @@ # What one card may carry, beat by beat. Every text field is DISPLAY text the skill already wrote in # plain language; the lists are one sentence per line. A `rows` or `recorded` key is refused. _FIELDS = ("row", "label", "question", "source", "status", "expected", "answer", "delta_pct", "single_cell", - "owner", "read", "how", "words", "disagreement", "change", "checks", "todo", "report_path", "diff", "sentence", "sql_yours", "sql_agami", "keep_allowed", "result", "fix", "fix_words", "prefill") + "owner", "read", "how", "words", "disagreement", "change", "checks", "todo", "report_path", "diff", "sentence", "sql_yours", "sql_agami", "sql_agami_steps", "keep_allowed", "result", "fix", "fix_words", "prefill") _LISTS = ("read", "how", "words", "change", "todo") _DIFF_KEYS = ("key", "state", "yours", "agami", "note", "yours_hi", "agami_hi", "renamed") _STATUSES = {"match", "match_unverified", "mismatch", "expected_doubtful", "error"} @@ -73,6 +73,9 @@ def _validate_item(item: dict, idx: int) -> None: value = item.get(key, []) if not isinstance(value, list) or not all(isinstance(s, str) for s in value): raise ValueError(f"item {idx}: '{key}' must be a list of sentences") + steps = item.get("sql_agami_steps", []) + if steps is not None and (not isinstance(steps, list) or not all(isinstance(s, str) for s in steps)): + raise ValueError(f"item {idx}: 'sql_agami_steps' must be a list of statements") if item.get("owner") is not None and item["owner"] not in _OWNERS: raise ValueError(f"item {idx}: 'owner' must be one of {sorted(_OWNERS)}") if item.get("delta_pct") is not None and not isinstance(item["delta_pct"], (int, float)): diff --git a/plugins/agami/scripts/run_golden_eval.py b/plugins/agami/scripts/run_golden_eval.py index 4c7c112b..5aae36ad 100644 --- a/plugins/agami/scripts/run_golden_eval.py +++ b/plugins/agami/scripts/run_golden_eval.py @@ -1042,7 +1042,7 @@ def _ask(args: argparse.Namespace) -> int: generator = GENERATOR(lambda question: _model_context(cached, question), timeout_s=args.timeout_s, **_effort(args)) generated = generator.generate(args.ask, tools.resolved_org_id(), args.profile) sql = generated.sql.strip() if generated.sql else "" - payload = {"question": args.ask, "sql": sql or None, "error": generated.error} + payload = {"question": args.ask, "sql": sql or None, "statements": _statements(generated, sql), "error": generated.error} if args.out: out = Path(args.out).expanduser() out.parent.mkdir(parents=True, exist_ok=True) @@ -1051,6 +1051,13 @@ def _ask(args: argparse.Namespace) -> int: return 0 if sql and generated.error is None else _NO_STATEMENT +def _statements(generated: Any, sql: str) -> list[str]: + """Every statement the client wrote, in order, `sql` last; a generator that did not say gives + the one statement it answered with.""" + statements = [s for s in (getattr(generated, "statements", None) or ()) if isinstance(s, str) and s.strip()] + return statements or ([sql] if sql else []) + + def _questions_from_file(path: Path) -> list[dict[str, Any]]: """`[{row, question}, ...]`, from a bare list or from `reconcile.py next-chunk`'s output (its `chunk`) or an intake file (its `rows`). A row without a question is skipped and named.""" @@ -1092,10 +1099,11 @@ def _ask_many(args: argparse.Namespace) -> int: def one(q: dict[str, Any]) -> dict[str, Any]: if not q["question"]: - return {"row": q["row"], "question": None, "sql": None, "error": "the row carries no question"} + return {"row": q["row"], "question": None, "sql": None, "statements": [], "error": "the row carries no question"} generated = generator.generate(q["question"], org, args.profile) sql = generated.sql.strip() if generated.sql else "" - return {"row": q["row"], "question": q["question"], "sql": sql or None, "error": generated.error} + return {"row": q["row"], "question": q["question"], "sql": sql or None, + "statements": _statements(generated, sql), "error": generated.error} from concurrent.futures import ThreadPoolExecutor workers = max(1, min(args.parallel, len(questions))) diff --git a/plugins/agami/shared/reconcile-report-template.html b/plugins/agami/shared/reconcile-report-template.html index ddd3118c..021c33b0 100644 --- a/plugins/agami/shared/reconcile-report-template.html +++ b/plugins/agami/shared/reconcile-report-template.html @@ -237,9 +237,17 @@

Send to Claude

// the receipt already shows. Never a result row. function sqlBlock(item) { if (!item.sql_yours && !item.sql_agami) return ''; + // agami may have written several statements to answer; every one is shown in order and the + // last, the one whose result was compared, is marked. + const steps = Array.isArray(item.sql_agami_steps) && item.sql_agami_steps.length > 1 ? item.sql_agami_steps : null; + const agami = steps + ? '
agami (' + steps.length + ' statements; the last is compared)' + + steps.map((s, i) => '
' + (i + 1) + (i === steps.length - 1 ? ' ยท compared' : '') + '
' + esc(s) + '
').join('') + + '
' + : '
agami
' + esc(item.sql_agami || '(none)') + '
'; return '
The two SQL statements
' + '
yours
' + esc(item.sql_yours || '(none)') + '
' - + '
agami
' + esc(item.sql_agami || '(none)') + '
'; + + agami + ''; } function words(item) { return (item.words || []).length diff --git a/plugins/agami/skills/agami-reconcile/SKILL.md b/plugins/agami/skills/agami-reconcile/SKILL.md index 54292d75..b4f285ee 100644 --- a/plugins/agami/skills/agami-reconcile/SKILL.md +++ b/plugins/agami/skills/agami-reconcile/SKILL.md @@ -187,11 +187,11 @@ python3 "$AGAMI_PLUGIN_ROOT/scripts/run_golden_eval.py" --profile \ --ask-file /tmp/agami-reconcile-chunk-.json --out-dir "/local/reconcile//rows" --parallel 4 ``` -`--ask-file` takes `next-chunk`'s output as it is (its `chunk`), fetches the model context once for the batch, spawns the operator's own client per question with every tool off, no MCP servers and no settings, several at a time, gives each the same context the golden run gives (the schema from the product's own tool, what the datasource means, the ranked prompt examples), and writes `rows//agami-answer.json` per row as `{row, question, sql, error}`. What is reused across the chunk is what does not depend on the question; the session itself is never reused, because a fresh one is the thing being measured. One question at a time is `--ask "" --out rows//agami-answer.json`. Per row, exit `0` for the batch, or a `sql` in the row's file, carries a statement: write it verbatim to `rows//agami.sql`, run it through the profile's tier exactly as 1.5b runs yours (stdout to `rows//actual.csv`), then `sm receipt --sql-file rows//agami.sql` and the chart report, as agami-query Phase 3 does. A row whose file has no `sql` carries one of the generator's four fixed sentences as its `error` (the client could not be started, timed out, exited without answering, or answered without a statement); the batch exits `3` when any row is like that. That row is `error` with the sentence as its `error`. **Never write agami's SQL yourself, and never retry with your own wording**; a row with no cold answer is an error row, and that is the finding. Exit `2` means the profile's context could not be built: stop the run and say so. +`--ask-file` takes `next-chunk`'s output as it is (its `chunk`), fetches the model context once for the batch, spawns the operator's own client per question with every tool off, no MCP servers and no settings, several at a time, gives each the same context the golden run gives (the schema from the product's own tool, what the datasource means, the ranked prompt examples), and writes `rows//agami-answer.json` per row as `{row, question, sql, statements, error}`: `statements` is every statement the client wrote, in order, and `sql` is the last of them, the one whose result answers the question. What is reused across the chunk is what does not depend on the question; the session itself is never reused, because a fresh one is the thing being measured. One question at a time is `--ask "" --out rows//agami-answer.json`. Per row, exit `0` for the batch, or a `sql` in the row's file, carries a statement: write it verbatim to `rows//agami.sql` (only `sql`; when `statements` has more than one, keep them all in the row record's `agami_statements` for the page and never run the earlier ones: the read-only rule refuses anything but a SELECT, so an earlier statement can only be a look at the data), run it through the profile's tier exactly as 1.5b runs yours (stdout to `rows//actual.csv`), then `sm receipt --sql-file rows//agami.sql` and the chart report, as agami-query Phase 3 does. A row whose file has no `sql` carries one of the generator's four fixed sentences as its `error` (the client could not be started, timed out, exited without answering, or answered without a statement); the batch exits `3` when any row is like that. That row is `error` with the sentence as its `error`. **Never write agami's SQL yourself, and never retry with your own wording**; a row with no cold answer is an error row, and that is the finding. Exit `2` means the profile's context could not be built: stop the run and say so. Capture, per row: -- The generated SQL, verbatim, from `agami-answer.json` +- The generated SQL, verbatim, from `agami-answer.json` (its `sql`; and its `statements` when the client wrote several) - The result (one cell, or the columns and a row count) - The full chart-template HTML report (so the user can drill in for mismatches) - The trust receipt (with confidence, signed-off-by, etc.) @@ -264,7 +264,8 @@ Per row: "comparison": {"scalar": } | {"result_set": } | null, "claims": , "finding_keys": [""], - "words": "" + "words": "", + "agami_statements": [""] // only when there were several, else [] } ``` diff --git a/tests/test_golden_run.py b/tests/test_golden_run.py index a5c99d44..c92d7f06 100644 --- a/tests/test_golden_run.py +++ b/tests/test_golden_run.py @@ -1036,3 +1036,44 @@ def _missing(*args, **kwargs): assert generated.sql == "" and generated.error == gr._GENERATION_UNAVAILABLE + + +# --- ACE-135: several statements in one answer --------------------------------------------------- + + +def test_a_list_of_statements_keeps_every_one_and_answers_with_the_last(monkeypatch): + reply = json.dumps({"sql": ["SELECT status FROM orders LIMIT 5", "SELECT COUNT(*) AS n FROM orders"]}) + monkeypatch.setattr(gr.subprocess, "run", _RecordedSpawn(stdout=reply)) + + generated = _cli_generator().generate(QUESTION, ORG, DATASOURCE) + + assert generated.error is None and generated.sql == "SELECT COUNT(*) AS n FROM orders" + assert generated.statements == ("SELECT status FROM orders LIMIT 5", "SELECT COUNT(*) AS n FROM orders") + + +def test_several_statements_in_one_string_are_cut_at_top_level_semicolons_only(monkeypatch): + text = "SELECT status FROM orders WHERE note = 'a;b' -- not; here\n; /* nor; here */ SELECT COUNT(*) AS n FROM orders;" + monkeypatch.setattr(gr.subprocess, "run", _RecordedSpawn(stdout=json.dumps({"sql": text}))) + + generated = _cli_generator().generate(QUESTION, ORG, DATASOURCE) + + assert generated.statements == ("SELECT status FROM orders WHERE note = 'a;b' -- not; here", + "/* nor; here */ SELECT COUNT(*) AS n FROM orders") + assert generated.sql == "/* nor; here */ SELECT COUNT(*) AS n FROM orders" + + +def test_one_statement_is_one_statement(monkeypatch): + monkeypatch.setattr(gr.subprocess, "run", _RecordedSpawn(stdout=ANSWER)) + generated = _cli_generator().generate(QUESTION, ORG, DATASOURCE) + assert generated.statements == (generated.sql,) and generated.sql.startswith("SELECT COUNT") + + +@pytest.mark.parametrize("raw", [[], ["", " "], ["SELECT 1", 3], " ; ; "]) +def test_a_list_carrying_anything_but_statements_is_unreadable(monkeypatch, raw): + monkeypatch.setattr(gr.subprocess, "run", _RecordedSpawn(stdout=json.dumps({"sql": raw}))) + generated = _cli_generator().generate(QUESTION, ORG, DATASOURCE) + assert generated.sql == "" and generated.error == gr._GENERATION_UNREADABLE and generated.statements == () + + +def test_the_prompt_says_how_to_answer_with_several_queries(): + assert "put them in order in a list under sql; the last must be the statement whose result answers the question" in gr._QUESTION_PROMPT diff --git a/tests/test_reconcile_learning_loop_skill.py b/tests/test_reconcile_learning_loop_skill.py index 4bc0c544..73c3d6a9 100644 --- a/tests/test_reconcile_learning_loop_skill.py +++ b/tests/test_reconcile_learning_loop_skill.py @@ -364,3 +364,10 @@ def test_agamis_answer_comes_from_a_cold_client_never_from_the_session(): assert "four fixed sentences" in phase_2b and "the batch exits `3` when any row is like that" in phase_2b assert "Invoke the same SQL-generation + execution path agami-query uses" not in phase_2b + + +def test_agamis_several_statements_are_kept_and_only_the_last_is_run(): + ask = _between(SKILL, "### 2b", "### 2c") + assert "{row, question, sql, statements, error}" in ask and "never run the earlier ones" in ask + record = _between(SKILL, "### 2d", "### 2e") + assert '"agami_statements":' in record and "sql is the last" in record diff --git a/tests/test_reconcile_report_items.py b/tests/test_reconcile_report_items.py index f645bce5..c9846535 100644 --- a/tests/test_reconcile_report_items.py +++ b/tests/test_reconcile_report_items.py @@ -423,3 +423,15 @@ def test_the_two_statements_still_say_what_they_are_when_the_data_could_not_be_c assert selects["state"] == "differs" and selects["yours"] == ["sum(orders.amount)"] and selects["agami"] == ["avg(orders.amount)"] # A structural match never makes the row keepable: the gate is the data's. assert items[1]["keep_allowed"] is False and items[1]["status"] == "error" + + +# --- ACE-135: agami's several statements on the card --------------------------------------------- + +def test_several_agami_statements_are_listed_and_the_rows_check_says_which_one_was_compared(tmp_path): + several = dict(SCALAR_MATCH, row=1, agami_statements=["SELECT status FROM orders LIMIT 5", SCALAR_MATCH.get("sql") or "SELECT SUM(amount) FROM orders"]) + one = dict(SCALAR_MATCH, row=2, agami_statements=[SCALAR_MATCH.get("sql") or "SELECT SUM(amount) FROM orders"]) + items = {i["row"]: i for i in reconcile.report_items(_run(tmp_path, [several, one]))} + assert items[1]["sql_agami_steps"] == several["agami_statements"] + answer = next(r for r in items[1]["diff"] if r["key"] == "answer") + assert answer["note"] == "agami ran 2 queries; the last one's result is compared" + assert items[2]["sql_agami_steps"] == [] and next(r for r in items[2]["diff"] if r["key"] == "answer")["note"] is None diff --git a/tests/test_render_reconcile_report.py b/tests/test_render_reconcile_report.py index c6dc40e7..969afb62 100644 --- a/tests/test_render_reconcile_report.py +++ b/tests/test_render_reconcile_report.py @@ -131,3 +131,15 @@ def test_owner_delta_and_check_states_come_from_closed_sets(): rr.render(title="t", profile="p", run="r", items=[{"row": 1, "question": "q", "status": "match", "checks": [{"step": "x", "state": "maybe"}]}]) with pytest.raises(ValueError, match="layout must be"): rr.render(title="t", profile="p", run="r", items=ITEMS, layout="table") + + +# --- ACE-135: several statements in the SQL block ----------------------------------------------- + +def test_the_sql_block_lists_every_statement_agami_wrote_and_marks_the_compared_one(): + item = {"row": 1, "question": "How many orders?", "status": "match", "sql_yours": "SELECT COUNT(*) FROM orders", + "sql_agami": "SELECT COUNT(*) AS n FROM orders", "sql_agami_steps": ["SELECT status FROM orders LIMIT 5", "SELECT COUNT(*) AS n FROM orders"]} + page = rr.render(title="t", profile="demo", run="r", items=[item], layout="cards") + assert "the last is compared" in page and "SELECT status FROM orders LIMIT 5" in page + import pytest + with pytest.raises(ValueError, match="sql_agami_steps"): + rr._validate_item(dict(item, sql_agami_steps="SELECT 1"), 0) diff --git a/tests/test_run_golden_eval_ask.py b/tests/test_run_golden_eval_ask.py index 645c8d27..ae52b337 100644 --- a/tests/test_run_golden_eval_ask.py +++ b/tests/test_run_golden_eval_ask.py @@ -46,7 +46,7 @@ def test_ask_answers_one_question_with_the_golden_runs_generator_and_context(mon out = tmp_path / "rows" / "1" / "agami-answer.json" assert rge.main(["--profile", "demo", "--ask", "How many orders?", "--top-k", "3", "--timeout-s", "45", "--out", str(out)]) == 0 printed = json.loads(capsys.readouterr().out) - assert printed == {"question": "How many orders?", "sql": "SELECT COUNT(*) AS n FROM orders", "error": None} + assert printed == {"question": "How many orders?", "sql": "SELECT COUNT(*) AS n FROM orders", "statements": ["SELECT COUNT(*) AS n FROM orders"], "error": None} assert json.loads(out.read_text()) == printed gen = _Generator.made[0] assert gen.timeout_s == 45.0 and gen.asked == ("How many orders?", "local", "demo", "schema for How many orders? with 3 examples") @@ -101,3 +101,13 @@ def generate(self, question, org, datasource): assert rge.main(["--profile", "demo", "--ask-file", str(tmp_path / "bad.json")]) == rge._CANNOT_START assert rge.main(["--profile", "demo", "--ask", "q", "--ask-file", str(chunk)]) == rge._CANNOT_START + + +def test_ask_writes_every_statement_the_client_wrote_and_answers_with_the_last(monkeypatch, tmp_path, capsys): + _wire(monkeypatch, tmp_path) + _Generator.answer = gr.GeneratedSql(sql="SELECT COUNT(*) AS n FROM orders", error=None, + statements=("SELECT status FROM orders LIMIT 5", "SELECT COUNT(*) AS n FROM orders")) + assert rge.main(["--profile", "demo", "--ask", "How many orders?"]) == 0 + printed = json.loads(capsys.readouterr().out) + assert printed["sql"] == "SELECT COUNT(*) AS n FROM orders" + assert printed["statements"] == ["SELECT status FROM orders LIMIT 5", "SELECT COUNT(*) AS n FROM orders"]