Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,14 @@ below corresponds to one such version.

### Added

- **Agami's answer may be several statements, and the reconcile page shows every one.** The cold
client's reply was read as one string under `sql`; a list, or several statements in one string,
lost everything but the first object or read as unreadable. The generator now keeps every
statement in order (`statements`), answers with the last, and the prompt says so; the ask door
writes `statements` beside `sql`; the row record carries `agami_statements`; the report card's
SQL block lists them numbered with the last marked "compared", and the rows check notes that
agami ran N queries. Only the last is run and graded. (ACE-135)

- **An eighth claim, `outputs`, says what a statement selects.** `sm claims` and the golden run
compared tables, filters, date window, group keys, join keys, ordering and limit, and never the
projection, so two statements selecting different expressions could still read as the same
Expand Down
70 changes: 67 additions & 3 deletions packages/agami-core/src/semantic_model/golden_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,10 @@ class GeneratedSql:

sql: str
error: Optional[str]
# Every statement the generator wrote, in order, when it wrote more than one; `sql` is the last
# of them, the one whose result answers the question. Empty when the generator did not say (an
# injected generator built before this field existed), which readers treat as `(sql,)`.
statements: tuple[str, ...] = ()


class SqlGenerator(Protocol):
Expand Down Expand Up @@ -616,6 +620,7 @@ class GenerationContext(NamedTuple):
{question}

Reply with a single JSON object and no other text: {{"sql": "<one SELECT statement>"}}
If answering takes more than one query, put them in order in a list under sql; the last must be the statement whose result answers the question.
"""


Expand Down Expand Up @@ -711,6 +716,54 @@ def _first_json_object(text: str) -> Optional[dict[str, Any]]:
return None


def _split_statements(text: str) -> list[str]:
"""The top-level statements in a reply string: split on `;` outside quotes and comments, each
piece stripped, empty pieces dropped. Text splitting and nothing else: no statement is parsed and
regenerated here (ACE-093), so what comes out is what the model wrote, cut at its semicolons."""
pieces: list[str] = []
buf: list[str] = []
quote: Optional[str] = None
i, n = 0, len(text)
while i < n:
ch = text[i]
if quote is not None:
buf.append(ch)
if ch == quote:
if i + 1 < n and text[i + 1] == quote: # a doubled quote inside the literal
buf.append(text[i + 1])
i += 2
continue
quote = None
i += 1
continue
if ch in ("'", '"', "`"):
quote = ch
buf.append(ch)
i += 1
continue
if text.startswith("--", i):
j = text.find("\n", i)
j = n if j == -1 else j
buf.append(text[i:j])
i = j
continue
if text.startswith("/*", i):
j = text.find("*/", i + 2)
j = n if j == -1 else j + 2
buf.append(text[i:j])
i = j
continue
if ch == ";":
pieces.append("".join(buf))
buf = []
i += 1
continue
buf.append(ch)
i += 1
pieces.append("".join(buf))
return [piece.strip() for piece in pieces if piece.strip()]


def _child_env() -> dict[str, str]:
"""The environment the child is given: the allowlist, and only the names that are actually set."""
return {key: os.environ[key] for key in _CHILD_ENV_KEYS if key in os.environ}
Expand Down Expand Up @@ -834,10 +887,21 @@ def _spawn(prompt: str, argv: list[str], timeout_s: float, *, system_prompt: str
if completed.returncode != 0:
return GeneratedSql(sql="", error=_GENERATION_EXITED)
answer = _first_json_object(completed.stdout)
sql = answer.get("sql") if answer else None
if not isinstance(sql, str) or not sql.strip():
raw = answer.get("sql") if answer else None
# One statement, or several: a list under `sql`, or one string cut at its top-level semicolons.
# Every statement is kept in order and the LAST is the answer, which is what the prompt asked
# for; a list carrying anything that is not a statement is unreadable as a whole.
if isinstance(raw, list):
statements = [item.strip() for item in raw if isinstance(item, str) and item.strip()]
if len(statements) != len(raw):
statements = []
elif isinstance(raw, str) and raw.strip():
statements = _split_statements(raw)
else:
statements = []
if not statements:
return GeneratedSql(sql="", error=_GENERATION_UNREADABLE)
return GeneratedSql(sql=sql.strip(), error=None)
return GeneratedSql(sql=statements[-1], error=None, statements=tuple(statements))


__all__ = [
Expand Down
14 changes: 12 additions & 2 deletions plugins/agami/scripts/reconcile.py
Original file line number Diff line number Diff line change
Expand Up @@ -1680,6 +1680,13 @@ def _receipt_metrics(receipt: Any) -> dict[str, str]:
return out


def _agami_steps(rec: dict) -> list[str]:
"""Every statement agami wrote for this row when there was more than one, in order, the last
being the one run and compared; empty for the usual single statement."""
steps = [s for s in (rec.get("agami_statements") or []) if isinstance(s, str) and s.strip()]
return steps if len(steps) > 1 else []


def _diff_rows(rec: dict, agami_receipt: Any) -> tuple[list[dict], list[str]]:
rows: list[dict] = []
words: list[str] = []
Expand Down Expand Up @@ -1711,9 +1718,11 @@ def add(key, state, yours=None, agami=None, note=None, yours_hi=None, agami_hi=N
runs = parts.get("runs")
if runs and runs["verdict"] != CONFIRMED:
yours_text = _PART_WORDS["runs"].get(_STATE[runs["verdict"]], yours_text)
steps = _agami_steps(rec)
steps_note = f"agami ran {len(steps)} queries; the last one's result is compared" if steps else None
if result_set:
same_rows = result_set.get("golden_row_count") == result_set.get("generated_row_count")
add("rows", "held" if same_rows else "defect", yours_text, agami_text)
add("rows", "held" if same_rows else "defect", yours_text, agami_text, note=steps_note)
yc = list(((rec.get("statement_recorded") or {}).get("columns")) or [])
ac = list(((rec.get("recorded") or {}).get("columns")) or [])
pairs = [tuple(p) for p in (result_set.get("column_pairs") or []) if isinstance(p, (list, tuple)) and len(p) == 2]
Expand Down Expand Up @@ -1785,7 +1794,7 @@ def add(key, state, yours=None, agami=None, note=None, yours_hi=None, agami_hi=N
note = None
if state == "defect" and isinstance(delta, (int, float)) and not isinstance(delta, bool):
note = f"agami is {delta * 100:+.1f}% from your number" # `delta_pct` is a signed fraction (2d)
add("answer", state, yours_text, agami_text, note=note or rec.get("error"),
add("answer", state, yours_text, agami_text, note=note or rec.get("error") or steps_note,
yours_hi=[yours_text] if state == "defect" and yours_text else None,
agami_hi=[agami_text] if state == "defect" and agami_text else None)

Expand Down Expand Up @@ -2272,6 +2281,7 @@ def report_items(run_dir: Path) -> list[dict]:
"sentence": _sentence(rec, diff) + (" " + clause if clause else ""),
"words": words, "disagreement": None, "change": list(change), "todo": list(todo),
"sql_yours": rec.get("statement") or None, "sql_agami": rec.get("sql") or None,
"sql_agami_steps": _agami_steps(rec),
"report_path": rec.get("report_path"),
})
return items
Expand Down
5 changes: 4 additions & 1 deletion plugins/agami/scripts/render_reconcile_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
# What one card may carry, beat by beat. Every text field is DISPLAY text the skill already wrote in
# plain language; the lists are one sentence per line. A `rows` or `recorded` key is refused.
_FIELDS = ("row", "label", "question", "source", "status", "expected", "answer", "delta_pct", "single_cell",
"owner", "read", "how", "words", "disagreement", "change", "checks", "todo", "report_path", "diff", "sentence", "sql_yours", "sql_agami", "keep_allowed", "result", "fix", "fix_words", "prefill")
"owner", "read", "how", "words", "disagreement", "change", "checks", "todo", "report_path", "diff", "sentence", "sql_yours", "sql_agami", "sql_agami_steps", "keep_allowed", "result", "fix", "fix_words", "prefill")
_LISTS = ("read", "how", "words", "change", "todo")
_DIFF_KEYS = ("key", "state", "yours", "agami", "note", "yours_hi", "agami_hi", "renamed")
_STATUSES = {"match", "match_unverified", "mismatch", "expected_doubtful", "error"}
Expand Down Expand Up @@ -73,6 +73,9 @@ def _validate_item(item: dict, idx: int) -> None:
value = item.get(key, [])
if not isinstance(value, list) or not all(isinstance(s, str) for s in value):
raise ValueError(f"item {idx}: '{key}' must be a list of sentences")
steps = item.get("sql_agami_steps", [])
if steps is not None and (not isinstance(steps, list) or not all(isinstance(s, str) for s in steps)):
raise ValueError(f"item {idx}: 'sql_agami_steps' must be a list of statements")
if item.get("owner") is not None and item["owner"] not in _OWNERS:
raise ValueError(f"item {idx}: 'owner' must be one of {sorted(_OWNERS)}")
if item.get("delta_pct") is not None and not isinstance(item["delta_pct"], (int, float)):
Expand Down
14 changes: 11 additions & 3 deletions plugins/agami/scripts/run_golden_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -1042,7 +1042,7 @@ def _ask(args: argparse.Namespace) -> int:
generator = GENERATOR(lambda question: _model_context(cached, question), timeout_s=args.timeout_s, **_effort(args))
generated = generator.generate(args.ask, tools.resolved_org_id(), args.profile)
sql = generated.sql.strip() if generated.sql else ""
payload = {"question": args.ask, "sql": sql or None, "error": generated.error}
payload = {"question": args.ask, "sql": sql or None, "statements": _statements(generated, sql), "error": generated.error}
if args.out:
out = Path(args.out).expanduser()
out.parent.mkdir(parents=True, exist_ok=True)
Expand All @@ -1051,6 +1051,13 @@ def _ask(args: argparse.Namespace) -> int:
return 0 if sql and generated.error is None else _NO_STATEMENT


def _statements(generated: Any, sql: str) -> list[str]:
"""Every statement the client wrote, in order, `sql` last; a generator that did not say gives
the one statement it answered with."""
statements = [s for s in (getattr(generated, "statements", None) or ()) if isinstance(s, str) and s.strip()]
return statements or ([sql] if sql else [])


def _questions_from_file(path: Path) -> list[dict[str, Any]]:
"""`[{row, question}, ...]`, from a bare list or from `reconcile.py next-chunk`'s output (its
`chunk`) or an intake file (its `rows`). A row without a question is skipped and named."""
Expand Down Expand Up @@ -1092,10 +1099,11 @@ def _ask_many(args: argparse.Namespace) -> int:

def one(q: dict[str, Any]) -> dict[str, Any]:
if not q["question"]:
return {"row": q["row"], "question": None, "sql": None, "error": "the row carries no question"}
return {"row": q["row"], "question": None, "sql": None, "statements": [], "error": "the row carries no question"}
generated = generator.generate(q["question"], org, args.profile)
sql = generated.sql.strip() if generated.sql else ""
return {"row": q["row"], "question": q["question"], "sql": sql or None, "error": generated.error}
return {"row": q["row"], "question": q["question"], "sql": sql or None,
"statements": _statements(generated, sql), "error": generated.error}

from concurrent.futures import ThreadPoolExecutor
workers = max(1, min(args.parallel, len(questions)))
Expand Down
10 changes: 9 additions & 1 deletion plugins/agami/shared/reconcile-report-template.html
Original file line number Diff line number Diff line change
Expand Up @@ -237,9 +237,17 @@ <h2>Send to Claude</h2>
// the receipt already shows. Never a result row.
function sqlBlock(item) {
if (!item.sql_yours && !item.sql_agami) return '';
// agami may have written several statements to answer; every one is shown in order and the
// last, the one whose result was compared, is marked.
const steps = Array.isArray(item.sql_agami_steps) && item.sql_agami_steps.length > 1 ? item.sql_agami_steps : null;
const agami = steps
? '<div><b>agami</b> <small>(' + steps.length + ' statements; the last is compared)</small>'
+ steps.map((s, i) => '<div class="step"><small>' + (i + 1) + (i === steps.length - 1 ? ' 路 compared' : '') + '</small><pre>' + esc(s) + '</pre></div>').join('')
+ '</div>'
: '<div><b>agami</b><pre>' + esc(item.sql_agami || '(none)') + '</pre></div>';
return '<details class="sql"><summary>The two SQL statements</summary><div class="sql2">'
+ '<div><b>yours</b><pre>' + esc(item.sql_yours || '(none)') + '</pre></div>'
+ '<div><b>agami</b><pre>' + esc(item.sql_agami || '(none)') + '</pre></div></div></details>';
+ agami + '</div></details>';
}
function words(item) {
return (item.words || []).length
Expand Down
7 changes: 4 additions & 3 deletions plugins/agami/skills/agami-reconcile/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -187,11 +187,11 @@ python3 "$AGAMI_PLUGIN_ROOT/scripts/run_golden_eval.py" --profile <profile> \
--ask-file /tmp/agami-reconcile-chunk-<ts>.json --out-dir "<artifacts_dir>/local/reconcile/<ts>/rows" --parallel 4
```

`--ask-file` takes `next-chunk`'s output as it is (its `chunk`), fetches the model context once for the batch, spawns the operator's own client per question with every tool off, no MCP servers and no settings, several at a time, gives each the same context the golden run gives (the schema from the product's own tool, what the datasource means, the ranked prompt examples), and writes `rows/<n>/agami-answer.json` per row as `{row, question, sql, error}`. What is reused across the chunk is what does not depend on the question; the session itself is never reused, because a fresh one is the thing being measured. One question at a time is `--ask "<question>" --out rows/<n>/agami-answer.json`. Per row, exit `0` for the batch, or a `sql` in the row's file, carries a statement: write it verbatim to `rows/<n>/agami.sql`, run it through the profile's tier exactly as 1.5b runs yours (stdout to `rows/<n>/actual.csv`), then `sm receipt --sql-file rows/<n>/agami.sql` and the chart report, as agami-query Phase 3 does. A row whose file has no `sql` carries one of the generator's four fixed sentences as its `error` (the client could not be started, timed out, exited without answering, or answered without a statement); the batch exits `3` when any row is like that. That row is `error` with the sentence as its `error`. **Never write agami's SQL yourself, and never retry with your own wording**; a row with no cold answer is an error row, and that is the finding. Exit `2` means the profile's context could not be built: stop the run and say so.
`--ask-file` takes `next-chunk`'s output as it is (its `chunk`), fetches the model context once for the batch, spawns the operator's own client per question with every tool off, no MCP servers and no settings, several at a time, gives each the same context the golden run gives (the schema from the product's own tool, what the datasource means, the ranked prompt examples), and writes `rows/<n>/agami-answer.json` per row as `{row, question, sql, statements, error}`: `statements` is every statement the client wrote, in order, and `sql` is the last of them, the one whose result answers the question. What is reused across the chunk is what does not depend on the question; the session itself is never reused, because a fresh one is the thing being measured. One question at a time is `--ask "<question>" --out rows/<n>/agami-answer.json`. Per row, exit `0` for the batch, or a `sql` in the row's file, carries a statement: write it verbatim to `rows/<n>/agami.sql` (only `sql`; when `statements` has more than one, keep them all in the row record's `agami_statements` for the page and never run the earlier ones: the read-only rule refuses anything but a SELECT, so an earlier statement can only be a look at the data), run it through the profile's tier exactly as 1.5b runs yours (stdout to `rows/<n>/actual.csv`), then `sm receipt --sql-file rows/<n>/agami.sql` and the chart report, as agami-query Phase 3 does. A row whose file has no `sql` carries one of the generator's four fixed sentences as its `error` (the client could not be started, timed out, exited without answering, or answered without a statement); the batch exits `3` when any row is like that. That row is `error` with the sentence as its `error`. **Never write agami's SQL yourself, and never retry with your own wording**; a row with no cold answer is an error row, and that is the finding. Exit `2` means the profile's context could not be built: stop the run and say so.

Capture, per row:

- The generated SQL, verbatim, from `agami-answer.json`
- The generated SQL, verbatim, from `agami-answer.json` (its `sql`; and its `statements` when the client wrote several)
- The result (one cell, or the columns and a row count)
- The full chart-template HTML report (so the user can drill in for mismatches)
- The trust receipt (with confidence, signed-off-by, etc.)
Expand Down Expand Up @@ -264,7 +264,8 @@ Per row:
"comparison": {"scalar": <the diff>} | {"result_set": <the compare-results score>} | null,
"claims": <the sm claims diff between the two statements, or null>,
"finding_keys": ["<keys of the findings this row contributed to>"],
"words": "<what the person wrote beside a wrong grade in Phase 2.5, or null>"
"words": "<what the person wrote beside a wrong grade in Phase 2.5, or null>",
"agami_statements": ["<every statement the client wrote, in order; sql is the last>"] // only when there were several, else []
}
```

Expand Down
41 changes: 41 additions & 0 deletions tests/test_golden_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -1036,3 +1036,44 @@ def _missing(*args, **kwargs):

assert generated.sql == "" and generated.error == gr._GENERATION_UNAVAILABLE



# --- ACE-135: several statements in one answer ---------------------------------------------------


def test_a_list_of_statements_keeps_every_one_and_answers_with_the_last(monkeypatch):
reply = json.dumps({"sql": ["SELECT status FROM orders LIMIT 5", "SELECT COUNT(*) AS n FROM orders"]})
monkeypatch.setattr(gr.subprocess, "run", _RecordedSpawn(stdout=reply))

generated = _cli_generator().generate(QUESTION, ORG, DATASOURCE)

assert generated.error is None and generated.sql == "SELECT COUNT(*) AS n FROM orders"
assert generated.statements == ("SELECT status FROM orders LIMIT 5", "SELECT COUNT(*) AS n FROM orders")


def test_several_statements_in_one_string_are_cut_at_top_level_semicolons_only(monkeypatch):
text = "SELECT status FROM orders WHERE note = 'a;b' -- not; here\n; /* nor; here */ SELECT COUNT(*) AS n FROM orders;"
monkeypatch.setattr(gr.subprocess, "run", _RecordedSpawn(stdout=json.dumps({"sql": text})))

generated = _cli_generator().generate(QUESTION, ORG, DATASOURCE)

assert generated.statements == ("SELECT status FROM orders WHERE note = 'a;b' -- not; here",
"/* nor; here */ SELECT COUNT(*) AS n FROM orders")
assert generated.sql == "/* nor; here */ SELECT COUNT(*) AS n FROM orders"


def test_one_statement_is_one_statement(monkeypatch):
monkeypatch.setattr(gr.subprocess, "run", _RecordedSpawn(stdout=ANSWER))
generated = _cli_generator().generate(QUESTION, ORG, DATASOURCE)
assert generated.statements == (generated.sql,) and generated.sql.startswith("SELECT COUNT")


@pytest.mark.parametrize("raw", [[], ["", " "], ["SELECT 1", 3], " ; ; "])
def test_a_list_carrying_anything_but_statements_is_unreadable(monkeypatch, raw):
monkeypatch.setattr(gr.subprocess, "run", _RecordedSpawn(stdout=json.dumps({"sql": raw})))
generated = _cli_generator().generate(QUESTION, ORG, DATASOURCE)
assert generated.sql == "" and generated.error == gr._GENERATION_UNREADABLE and generated.statements == ()


def test_the_prompt_says_how_to_answer_with_several_queries():
assert "put them in order in a list under sql; the last must be the statement whose result answers the question" in gr._QUESTION_PROMPT
Loading
Loading