From 987f4478ee98c7676e629cf551ff07250fb42957 Mon Sep 17 00:00:00 2001 From: Tristan Croiset Date: Sat, 15 Aug 2026 18:15:08 +0200 Subject: [PATCH] fix: forbid citing table doc profiling stats as data answers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a SQL Query Rules entry so the agent never answers a data question with row counts or aggregates from table documentation files (columns.md, preview.md, ai_summary.md, how_to_use.md) — those are stale profiling snapshots. Any number presented as an answer to a data question must come from a query executed in the conversation (or be computed from its results), which also keeps citation query ids real. Co-Authored-By: Claude Fable 5 --- apps/backend/src/components/ai/system-prompt.tsx | 7 +++++++ apps/backend/tests/system-prompt.test.ts | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/apps/backend/src/components/ai/system-prompt.tsx b/apps/backend/src/components/ai/system-prompt.tsx index 455f1caae..bbd4ae9e9 100644 --- a/apps/backend/src/components/ai/system-prompt.tsx +++ b/apps/backend/src/components/ai/system-prompt.tsx @@ -201,6 +201,13 @@ export function SystemPrompt({ an "exact" count based on the number of rows a limited query returned. To count rows, run a separate query using COUNT(*) (or COUNT over a subquery) without a LIMIT/TOP clause. , + + Table documentation files (columns.md, preview.md, ai_summary.md, how_to_use.md, ...) are for + understanding schema and semantics only. Their statistics (row counts, previews, aggregates) are + stale profiling snapshots — never present them as the answer to a data question. Any number you + present as an answer to a data question must come from a query executed in this conversation, or + be computed from such results. + , ...dialectSqlQueryRules, ]} diff --git a/apps/backend/tests/system-prompt.test.ts b/apps/backend/tests/system-prompt.test.ts index b01db95d0..481bb8d12 100644 --- a/apps/backend/tests/system-prompt.test.ts +++ b/apps/backend/tests/system-prompt.test.ts @@ -247,6 +247,17 @@ describe('SystemPrompt built-in skills', () => { }); }); +describe('SystemPrompt SQL query rules', () => { + it('forbids citing table documentation statistics as answers to data questions', () => { + const markdown = renderToMarkdown(SystemPrompt({})); + + expect(markdown).toContain('never present them as the answer to a data question'); + expect(markdown).toContain( + 'must come from a query executed in this conversation, or be computed from such results', + ); + }); +}); + describe('SystemPrompt display_map rules', () => { it('includes the display_map rule by default', () => { expect(renderToMarkdown(SystemPrompt({}))).toContain('display_map');