diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index d8f7e4c2..d5e5e54b 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -1209,6 +1209,270 @@ "run": 3, "sourcePath": "claude-code-sonnet-5/build-docs-003-api-keys-guide/run-3/result.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-006-database-tables", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "security", + "sql" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the app's tables exist and accept the rows its queries write", + "passed": true + }, + { + "name": "row level security is enabled on every table in the public schema", + "passed": true + }, + { + "name": "every table with row level security enabled carries at least one policy", + "passed": true + }, + { + "name": "a signed-out visitor can browse the starter routine library", + "passed": true + }, + { + "name": "a signed-out visitor cannot read anyone's routines", + "passed": true + }, + { + "name": "the signed-in owner reads their own routines and nobody else's", + "passed": true + }, + { + "name": "a signed-out visitor cannot create a routine", + "passed": true + }, + { + "name": "the agent read the Tables and Data guide the prompt referenced", + "passed": true, + "notes": "web_fetch" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Extract full guidance on creating tables in Supabase: schema conventions, primary keys, foreign key relationships, data types, column defaults (timestamps), NOT NULL, casing conventions, using the Table Editor vs SQL, and any specific recommendations for columns like created_at, and referencing auth.users for ownership.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 1832 + } + ] + }, + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", + "run": 1, + "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-1/result.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-006-database-tables", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "security", + "sql" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the app's tables exist and accept the rows its queries write", + "passed": true + }, + { + "name": "row level security is enabled on every table in the public schema", + "passed": true + }, + { + "name": "every table with row level security enabled carries at least one policy", + "passed": true + }, + { + "name": "a signed-out visitor can browse the starter routine library", + "passed": true + }, + { + "name": "a signed-out visitor cannot read anyone's routines", + "passed": true + }, + { + "name": "the signed-in owner reads their own routines and nobody else's", + "passed": true + }, + { + "name": "a signed-out visitor cannot create a routine", + "passed": true + }, + { + "name": "the agent read the Tables and Data guide the prompt referenced", + "passed": true, + "notes": "web_fetch" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Extract full guidance on creating tables in Supabase: syntax for CREATE TABLE, column types, primary keys, foreign key relationships, RLS enabling recommendations, and any best practices mentioned.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 1855 + } + ] + }, + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-2/result.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-006-database-tables", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "security", + "sql" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the app's tables exist and accept the rows its queries write", + "passed": true + }, + { + "name": "row level security is enabled on every table in the public schema", + "passed": true + }, + { + "name": "every table with row level security enabled carries at least one policy", + "passed": true + }, + { + "name": "a signed-out visitor can browse the starter routine library", + "passed": true + }, + { + "name": "a signed-out visitor cannot read anyone's routines", + "passed": true + }, + { + "name": "the signed-in owner reads their own routines and nobody else's", + "passed": true + }, + { + "name": "a signed-out visitor cannot create a routine", + "passed": true + }, + { + "name": "the agent read the Tables and Data guide the prompt referenced", + "passed": true, + "notes": "web_fetch" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Summarize all guidance on creating tables in Postgres/Supabase: syntax for CREATE TABLE, data types, primary keys, foreign key relationships, default column patterns (like created_at, id), schema conventions, and best practices for column naming, nullability, and constraints. Include full detail on any recommended patterns.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 2394 + } + ] + }, + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", + "run": 3, + "sourcePath": "claude-code-sonnet-5/build-docs-006-database-tables/run-3/result.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "regression", @@ -4985,6 +5249,263 @@ "run": 3, "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-003-api-keys-guide/run-3/result.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-006-database-tables", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "security", + "sql" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the app's tables exist and accept the rows its queries write", + "passed": true + }, + { + "name": "row level security is enabled on every table in the public schema", + "passed": true + }, + { + "name": "every table with row level security enabled carries at least one policy", + "passed": true + }, + { + "name": "a signed-out visitor can browse the starter routine library", + "passed": true + }, + { + "name": "a signed-out visitor cannot read anyone's routines", + "passed": true + }, + { + "name": "the signed-in owner reads their own routines and nobody else's", + "passed": true + }, + { + "name": "a signed-out visitor cannot create a routine", + "passed": true + }, + { + "name": "the agent read the Tables and Data guide the prompt referenced", + "passed": true, + "notes": "web_fetch" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Summarize how Supabase recommends creating tables: syntax, primary keys, data types, foreign key relationships (including one-to-many), schemas, and any RLS considerations mentioned in this guide.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 1736 + } + ] + }, + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", + "run": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-1/result.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-006-database-tables", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "security", + "sql" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the app's tables exist and accept the rows its queries write", + "passed": true + }, + { + "name": "row level security is enabled on every table in the public schema", + "passed": true + }, + { + "name": "every table with row level security enabled carries at least one policy", + "passed": true + }, + { + "name": "a signed-out visitor can browse the starter routine library", + "passed": true + }, + { + "name": "a signed-out visitor cannot read anyone's routines", + "passed": true + }, + { + "name": "the signed-in owner reads their own routines and nobody else's", + "passed": true + }, + { + "name": "a signed-out visitor cannot create a routine", + "passed": true + }, + { + "name": "the agent read the Tables and Data guide the prompt referenced", + "passed": true, + "notes": "web_fetch, web_fetch" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Summarize how to create tables in Supabase: recommended way to create migrations, primary keys, foreign keys, default values, timestamps, enabling RLS, naming conventions, and any best practices mentioned.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 1409 + }, + { + "source": "web_fetch", + "query": "Does this page mention Row Level Security (RLS), enabling RLS on tables, or any security warnings about tables in the public schema exposed via the API? Quote the relevant section verbatim if present.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 692 + } + ] + }, + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", + "run": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-2/result.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-docs-006-database-tables", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "rls", + "security", + "sql" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "the app's tables exist and accept the rows its queries write", + "passed": true + }, + { + "name": "row level security is enabled on every table in the public schema", + "passed": true + }, + { + "name": "every table with row level security enabled carries at least one policy", + "passed": true + }, + { + "name": "a signed-out visitor can browse the starter routine library", + "passed": true + }, + { + "name": "a signed-out visitor cannot read anyone's routines", + "passed": true + }, + { + "name": "the signed-in owner reads their own routines and nobody else's", + "passed": true + }, + { + "name": "a signed-out visitor cannot create a routine", + "passed": true + }, + { + "name": "the agent read the Tables and Data guide the prompt referenced", + "passed": true, + "notes": "web_fetch" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_fetch", + "query": "Summarize the guide's recommendations for creating tables, including: primary keys, foreign keys, schemas, data types, RLS enabling, and best practices/naming conventions. Include any exact syntax examples shown.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/tables.md" + } + ], + "resultChars": 1646 + } + ] + }, + "prompt": "I'm building a habit tracker. People sign up, create their own routines, and\ntick them off each day. There's also a starter set of routines anyone can\nbrowse before they sign up.\n\nCreate the database for it.\n\nThe rest of the team is already building against `src/queries.ts`, so use the\ntable and column names it reads and writes:\n\n- `routines` with `owner_id`, `title`, `cadence`, and `created_at`\n- `routine_logs` with `routine_id` and `completed_on`\n- `routine_library` with `title` and `category`\n\nRead the guide below before you start and rely on it for how to set this up,\nrather than on what you already know.\n\nREFERENCE\nhttps://supabase.com/docs/guides/database/tables.md", + "promptSourcePath": "evals/build-docs-006-database-tables/PROMPT.md", + "run": 3, + "sourcePath": "claude-code-sonnet-5-no-skills/build-docs-006-database-tables/run-3/result.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "regression", diff --git a/evals/build-docs-006-database-tables/EVAL.ts b/evals/build-docs-006-database-tables/EVAL.ts new file mode 100644 index 00000000..b7a3fd13 --- /dev/null +++ b/evals/build-docs-006-database-tables/EVAL.ts @@ -0,0 +1,124 @@ +import { + buildDocsResult, + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +import { + checkProtectedTablesHavePolicies, + checkRlsEnabled, + loadPolicies, + loadTableState, +} from './catalog.js'; +import { + checkAnonCannotCreateRoutine, + checkAppTablesAcceptItsRows, + checkOwnerReadsOwnRoutines, + checkRoutinesAreHidden, + checkStarterLibraryIsBrowsable, + setupFixtures, + type Fixtures, +} from './access.js'; + +const GUIDE_PATH = 'guides/database/tables'; + +const scorer: LocalStackScorer = async (ctx) => { + try { + // Snapshot the catalog before anything writes, so no probe can change what + // the schema checks see. + const tables = await loadTableState(ctx); + const policies = await loadPolicies(ctx); + + const setup = await setupFixtures(ctx); + const fixtures = 'fixtures' in setup ? setup.fixtures : undefined; + + const checks: CheckResult[] = [ + checkAppTablesAcceptItsRows(setup), + checkRlsEnabled(tables), + checkProtectedTablesHavePolicies(tables, policies), + await gated( + fixtures, + 'a signed-out visitor can browse the starter routine library', + checkStarterLibraryIsBrowsable + ), + await gated( + fixtures, + "a signed-out visitor cannot read anyone's routines", + checkRoutinesAreHidden + ), + await gated( + fixtures, + "the signed-in owner reads their own routines and nobody else's", + checkOwnerReadsOwnRoutines + ), + // Last of the probes: it writes to `routines` when the schema lets it, + // and every probe that reads that table has to have run already. + await gated( + fixtures, + 'a signed-out visitor cannot create a routine', + (f) => checkAnonCannotCreateRoutine(ctx, f) + ), + checkGuideWasRead(ctx), + ]; + + return { passed: checks.every((check) => check.passed), checks }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + passed: false, + checks: [ + { + name: 'scorer evaluated the schema the agent created', + passed: false, + notes: message, + }, + ], + }; + } +}; + +export default scorer; + +/** + * Runs a probe, or reports it failed for want of a schema to probe. + * + * The name is passed in rather than read off the result so the check list is + * the same length and carries the same names on every run. A setup failure + * that collapsed the list instead would be invisible in a summary, and the + * published series is keyed on these strings. + */ +async function gated( + fixtures: Fixtures | undefined, + name: string, + probe: (fixtures: Fixtures) => Promise +): Promise { + if (!fixtures) { + return { + name, + passed: false, + notes: + "not run: the app's tables did not accept the rows its queries write", + }; + } + return probe(fixtures); +} + +// A search_docs hit carries the guide's url in its result, not its request, so +// reuse the harness's own resolution rather than scanning the raw tool call. +function checkGuideWasRead(ctx: LocalStackEvalContext): CheckResult { + const calls = buildDocsResult(ctx.toolCalls).calls.filter((call) => + call.pages?.some((page) => page.url.includes(GUIDE_PATH)) + ); + const withContent = calls.filter((call) => call.hasContent); + return { + name: 'the agent read the Tables and Data guide the prompt referenced', + passed: withContent.length > 0, + notes: + withContent.length > 0 + ? withContent.map((call) => call.source).join(', ') + : calls.length > 0 + ? `reached the guide via ${calls.map((call) => call.source).join(', ')} but retrieved no page content` + : 'no docs call reached the guide', + }; +} diff --git a/evals/build-docs-006-database-tables/PROMPT.md b/evals/build-docs-006-database-tables/PROMPT.md new file mode 100644 index 00000000..eeac3f45 --- /dev/null +++ b/evals/build-docs-006-database-tables/PROMPT.md @@ -0,0 +1,45 @@ +--- +stage: build +suite: regression +interface: cli +product: + - database + - data-api +topic: + - rls + - security + - sql +services: + - gotrue + - kong + - postgrest +motivation: >- + the Tables and Data guide is the foundational page agents are pointed at to + create a schema, and it never once says to protect a table, so an agent that + follows it leaves every row readable over the Data API. Customers keep asking + the platform to close this by default (FDBKIN-2454, FDBKIN-33286, + FDBKIN-30897, FDBKIN-32570, FDBKIN-25441, FDBKIN-5065, FDBKIN-5041, + FDBKIN-28984), which says the documentation does not close it. This eval + determines whether the guide gets an agent to protect the tables it creates + when a user asks for a database and never mentions security. The prompt + deliberately omits that vocabulary, so read README.md before editing it. +--- + +I'm building a habit tracker. People sign up, create their own routines, and +tick them off each day. There's also a starter set of routines anyone can +browse before they sign up. + +Create the database for it. + +The rest of the team is already building against `src/queries.ts`, so use the +table and column names it reads and writes: + +- `routines` with `owner_id`, `title`, `cadence`, and `created_at` +- `routine_logs` with `routine_id` and `completed_on` +- `routine_library` with `title` and `category` + +Read the guide below before you start and rely on it for how to set this up, +rather than on what you already know. + +REFERENCE +https://supabase.com/docs/guides/database/tables.md diff --git a/evals/build-docs-006-database-tables/README.md b/evals/build-docs-006-database-tables/README.md new file mode 100644 index 00000000..3c0412dc --- /dev/null +++ b/evals/build-docs-006-database-tables/README.md @@ -0,0 +1,137 @@ +# build-docs-006-database-tables + +## What this eval measures + +**The page is the subject, not the agent.** The prompt is a product request plus +the url of [Tables and Data](https://supabase.com/docs/guides/database/tables), +and the checks say whether an agent that read that page produced a schema that +protects its own data. A gap in the page counts as a failure here. + +The one claim: **a page that teaches table creation and never mentions row level +security leaves an agent creating world-readable tables.** + +The page covers its own subject competently. Primary keys get a section +recommending `bigint generated always as identity`, foreign keys get a section, +and the `private` schema example is correct. It never once says to protect a +table. `row level security` appears one time in 573 lines, at the `security_invoker` +note about views. There is no `alter table ... enable row level security`, no +policy, and no `auth.uid()` anywhere on the page. + +## Do not reintroduce the vocabulary + +These words are stripped from `PROMPT.md` and from every seed file, and putting +any of them back turns the eval into a test of whether an agent can follow an +instruction: + +> RLS, row level security, policy, policies, secure, security, private, public, +> permission, access, grant, `auth.uid`, tenant, isolation + +`local/README.md` and the comments in `local/src/queries.ts` are written in +product vocabulary for the same reason. "This runs for visitors who have not +signed up yet" is a fact about the product. "This table must be publicly +readable" would be the answer. + +## The prompt carries the contract + +`local/` seeds a `supabase/` project and the app's data layer, and **no +migrations**. The schema is what the agent produces, so seeding one would +remove the subject. + +`PROMPT.md` and `local/src/queries.ts` both fix the table and column names: +`routines` (`owner_id`, `title`, `cadence`, `created_at`), `routine_logs` +(`routine_id`, `completed_on`), and `routine_library` (`title`, `category`). + +**The prompt states them because the page does not.** The scorer finds the +agent's tables by name, so a schema it cannot find scores near zero however well +that schema is protected. Leaving the names to be inferred measures whether the +agent read the data layer, which is a fact about the agent rather than about the +page. Naming them costs a discovery question and buys a positive control the +scorer can prove. + +`routines.id` has no fixed type on purpose. `uuid` and +`bigint generated always as identity` are both correct, so the scorer resolves +routine ids by title through a subselect rather than inventing one. + +## Do not drop the positive controls + +Two checks pass for an agent that built nothing useful, and both are what make +the rest mean something: + +- **`the app's tables exist and accept the rows its queries write`** gates all + four probes. Without it, `no rows came back` and `the data is protected` are + the same result: `!error && rows.length === 0` is true of an empty set. +- **`a signed-out visitor can browse the starter routine library`** and + **`the signed-in owner reads their own routines and nobody else's`** are why a + schema that refuses everyone is not a pass. Enabling row level security and + writing no policy fails both. + +The marker rows are written **after** the agent's code exists, with a value +scoped to the run, so a hardcoded array cannot clear a read control and no +literal leaks between runs. + +The probe that writes runs last. It inserts into `routines` when the schema lets +it, and every probe that reads that table has to have run first. + +**The write probe has to send the whole contract row.** An earlier version sent +`owner_id` and `title` only, so on a schema whose `cadence` is `not null` the +insert came back as a not-null violation and the check read that as a refusal, so a +wide-open table scored as protected. A probe asserting that a request was +*refused* has to send a request the database would otherwise accept, confirm the +row's absence as the superuser, and treat any error outside `42501` as +`could not measure` rather than as a pass. Do not narrow that insert again. + +## The guide has to actually be read + +`the agent read the Tables and Data guide the prompt referenced` resolves the url +from the harness's own docs result rather than the raw tool call, because a +`search_docs` hit carries the guide's url in its result and not in its request. + +**The reliance instruction in `PROMPT.md` is load-bearing. Do not remove it.** +The first baseline shipped without it, and all six runs came back with zero docs +calls. Agents wrote a correctly protected schema from memory, never opened the +page, and still scored 7/8. Without that sentence a pass is evidence about the +model rather than about the page. + +Even with it, the check proves the page was opened and not that the page caused +the outcome. A model that already knows to enable row level security produces the +same schema from memory, so read a pass as regression cover rather than as +attribution. + +## What this eval does not score + +- **Data types.** `schema-data-types.md` in `supabase-postgres-best-practices` + is impact HIGH, and the page lists `timestamp`/`timestamptz` and `int`/`bigint` + in a reference table while recommending neither side. It is a second claim and + a storage bug rather than a breach, so it is named here rather than measured. +- **Primary keys, foreign keys, `not null`, lowercase identifiers.** The page + transmits these, so a check on them would pass a solution written from the page + and measure nothing. They are the page working, not the page failing. +- **Views and `security_invoker`.** The one thing the page does say about row + level security. `build-docs-002-rls-guide` owns it. +- **Bulk loading with `COPY`.** Real content on the page, no security + consequence, and no affordance in the seed to exercise it. +- **Schema modeling.** The prompt names the tables and columns, so nothing here + measures whether an agent derives them. One shape protected correctly and + another shape protected correctly score the same. +- **Whether `anon` holds a write grant.** Planned, then dropped deliberately. Supabase's + default privileges grant `anon` write on new tables in `public` and the + standard pattern leaves them in place while row level security gates the rows, + so asserting the grant is gone would fail the canonical correct answer. + `a signed-out visitor cannot create a routine` measures the outcome instead. + +## Overlap with build-docs-002-rls-guide + +002 carries a check with almost this name and a much larger policy suite, but it +is a different question. 002 asks the agent to set up access rules and points at +the Row Level Security guide, so it measures whether *that* guide transmits row +level security **when row level security is what was asked for**. This eval asks +for a database, points at a page that is silent on the subject, and measures +whether an agent gets there unprompted. + +## A risk worth knowing + +This measures a docs gap that customers are asking the **platform** to close +(FDBKIN-2454, FDBKIN-30897). If the default grants for `anon` on new `public` +tables ever change, the central check saturates and the finding goes stale. That +is the "check measures the platform, not the page" trap, and it is better named +here than discovered on a CLI bump. diff --git a/evals/build-docs-006-database-tables/access.ts b/evals/build-docs-006-database-tables/access.ts new file mode 100644 index 00000000..5b9ce002 --- /dev/null +++ b/evals/build-docs-006-database-tables/access.ts @@ -0,0 +1,328 @@ +import { randomUUID } from 'node:crypto'; +import type { + CheckResult, + LocalStackEvalContext, + SupabaseClient, +} from '@supabase-evals/core'; +import { stripIndent } from 'common-tags'; + +const PASSWORD = 'secret123'; + +/** + * The error codes that mean access control did its job. `42501` is + * `insufficient_privilege`, which covers both a revoked grant and a row that + * fails a policy's `with check`. Row level security denying a SELECT produces + * no error at all, only an empty result. + */ +const REFUSAL = new Set(['42501']); + +export type Fixtures = { + anonClient: SupabaseClient; + ownerClient: SupabaseClient; + ownerId: string; + strangerId: string; + ownerRoutine: string; + strangerRoutine: string; + libraryRoutine: string; + intruderRoutine: string; +}; + +export type Setup = { fixtures: Fixtures } | { failure: string }; + +/** + * Signs two people up, then writes one row per table **after** the agent's + * schema exists, every value scoped to this run. + * + * Writing the rows here rather than in a seed migration is what makes the + * probes mean something: a hardcoded array in the agent's code cannot contain + * a marker that did not exist when the code was written, and no literal leaks + * between runs. + * + * The ids come back from `routines` by title rather than being chosen here, + * because the agent picks that column's type. A `uuid` and a + * `bigint generated always as identity` are both correct answers and only one + * of them accepts a value we invent. + */ +export async function setupFixtures( + ctx: LocalStackEvalContext +): Promise { + const anonClient = await ctx.getClient(); + const ownerClient = await ctx.getClient(); + const strangerClient = await ctx.getClient(); + const run = randomUUID().slice(0, 8); + + const { data: owner, error: ownerError } = await ownerClient.auth.signUp({ + email: `routines-owner-${run}@example.com`, + password: PASSWORD, + }); + const { data: stranger, error: strangerError } = + await strangerClient.auth.signUp({ + email: `routines-stranger-${run}@example.com`, + password: PASSWORD, + }); + + if ( + ownerError || + strangerError || + !owner.user?.id || + !owner.session || + !stranger.user?.id + ) { + return { + failure: + ownerError?.message ?? + strangerError?.message ?? + 'signed up without a session', + }; + } + + const fixtures: Fixtures = { + anonClient, + ownerClient, + ownerId: owner.user.id, + strangerId: stranger.user.id, + ownerRoutine: `routine-owner-${run}`, + strangerRoutine: `routine-stranger-${run}`, + libraryRoutine: `library-routine-${run}`, + intruderRoutine: `routine-intruder-${run}`, + }; + + const seeded = await execSql( + ctx, + stripIndent` + BEGIN; + + INSERT INTO routine_library (title, category) + SELECT '${fixtures.libraryRoutine}', 'starter' + WHERE NOT EXISTS ( + SELECT 1 FROM routine_library WHERE title = '${fixtures.libraryRoutine}' + ); + + INSERT INTO routines (owner_id, title, cadence) + SELECT v.owner_id, v.title, 'daily' + FROM (VALUES + ('${fixtures.ownerId}'::uuid, '${fixtures.ownerRoutine}'), + ('${fixtures.strangerId}'::uuid, '${fixtures.strangerRoutine}') + ) AS v(owner_id, title) + WHERE NOT EXISTS ( + SELECT 1 FROM routines WHERE title = v.title + ); + + INSERT INTO routine_logs (routine_id, completed_on) + SELECT r.id, DATE '2026-01-01' + FROM routines r + WHERE r.title = '${fixtures.ownerRoutine}'; + + COMMIT; + ` + ); + + if (!seeded.ok) return { failure: seeded.message }; + return { fixtures }; +} + +/** + * The control every probe below is gated on. It fails for an agent that built + * nothing, which is what stops `no rows came back` from reading as `the data is + * protected`. An empty result set satisfies both. + */ +export function checkAppTablesAcceptItsRows(setup: Setup): CheckResult { + const failed = 'failure' in setup; + return { + name: "the app's tables exist and accept the rows its queries write", + passed: !failed, + notes: failed ? setup.failure : undefined, + }; +} + +/** A visitor who has not signed up still has to see the marketing page. */ +export async function checkStarterLibraryIsBrowsable( + fixtures: Fixtures +): Promise { + const { data, error } = await fixtures.anonClient + .from('routine_library') + .select('title'); + const titles = titlesOf(data); + + return { + name: 'a signed-out visitor can browse the starter routine library', + passed: !error && titles.includes(fixtures.libraryRoutine), + notes: error + ? describeError(error) + : titles.includes(fixtures.libraryRoutine) + ? undefined + : `read the library and did not get the seeded routine back, got ${titles.length} row(s)`, + }; +} + +/** + * The claim. + * + * Row level security denies a SELECT by returning no rows rather than by + * erroring, and a revoked grant returns `42501`. Any other error means the read + * failed for a reason that has nothing to do with access control, and the probe + * reports that it could not measure rather than banking a pass. An error the + * check cannot attribute is not evidence the data was protected. + */ +export async function checkRoutinesAreHidden( + fixtures: Fixtures +): Promise { + const name = "a signed-out visitor cannot read anyone's routines"; + const { data, error } = await fixtures.anonClient + .from('routines') + .select('title'); + + if (error && !REFUSAL.has(error.code ?? '')) { + return { name, passed: false, notes: cannotMeasure(error) }; + } + + const titles = titlesOf(data); + const leaked = titles.filter( + (title) => + title === fixtures.ownerRoutine || title === fixtures.strangerRoutine + ); + + return { + name, + passed: leaked.length === 0, + notes: + leaked.length === 0 + ? error + ? `refused with ${describeError(error)}` + : undefined + : `read ${leaked.length} routine(s) belonging to someone else: ${leaked.join(', ')}`, + }; +} + +/** + * Read-side checks miss write exposure entirely. + * + * **The row is the whole contract row.** An earlier version sent only + * `owner_id` and `title`, so on a schema whose `cadence` is `not null` the + * insert was rejected as a not-null violation and the check read that as a + * refusal, so a wide-open table scored as protected. The insert now has to be one + * the database would accept if access control permitted it, and absence is + * confirmed as the superuser rather than inferred from the error. + */ +export async function checkAnonCannotCreateRoutine( + ctx: LocalStackEvalContext, + fixtures: Fixtures +): Promise { + const name = 'a signed-out visitor cannot create a routine'; + const { error } = await fixtures.anonClient.from('routines').insert({ + owner_id: fixtures.ownerId, + title: fixtures.intruderRoutine, + cadence: 'daily', + }); + + if (error && !REFUSAL.has(error.code ?? '')) { + return { name, passed: false, notes: cannotMeasure(error) }; + } + + const { rows } = await ctx.query( + `SELECT 1 FROM routines WHERE title = '${fixtures.intruderRoutine}'` + ); + const landed = rows.length > 0; + + return { + name, + passed: Boolean(error) && !landed, + notes: landed + ? `inserted ${fixtures.intruderRoutine} into routines while signed out` + : error + ? undefined + : 'the insert reported no error', + }; +} + +/** + * The other half of the claim. A schema that refuses everyone is not protected, + * it is broken, and without this check enabling row level security and writing + * no policy would score as a pass. + */ +export async function checkOwnerReadsOwnRoutines( + fixtures: Fixtures +): Promise { + const { data, error } = await fixtures.ownerClient + .from('routines') + .select('title'); + const titles = titlesOf(data); + const mine = titles.includes(fixtures.ownerRoutine); + const theirs = titles.includes(fixtures.strangerRoutine); + + return { + name: "the signed-in owner reads their own routines and nobody else's", + passed: !error && mine && !theirs, + notes: error + ? describeError(error) + : !mine + ? 'the owner could not read a routine they own' + : theirs + ? "the owner also read a stranger's routine" + : undefined, + }; +} + +/** + * Runs non-SELECT SQL against the local stack database as the superuser. + * + * `supabase status` writes two kinds of noise to stderr: a `Stopped services` + * line naming every service this eval does not start, and a CLI upgrade notice. + * Both are normal. Left in, they lead the failure note and bury the line that + * says what went wrong, which is how a genuine missing-table failure got read as + * a lost infrastructure run once already. Silence the status call and hoist the + * database's own error to the front. + */ +async function execSql( + ctx: LocalStackEvalContext, + sql: string +): Promise<{ ok: boolean; message: string }> { + const encoded = Buffer.from(sql, 'utf8').toString('base64'); + const result = await ctx.exec( + stripIndent` + DB_URL=$(supabase status -o json 2>/dev/null | node -e 'let input = ""; process.stdin.on("data", data => input += data); process.stdin.on("end", () => console.log(JSON.parse(input).DB_URL));') + echo ${encoded} | base64 -d | psql "$DB_URL" -v ON_ERROR_STOP=1 + ` + ); + + return { + ok: result.ok, + message: result.ok ? '' : significantLines(result.stderr || result.stdout), + }; +} + +/** Drops the known-benign CLI chatter and puts the database's error first. */ +function significantLines(output: string): string { + const noise = [ + /^Stopped services:/, + /^A new version of Supabase CLI/, + /^We recommend updating regularly/, + ]; + const lines = output + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .filter((line) => !noise.some((pattern) => pattern.test(line))); + + const errors = lines.filter((line) => + /^(ERROR|FATAL|DETAIL|HINT)\b/.test(line) + ); + const ordered = + errors.length > 0 + ? [...errors, ...lines.filter((line) => !errors.includes(line))] + : lines; + return ordered.join('; ') || output.trim(); +} + +function titlesOf(data: unknown): string[] { + if (!Array.isArray(data)) return []; + return data.map((row) => String((row as { title?: unknown }).title ?? '')); +} + +function cannotMeasure(error: { code?: string; message: string }): string { + return `could not measure: the app's own request failed with ${describeError(error)}, which is not a refusal`; +} + +function describeError(error: { code?: string; message: string }): string { + return error.code ? `error ${error.code}: ${error.message}` : error.message; +} diff --git a/evals/build-docs-006-database-tables/catalog.ts b/evals/build-docs-006-database-tables/catalog.ts new file mode 100644 index 00000000..9c497b7d --- /dev/null +++ b/evals/build-docs-006-database-tables/catalog.ts @@ -0,0 +1,123 @@ +import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core'; +import { stripIndent } from 'common-tags'; + +/** The tables `local/src/queries.ts` builds against. The seed fixes these names. */ +export const CONTRACT_TABLES = [ + 'routines', + 'routine_logs', + 'routine_library', +] as const; + +export type TableState = { + relname: string; + relkind: string; + relrowsecurity: boolean; +}; + +export type PolicyRow = { + tablename: string; + policyname: string; +}; + +/** + * Every relation in the exposed schema, not only the three the app names. An + * extra table the agent invented is just as reachable over the Data API. + * + * The `pg_depend` anti-join drops objects an extension owns: pgTAP creates + * views in `public` when it is installed there, and those are not the agent's. + */ +export async function loadTableState( + ctx: LocalStackEvalContext +): Promise { + const { rows } = await ctx.query(stripIndent` + SELECT + c.relname, + c.relkind::text AS relkind, + c.relrowsecurity + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' + AND c.relkind IN ('r', 'p') + AND NOT EXISTS ( + SELECT 1 + FROM pg_depend d + WHERE d.classid = 'pg_class'::regclass + AND d.objid = c.oid + AND d.deptype = 'e' + ) + `); + return rows as TableState[]; +} + +export async function loadPolicies( + ctx: LocalStackEvalContext +): Promise { + const { rows } = await ctx.query(stripIndent` + SELECT tablename, policyname + FROM pg_policies + WHERE schemaname = 'public' + `); + return rows as PolicyRow[]; +} + +/** + * The central catalog claim. Named for the class rather than the method: the + * page could close this gap by linking the Row Level Security guide, by showing + * `alter table ... enable row level security` inline, or by moving the tables + * into an unexposed schema, and only the first two land here. + */ +export function checkRlsEnabled(tables: TableState[]): CheckResult { + const absent = CONTRACT_TABLES.filter( + (name) => !tables.some((table) => table.relname === name) + ); + const unprotected = tables + .filter((table) => table.relrowsecurity !== true) + .map((table) => table.relname); + const passed = absent.length === 0 && unprotected.length === 0; + + return { + name: 'row level security is enabled on every table in the public schema', + passed, + notes: passed + ? undefined + : [ + absent.length > 0 + ? `the app's tables are missing: ${absent.join(', ')}` + : null, + unprotected.length > 0 + ? `row level security not enabled on: ${unprotected.join(', ')}` + : null, + ] + .filter(Boolean) + .join('; '), + }; +} + +/** + * Enabling row level security and stopping there locks the app out of its own + * data, which is the failure FDBKIN-5041 describes. Kept to a low bar of one + * policy, because whether the policies are *right* is what the behavioral + * probes settle, and an ambitious check here would fail the whole eval. + */ +export function checkProtectedTablesHavePolicies( + tables: TableState[], + policies: PolicyRow[] +): CheckResult { + const withPolicy = new Set(policies.map((policy) => policy.tablename)); + const bare = tables + .filter((table) => table.relrowsecurity === true) + .filter((table) => !withPolicy.has(table.relname)) + .map((table) => table.relname); + const enabled = tables.filter((table) => table.relrowsecurity === true); + + return { + name: 'every table with row level security enabled carries at least one policy', + passed: enabled.length > 0 && bare.length === 0, + notes: + enabled.length === 0 + ? 'no table has row level security enabled, so there is nothing to police' + : bare.length === 0 + ? undefined + : `enabled with no policy, so nothing reaches these tables: ${bare.join(', ')}`, + }; +} diff --git a/evals/build-docs-006-database-tables/local/README.md b/evals/build-docs-006-database-tables/local/README.md new file mode 100644 index 00000000..b80e5400 --- /dev/null +++ b/evals/build-docs-006-database-tables/local/README.md @@ -0,0 +1,10 @@ +# Routines + +A habit tracker. People sign up, add routines of their own, and tick each one +off on the days they do it. + +The marketing page carries a starter set of routines that visitors can browse +before they sign up. + +`src/queries.ts` is the data layer both clients import. The database it talks to +does not exist yet. diff --git a/evals/build-docs-006-database-tables/local/src/queries.ts b/evals/build-docs-006-database-tables/local/src/queries.ts new file mode 100644 index 00000000..d8ecb76f --- /dev/null +++ b/evals/build-docs-006-database-tables/local/src/queries.ts @@ -0,0 +1,60 @@ +// The data layer. The web app and the mobile app both import from here, so the +// table and column names below are fixed — the rest of the team is already +// building against them. +// +// Nothing in this file creates the database. That part is still missing. + +import type { SupabaseClient } from '@supabase/supabase-js'; + +/** + * The starter routines on the marketing page. This runs for visitors who have + * not signed up yet, so it goes out with no one signed in. + */ +export async function listStarterRoutines(supabase: SupabaseClient) { + return supabase + .from('routine_library') + .select('id, title, category') + .order('title'); +} + +/** The signed-in person's own routines, newest last. */ +export async function listMyRoutines(supabase: SupabaseClient) { + return supabase + .from('routines') + .select('id, title, cadence, created_at') + .order('created_at'); +} + +/** Called from the "new routine" form. `ownerId` is the signed-in person. */ +export async function createRoutine( + supabase: SupabaseClient, + ownerId: string, + title: string, + cadence: string +) { + return supabase + .from('routines') + .insert({ owner_id: ownerId, title, cadence }); +} + +/** Ticking a routine off for a given day. `day` is a calendar date. */ +export async function tickOff( + supabase: SupabaseClient, + routineId: string, + day: string +) { + return supabase + .from('routine_logs') + .insert({ routine_id: routineId, completed_on: day }); +} + +/** The tick-off history behind a routine's streak counter. */ +export async function logsForRoutine( + supabase: SupabaseClient, + routineId: string +) { + return supabase + .from('routine_logs') + .select('id, completed_on') + .eq('routine_id', routineId); +} diff --git a/evals/build-docs-006-database-tables/local/supabase/config.toml b/evals/build-docs-006-database-tables/local/supabase/config.toml new file mode 100644 index 00000000..4b74848d --- /dev/null +++ b/evals/build-docs-006-database-tables/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-database-tables" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)"