From 34e4ffd0e1cef58d95df34aa33d386971e5f4ec8 Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Mon, 27 Jul 2026 01:52:34 +0200 Subject: [PATCH 1/5] =?UTF-8?q?chore(backlog):=20TASK-53=20wave-B=20regexp?= =?UTF-8?q?=20family=20=E2=80=94=20scope=20+=20differential-spike=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- ...ons-~-~-SIMILAR-TO-deferred-star-forms.md" | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 "backlog/tasks/task-53 - Specializer-wave-B-\342\200\224-regexp-family-regexp_-functions-~-~-SIMILAR-TO-deferred-star-forms.md" diff --git "a/backlog/tasks/task-53 - Specializer-wave-B-\342\200\224-regexp-family-regexp_-functions-~-~-SIMILAR-TO-deferred-star-forms.md" "b/backlog/tasks/task-53 - Specializer-wave-B-\342\200\224-regexp-family-regexp_-functions-~-~-SIMILAR-TO-deferred-star-forms.md" new file mode 100644 index 0000000..98ee7b2 --- /dev/null +++ "b/backlog/tasks/task-53 - Specializer-wave-B-\342\200\224-regexp-family-regexp_-functions-~-~-SIMILAR-TO-deferred-star-forms.md" @@ -0,0 +1,41 @@ +--- +id: TASK-53 +title: >- + Specializer wave B — regexp family (regexp_* functions, ~/!~/SIMILAR TO, + deferred star forms) +status: In Progress +assignee: [] +created_date: '2026-07-26 23:50' +labels: [] +milestone: m-7 +dependencies: + - TASK-52 +priority: high +type: feature +ordinal: 47000 +--- + +## Description + + +Post-wave-5 pool (census at master ba8bf98: 484 match / 192 unsupported / 2 known-divergent): the regexp bucket is the largest coherent workable slice (~25-30 cases + unlocks). Scope: + +- regexp_matches / regexp_full_match / regexp_extract / regexp_replace (scalar forms; list/table-returning forms classify clean as non-scalar) +- operators ~ and !~ (DuckDB binds them to regexp_full_match), SIMILAR TO on values +- deferred wave-5 star forms: * SIMILAR TO 're' (unanchored name search + the pinned NOT asymmetry) and COLUMNS('re') + +Engine decision: Rust `regex` crate (RE2-lineage, pure Rust; NOT in the tree yet — new direct dependency). DuckDB uses RE2 — the pins fleet must run a DIFFERENTIAL battery (duckdb vs rust-regex side by side) to pin exactly where they disagree (\d/\w/\s Unicode-ness, (?i) fold scope, empty matches in replace, error classes for invalid patterns); divergent corners classify clean-unsupported rather than serving wrong answers. + +Pins-first; wave-3 over-generalization precedent applies — every claim needs an executed query/program recorded. + + +## Acceptance Criteria + +- [ ] #1 Pins-first: wave-B pins spec (md + JSON) committed before implementation, incl. the duckdb-vs-rust-regex differential battery and the crate decision +- [ ] #2 regexp_matches / regexp_full_match / regexp_extract / regexp_replace serve per pins (options strings, group semantics, backrefs, NULL handling); list/table forms classify clean +- [ ] #3 Operators ~ / !~ and SIMILAR TO on values serve per pins (anchoring semantics measured, not assumed) +- [ ] #4 Deferred star forms serve: * SIMILAR TO name filter (incl. the pinned NOT asymmetry) and COLUMNS('re') expansion +- [ ] #5 Patterns whose semantics differ between RE2 and rust-regex classify clean-unsupported (guard at bind time from the pinned divergence list) — never a wrong answer +- [ ] #6 Corpus replay: three-outcome contract holds, zero FAILs, match count reported +- [ ] #7 Gate green both backends, clippy clean, serving-bench parity gate passes + From b0031223554e1390c49c4255c2af355b1d5d3515 Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Mon, 27 Jul 2026 02:06:45 +0200 Subject: [PATCH 2/5] =?UTF-8?q?docs(specializer):=20wave-B=20regexp=20pins?= =?UTF-8?q?=20=E2=80=94=206-area=20fleet=20incl.=20RE2-vs-rust-regex=20dif?= =?UTF-8?q?ferential,=20measured=20before=20implementation=20(TASK-53)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- .../specs/2026-07-27-waveB-regexp-pins.md | 126 ++++++++ .../superpowers/specs/pins-waveB/extract.json | 197 ++++++++++++ .../specs/pins-waveB/matches-operators.json | 178 +++++++++++ .../specs/pins-waveB/options-errors.json | 218 +++++++++++++ .../specs/pins-waveB/re2-vs-rust-regex.json | 131 ++++++++ .../superpowers/specs/pins-waveB/replace.json | 286 ++++++++++++++++++ .../pins-waveB/star-similar-columns.json | 243 +++++++++++++++ 7 files changed, 1379 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-27-waveB-regexp-pins.md create mode 100644 docs/superpowers/specs/pins-waveB/extract.json create mode 100644 docs/superpowers/specs/pins-waveB/matches-operators.json create mode 100644 docs/superpowers/specs/pins-waveB/options-errors.json create mode 100644 docs/superpowers/specs/pins-waveB/re2-vs-rust-regex.json create mode 100644 docs/superpowers/specs/pins-waveB/replace.json create mode 100644 docs/superpowers/specs/pins-waveB/star-similar-columns.json diff --git a/docs/superpowers/specs/2026-07-27-waveB-regexp-pins.md b/docs/superpowers/specs/2026-07-27-waveB-regexp-pins.md new file mode 100644 index 0000000..d8eb58c --- /dev/null +++ b/docs/superpowers/specs/2026-07-27-waveB-regexp-pins.md @@ -0,0 +1,126 @@ +# Wave B — regexp family: pins (TASK-53) + +Measured 2026-07-27 against DuckDB 1.5.5 by a 6-agent fleet (5 DuckDB areas ++ one DuckDB-RE2 vs rust-regex DIFFERENTIAL battery); every claim is backed +by an executed query/program recorded verbatim in `pins-waveB/*.json`. +Baseline at master ba8bf98: 484 match / 192 unsupported / 2 known-divergent. + +## The engine decision (pins-waveB/re2-vs-rust-regex.json) + +**rust `regex` 1.13.1 is the engine, behind a bind-time translation layer; +raw pass-through is wrong for any pattern with a Perl class.** All 98 +differential battery entries are byte-identical or identically-rejected +with the following applied: + +- Config: `RegexBuilder::octal(true)`; **keep default Unicode mode** — + `unicode(false)` was measured to BREAK `(?i)` folding parity (KELVIN, + sharp-s agree by default; the "obvious" global switch is the wrong fix). +- Pattern rewrite (Perl classes are the whole Unicode gap — RE2 ASCII, + rust Unicode), each verified byte-for-byte incl. inside `[...]` classes: + `\d`→`(?-u:\d)`, `\D`→`[^0-9]`, `\w`→`[0-9A-Za-z_]`, + `\W`→`[^0-9A-Za-z_]`, `\s`→`[\t\n\f\r ]`, `\S`→`[^\t\n\f\r ]`, + `\b`→`(?-u:\b)`. NOTE: `(?-u:\s)` is the WRONG `\s` rewrite (rust ASCII + `\s` includes VT); `(?-u:\W)` does not compile — negated classes must be + spelled as Unicode-mode `[^...]`. +- Bind-time REJECT list (each measured; these differ or are unservable): + `\B` (DuckDB itself dies at runtime on non-ASCII — RE2's ASCII \B matches + inside multibyte chars), `(?...)` angle groups (DuckDB rejects), + duplicate group names (rust rejects, DuckDB accepts), repetition bounds + > 1000 (`a{1001}` DuckDB error), **stacked quantifiers `a*+` (DuckDB + errors; rust silently reinterprets as `(a*)+` — a wrong-answer risk, not + an error-shape mismatch)**, `\uXXXX` escapes, `\Q...\E` (rust rejects; + optionally pre-expandable via regex::escape). +- Replacement templates translate, not the pattern: DuckDB is `\N` with + literal `$`; rust is `$N` with literal `\N` — map `\N`→`${N}`, `$`→`$$`. +- Agrees with zero config: leftmost-first alternation, greedy/lazy/(?U)/ + (?s), empty-match iteration incl. replace_all adjacency skip, `\A`/`\z`, + `$` without trailing-newline magic, mid-pattern inline flags, `\p{...}`, + `[]a]` (legal, same meaning), backrefs/lookaround/`\Z`/`a{2,1}` rejected + by both. + +## Operators and match functions (pins-waveB/matches-operators.json) + +- `regexp_matches(s, p)` = **unanchored SEARCH**; + `regexp_full_match(s, p)` = whole-string. +- **`~` is FULL match, not search** (binds to regexp_full_match — the + binder error names it; diverges from PostgreSQL). `!~` = NOT(~) with + standard NULL propagation. +- **SIMILAR TO does NO wildcard translation**: the pattern goes RAW to RE2 + with full-match anchoring (`'hello' SIMILAR TO 'h%o'` is FALSE; `h.llo` + is TRUE). It is exactly regexp_full_match. `SIMILAR TO ... ESCAPE` is + DuckDB "Not implemented Error: Custom escape in SIMILAR TO". +- Empty pattern: regexp_matches(s,'') true for EVERY non-NULL string; + full-match/`~` with '' true only for ''. +- NULL string/pattern → NULL; **NULL options arg → error** ("Regex options + field must not be NULL"), not NULL. All results BOOLEAN. +- Constant patterns compile at BIND time (errors fire under WHERE 1=0 / + EXPLAIN / PREPARE); column patterns compile at EXECUTION (WHERE false + suppresses) — column patterns stay unsupported in v0 (prepare-time + compilation is the engine model). + +## regexp_extract (pins-waveB/extract.json) + +Default/group-0 = whole match; groups 1..n; **no match → `''`, NEVER +NULL** — same for non-participating groups and in-range indexes above the +pattern's group count. Unanchored leftmost-first search. **Group index is a +flat 0..9 range check unrelated to the pattern** ("Group index must be +between 0 and 9!" even when 10 groups exist; also for negatives; fires even +on non-matching subjects). NULL subject/pattern → NULL but **NULL group → +`''`**. Options as 4th arg only (no (V,V,VARCHAR) overload — 3rd-arg +options is a binder ambiguity). Group index and options must be constants. +Name-list STRUCT form: non-scalar, classify clean. Result VARCHAR. + +## regexp_replace (pins-waveB/replace.json) + +First-match-only by default; `'g'` global. Rewrite grammar: `\0` whole +match, `\1`-`\9`, `\\` literal backslash, `\10` = `\1` + literal '0'; +`$1`/`$0`/`&` are PLAIN LITERALS. **Invalid rewrites never error, and +global vs non-global diverge**: non-global → input unchanged for ANY +invalid rewrite; global → out-of-range backref is still a full no-op +(MaxSubmatch pre-check) but a bad escape CONSUMES each match and emits only +the prefix before the bad escape (`('hello','h','\x','g')` = `'ello'`). +Non-participating in-range group → empty. Empty-match global insert at +every codepoint boundary incl. both ends (`'abc'` → `'XaXbXcX'`), no empty +match immediately after a nonempty one, never inside UTF-8 bytes. ANY NULL +arg (incl. options — asymmetric with the other functions!) → NULL. + +## Options + compile errors (pins-waveB/options-errors.json) + +Alphabet: `c i l m n p s` everywhere, `g` ONLY regexp_replace ("Option 'g' +(global replace) is only valid for regexp_replace" elsewhere); unknown → +"Unrecognized Regex option ". Parsed left-to-right, whitespace +skipped, LAST conflicting letter wins ('ci' insensitive, 'ic' sensitive). +**`m`/`n`/`p` are functional no-ops** (they do NOT enable per-line ^/$; +multiline exists only via inline `(?m)`); only `s` (dotall) changes +matching; `l` = literal. Inline flags override the options arg. Verbatim +compile-error texts recorded (missing ]/), bad repetition, `**`, trailing +backslash, `\1` backref, lookahead) — all "Invalid Input Error: " + RE2 +text; ours mirror the shape with rust-regex's message where texts differ +(class-correct; corpus success-cases never see them). + +## Star forms (pins-waveB/star-similar-columns.json) + +Deepens the wave-5 star pins: positive `* SIMILAR TO` / `COLUMNS('re')` = +unanchored RE2 search over declared-case names; `* NOT SIMILAR TO` = +NOT-full-match — **two independent predicates, never derived from each +other** ('a.*' selects {abc,abd,Weird Name} while NOT 'a.*' selects +{xyz,Weird Name}: "Weird Name" is in BOTH). NOT-form cannot produce the +zero-match error at all. Pattern must be a bare string literal. COLUMNS +expansion is always table-declaration order (even for alternation and +list forms); an alias stamps EVERY expansion (duplicates OK — feeds the +wave-5 dedup); un-aliased COLUMNS exprs keep bare column names. Grammar: +EXCLUDE-then-filter only. COLUMNS('re') in scope: SELECT-list expansion +(incl. inside an expression, zipped per-column); COLUMNS in WHERE +(AND-conjunction) and multi-set expressions stay unsupported. + +## Implementation stages (each lands with tests + corpus replay green) + +1. Regex infrastructure: `regex = "1"` dependency; translation module + (pattern rewrite + reject list + options parser + replacement-template + translation); Program-level compiled-regex table (patterns are + prepare-time constants); IR ops + print/parse/verify + both backends. +2. Functions + operators: regexp_matches / regexp_full_match / + regexp_extract / regexp_replace, `~` / `!~`, SIMILAR TO on values. +3. Star forms: * SIMILAR TO / NOT SIMILAR TO name filters (marker codes in + rewrite.rs), COLUMNS('re') interception + expansion. +4. Census + bench parity + close-out. diff --git a/docs/superpowers/specs/pins-waveB/extract.json b/docs/superpowers/specs/pins-waveB/extract.json new file mode 100644 index 0000000..1a70fca --- /dev/null +++ b/docs/superpowers/specs/pins-waveB/extract.json @@ -0,0 +1,197 @@ +{ + "area": "extract", + "duckdb_version": "v1.5.5", + "setup": "CREATE TABLE t(s VARCHAR); INSERT INTO t VALUES ('hello world'), ('abc123def'), (''), (NULL), ('héllo') -- multi-statement pins: query lines map 1:1 to ' | '-separated observed segments", + "pins": [ + { + "claim": "Default (no group arg) returns the WHOLE match (group 0 semantics), even when capture groups exist", + "query": "SELECT regexp_extract('hello world', 'l+')\nSELECT regexp_extract('hello world', '(\\w+) (\\w+)')", + "observed": "[('ll',)] | [('hello world',)]" + }, + { + "claim": "Explicit group 0 = whole match and is identical to the 2-arg form", + "query": "SELECT regexp_extract('hello world', '(\\w+) (\\w+)', 0)\nSELECT regexp_extract('abc123','[0-9]+') = regexp_extract('abc123','[0-9]+',0)", + "observed": "[('hello world',)] | [(True,)]" + }, + { + "claim": "Explicit groups 1 and 2 return the capture texts", + "query": "SELECT regexp_extract('hello world', '(\\w+) (\\w+)', 1), regexp_extract('hello world', '(\\w+) (\\w+)', 2)", + "observed": "[('hello', 'world')]" + }, + { + "claim": "Group index beyond the pattern's group count but <= 9 is NOT an error — returns '' (group 3 of 2-group pattern; group 1 of 0-group pattern; group 9 of 2-group pattern)", + "query": "SELECT regexp_extract('hello world', '(\\w+) (\\w+)', 3), regexp_extract('hello world', 'l+', 1), regexp_extract('ab', '(a)(b)', 9)", + "observed": "[('', '', '')]" + }, + { + "claim": "Group index is hard-capped at 0..9: group 10 errors EVEN with a 10-group pattern; group 9 of that pattern works", + "query": "SELECT regexp_extract('abcdefghij', '(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)', 10)\nSELECT regexp_extract('abcdefghij', '(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)', 9)", + "observed": "ERROR InvalidInputException: Invalid Input Error: Group index must be between 0 and 9! | [('i',)]" + }, + { + "claim": "Negative group index -> same range error; the error fires regardless of whether the subject would match", + "query": "SELECT regexp_extract('abc', '(a)', -1)\nSELECT regexp_extract('zzz', '(a)', 100)", + "observed": "ERROR InvalidInputException: Invalid Input Error: Group index must be between 0 and 9! | ERROR InvalidInputException: Invalid Input Error: Group index must be between 0 and 9!" + }, + { + "claim": "NO MATCH returns empty string '', NOT NULL — with or without an explicit group", + "query": "SELECT regexp_extract('abc', 'x'), regexp_extract('abc', 'x') IS NULL, regexp_extract('abc', 'x') = ''\nSELECT regexp_extract('abc', '(x)', 1), regexp_extract('abc', '(x)', 1) IS NULL", + "observed": "[('', False, True)] | [('', False)]" + }, + { + "claim": "Matching is an unanchored SEARCH (pattern found anywhere), not a full-string match", + "query": "SELECT regexp_extract('abc123def', '[0-9]+'), regexp_extract('hello world', 'lo wo')", + "observed": "[('123', 'lo wo')]" + }, + { + "claim": "Explicit anchors are honored within the search: ^world fails, ^hello and world$ succeed", + "query": "SELECT regexp_extract('hello world', '^world'), regexp_extract('hello world', '^hello'), regexp_extract('hello world', 'world$')", + "observed": "[('', 'hello', 'world')]" + }, + { + "claim": "Only the FIRST (leftmost) match is returned; alternation is leftmost-FIRST not longest ('a|aa' on 'aaa' -> 'a')", + "query": "SELECT regexp_extract('a1b22c333', '[0-9]+'), regexp_extract('aaa', 'a|aa')", + "observed": "[('1', 'a')]" + }, + { + "claim": "NULL subject -> NULL; NULL pattern -> NULL", + "query": "SELECT regexp_extract(NULL, 'a'), regexp_extract(NULL, 'a') IS NULL, regexp_extract('a', NULL), regexp_extract('a', NULL) IS NULL", + "observed": "[(None, True, None, True)]" + }, + { + "claim": "SURPRISE: NULL group index does NOT return NULL — it returns '' (both literal NULL and CAST(NULL AS INTEGER))", + "query": "SELECT regexp_extract('abc', '(a)', NULL), regexp_extract('abc', '(a)', NULL) IS NULL, regexp_extract('abc', '(a)', CAST(NULL AS INTEGER)) = ''", + "observed": "[('', False, True)]" + }, + { + "claim": "Empty pattern / empty subject / zero-length matches all return '' (never NULL): empty pattern on 'abc' and ''; group 1 of empty pattern; 'a' and '.*' on ''; 'x*','b*' on 'abc'", + "query": "SELECT regexp_extract('abc', ''), regexp_extract('abc', '') IS NULL, regexp_extract('', ''), regexp_extract('abc', '', 1)\nSELECT regexp_extract('', 'a'), regexp_extract('', 'a') = '', regexp_extract('', '.*'), regexp_extract('', '.*') = ''\nSELECT regexp_extract('abc', 'x*'), regexp_extract('abc', 'b*')", + "observed": "[('', False, '', '')] | [('', True, '', True)] | [('', '')]" + }, + { + "claim": "Result type is VARCHAR for both the 2-arg and 3-arg integer forms", + "query": "SELECT typeof(regexp_extract('abc', 'b')), typeof(regexp_extract('abc', '(b)', 1))", + "observed": "[('VARCHAR', 'VARCHAR')]" + }, + { + "claim": "Nested groups number left-to-right by open paren: '(a(b)c)' on 'abc' -> g0='abc', g1='abc', g2='b', g3=''", + "query": "SELECT regexp_extract('abc', '(a(b)c)', 0), regexp_extract('abc', '(a(b)c)', 1), regexp_extract('abc', '(a(b)c)', 2), regexp_extract('abc', '(a(b)c)', 3)", + "observed": "[('abc', 'abc', 'b', '')]" + }, + { + "claim": "Non-participating groups return '' NOT NULL: alternation '(a)|(b)' on 'b' g1='' (IS NULL False) g2='b'; optional 'a(b)?c' on 'ac' g1=''", + "query": "SELECT regexp_extract('b', '(a)|(b)', 1), regexp_extract('b', '(a)|(b)', 1) IS NULL, regexp_extract('b', '(a)|(b)', 2)\nSELECT regexp_extract('ac', 'a(b)?c', 1), regexp_extract('ac', 'a(b)?c', 1) = ''", + "observed": "[('', False, 'b')] | [('', True)]" + }, + { + "claim": "Name-list 3rd arg returns a STRUCT (non-scalar; out of scalar-engine scope): one field per name, values = groups 1..N positionally, field order = list order", + "query": "SELECT regexp_extract('2023-04-15', '(\\d+)-(\\d+)-(\\d+)', ['y','m','d'])\nSELECT typeof(regexp_extract('2023-04-15', '(\\d+)-(\\d+)-(\\d+)', ['y','m','d']))\nSELECT regexp_extract('2023-04', '(\\d+)-(\\d+)', ['second_named_first','other'])", + "observed": "[({'y': '2023', 'm': '04', 'd': '15'},)] | [('STRUCT(y VARCHAR, m VARCHAR, d VARCHAR)',)] | [({'second_named_first': '2023', 'other': '04'},)]" + }, + { + "claim": "Name-list form: NO MATCH -> struct of all '' fields; NULL subject -> NULL (whole struct)", + "query": "SELECT regexp_extract('abc', '(\\d+)-(\\d+)', ['a','b'])\nSELECT regexp_extract(NULL, '(\\d+)', ['a']), regexp_extract(NULL, '(\\d+)', ['a']) IS NULL", + "observed": "[({'a': '', 'b': ''},)] | [(None, True)]" + }, + { + "claim": "Name-list may name FEWER groups than the pattern has — takes the first N", + "query": "SELECT regexp_extract('2023-04', '(\\d+)-(\\d+)', ['a'])", + "observed": "[({'a': '2023'},)]" + }, + { + "claim": "Name-list bind errors (verbatim): more names than groups; empty list; duplicate name; NULL name", + "query": "SELECT regexp_extract('2023-04', '(\\d+)-(\\d+)', ['a','b','c'])\nSELECT regexp_extract('abc', '(a)', [])\nSELECT regexp_extract('ab', '(a)(b)', ['x','x'])\nSELECT regexp_extract('ab', '(a)(b)', ['x', NULL])", + "observed": "ERROR BinderException: Binder Error: Not enough capturing groups (2) for provided names (3) | ERROR BinderException: Binder Error: Group name list must be non-empty | ERROR BinderException: Binder Error: Duplicate group name 'x' in regexp_extract | ERROR BinderException: Binder Error: NULL group name in regexp_extract" + }, + { + "claim": "Empty-string name is ALLOWED and yields an unnamed struct field: typeof STRUCT(VARCHAR, VARCHAR)", + "query": "SELECT regexp_extract('ab', '(a)(b)', ['', 'y'])\nSELECT typeof(regexp_extract('ab', '(a)(b)', ['', 'y']))", + "observed": "[(('a', 'b'),)] | [('STRUCT(VARCHAR, VARCHAR)',)]" + }, + { + "claim": "Name-list requires a CONSTANT pattern (NULL pattern is a bind error here — unlike scalar form) and a CONSTANT list (column list is a runtime error)", + "query": "SELECT regexp_extract('ab', NULL, ['x'])\nSELECT regexp_extract('ab', '(a)(b)', l) FROM (VALUES (['x','y'])) tl(l)", + "observed": "ERROR BinderException: Binder Error: regexp_extract with LIST of group names requires a constant pattern | ERROR InvalidInputException: Invalid Input Error: Group specification field must be a constant!" + }, + { + "claim": "4th options arg: 'i' = case-insensitive; extracted text preserves ORIGINAL case; works for int-group and name-list forms", + "query": "SELECT regexp_extract('ABC123', '([a-z]+)', 1, 'i'), regexp_extract('ABC', 'abc', 0, 'i')\nSELECT regexp_extract('ABC-12', '([a-z]+)-(\\d+)', ['w','n'], 'i')", + "observed": "[('ABC', 'ABC')] | [({'w': 'ABC', 'n': '12'},)]" + }, + { + "claim": "No (VARCHAR,VARCHAR,VARCHAR) overload — options as 3rd arg is an ambiguity bind error; BIGINT group index is not implicitly cast (error lists the full 5-overload set)", + "query": "SELECT regexp_extract('ABC', 'abc', 'i')\nSELECT regexp_extract('abc123', '([a-z]+)([0-9]+)', CAST(2 AS BIGINT))", + "observed": "ERROR BinderException: Binder Error: Could not choose a best candidate function for the function call \"regexp_extract(STRING_LITERAL, STRING_LITERAL, STRING_LITERAL)\". In order to select one, please add explicit type casts.\n\tCandidate functions:\n\tregexp_extract(VARCHAR, VARCHAR, VARCHAR[]) -> VARCHAR\n\tregexp_extract(VARCHAR, VARCHAR, INTEGER) -> VARCHAR\n\n\nLINE 1: SELECT regexp_extract('ABC', 'abc', 'i')\n ^ | ERROR BinderException: Binder Error: No function matches the given name and argument types 'regexp_extract(STRING_LITERAL, STRING_LITERAL, BIGINT)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tregexp_extract(VARCHAR, VARCHAR) -> VARCHAR\n\tregexp_extract(VARCHAR, VARCHAR, INTEGER) -> VARCHAR\n\tregexp_extract(VARCHAR, VARCHAR, INTEGER, VARCHAR) -> VARCHAR\n\tregexp_extract(VARCHAR, VARCHAR, VARCHAR[]) -> VARCHAR\n\tregexp_extract(VARCHAR, VARCHAR, VARCHAR[], VARCHAR) -> VARCHAR\n\n\nLINE 1: SELECT regexp_extract('abc123', '([a-z]+)([0-9]+)', CAST(2 AS BIGINT...\n ^" + }, + { + "claim": "Options errors (verbatim): unrecognized 'q'; 'g' only valid for regexp_replace; NULL options is an ERROR not NULL; options must be a constant", + "query": "SELECT regexp_extract('a', 'a', 0, 'q')\nSELECT regexp_extract('aaa', 'a', 0, 'g')\nSELECT regexp_extract('ABC', 'abc', 0, NULL)\nSELECT regexp_extract('ABC', 'abc', 0, o) FROM (VALUES ('i')) to_(o)", + "observed": "ERROR InvalidInputException: Invalid Input Error: Unrecognized Regex option q | ERROR InvalidInputException: Invalid Input Error: Option 'g' (global replace) is only valid for regexp_replace | ERROR InvalidInputException: Invalid Input Error: Regex options field must not be NULL | ERROR InvalidInputException: Invalid Input Error: Regex options field must be a constant" + }, + { + "claim": "Option 'c' = case-sensitive (the default); empty options string = default; with conflicting flags the LAST wins ('ci' matches, 'ic' does not); 'l' = literal matching", + "query": "SELECT regexp_extract('ABC', 'abc', 0, 'c'), regexp_extract('ABC', 'abc', 0, ''), regexp_extract('ABC', 'abc', 0, 'ci'), regexp_extract('ABC', 'abc', 0, 'ic')\nSELECT regexp_extract('a.c', 'a.c', 0, 'l'), regexp_extract('axc', 'a.c', 0, 'l')", + "observed": "[('', '', 'ABC', '')] | [('a.c', '')]" + }, + { + "claim": "Dot does NOT match newline by default; option 's' makes it match; 'n' keeps it non-matching", + "query": "SELECT regexp_extract('a' || chr(10) || 'b', 'a.b'), regexp_extract('a' || chr(10) || 'b', 'a.b', 0, 's'), regexp_extract('a' || chr(10) || 'b', 'a.b', 0, 'n')", + "observed": "[('', 'a\\nb', '')]" + }, + { + "claim": "SURPRISE: options 'm'/'n'/'p' do NOT enable multiline ^/$ anchors ('' for all); inline (?m) DOES; inline (?s) and (?i) also work", + "query": "SELECT regexp_extract('a' || chr(10) || 'b', '^b', 0, 'm'), regexp_extract('a' || chr(10) || 'b', 'a$', 0, 'm'), regexp_extract('a' || chr(10) || 'b', '^b', 0, 'p'), regexp_extract('a' || chr(10) || 'b', '^b', 0, 'n')\nSELECT regexp_extract('a' || chr(10) || 'b', '(?m)^b'), regexp_extract('a' || chr(10) || 'b', '(?s)a.b'), regexp_extract('ABC', '(?i)abc')", + "observed": "[('', '', '', '')] | [('b', 'a\\nb', 'ABC')]" + }, + { + "claim": "UTF-8 aware: '.' matches one CODEPOINT — 'é', a full emoji, one Arabic-Indic digit; non-ASCII capture works", + "query": "SELECT regexp_extract('héllo', 'h.llo'), regexp_extract('héllo', 'h(é)llo', 1)\nSELECT regexp_extract('a😀b', 'a(.)b', 1), regexp_extract('a😀b', '.'), regexp_extract('٣٤', '.')", + "observed": "[('héllo', 'é')] | [('😀', 'a', '٣')]" + }, + { + "claim": "ASCII-only classes (RE2): [0-9] and \\d do NOT match Arabic-Indic '٣٤' (-> ''); \\w does not match 'é' so (\\w+) on 'héllo' captures only 'h'", + "query": "SELECT regexp_extract('٣٤', '[0-9]+'), regexp_extract('٣٤', '\\d+'), regexp_extract('٣٤', '[0-9]+') = ''\nSELECT regexp_extract('héllo', '(\\w+)', 1)", + "observed": "[('', '', True)] | [('h',)]" + }, + { + "claim": "Case-insensitive 'i' uses simple folding: 'i' does NOT match 'İ' (U+0130) in 'İstanbul'; '.' still matches 'İ' as one codepoint", + "query": "SELECT regexp_extract('İstanbul', 'i', 0, 'i'), regexp_extract('İstanbul', 'i', 0, 'i') = ''\nSELECT regexp_extract('İstanbul', '^.')", + "observed": "[('', True)] | [('İ',)]" + }, + { + "claim": "Invalid regex patterns: verbatim RE2 errors for '(' and '['", + "query": "SELECT regexp_extract('a', '(')\nSELECT regexp_extract('a', '[')", + "observed": "ERROR InvalidInputException: Invalid Input Error: missing ): ( | ERROR InvalidInputException: Invalid Input Error: missing ]: [" + }, + { + "claim": "Per-row over t('hello world','abc123def','',NULL,'héllo'): NULL row -> NULL, others follow scalar rules (note 'héllo' -> 'h' via ASCII \\w)", + "query": "SELECT s, regexp_extract(s, '(\\w+)', 1) FROM t\nSELECT s, regexp_extract(s, 'l+') FROM t", + "observed": "[('hello world', 'hello'), ('abc123def', 'abc123def'), ('', ''), (None, None), ('héllo', 'h')] | [('hello world', 'll'), ('abc123def', ''), ('', ''), (None, None), ('héllo', 'll')]" + }, + { + "claim": "Pattern MAY be a non-constant column (per-row compile; NULL pattern row -> NULL); group index must be a constant (verbatim runtime error), though constant-foldable exprs like 1+1 are fine", + "query": "SELECT regexp_extract('abc123', p) FROM (VALUES ('[0-9]+'), ('x'), (NULL)) tp(p)\nSELECT regexp_extract('abc123def', '([a-z]+)([0-9]+)', g) FROM (VALUES (1),(2)) tg(g)\nSELECT regexp_extract('abc123', '([a-z]+)([0-9]+)', 1+1)", + "observed": "[('123',), ('',), (None,)] | ERROR InvalidInputException: Invalid Input Error: Group specification field must be a constant! | [('123',)]" + }, + { + "claim": "RE2 named groups (?P...) allowed; integer indexing ignores the name; name-list maps positionally", + "query": "SELECT regexp_extract('abc123', '(?P[a-z]+)', 1)\nSELECT regexp_extract('abc123', '(?P[a-z]+)([0-9]+)', ['l','n'])", + "observed": "[('abc',)] | [({'l': 'abc', 'n': '123'},)]" + } + ], + "surprises": [ + "NULL group index returns '' (empty string), NOT NULL — unlike NULL subject/pattern which return NULL.", + "NULL options string is an InvalidInputException ('Regex options field must not be NULL'), not a NULL result.", + "Group index is hard-capped at 0..9 ('Group index must be between 0 and 9!'): group 10 errors even when the pattern HAS 10 capturing groups, and the error fires even when the subject wouldn't match; but any index in 0..9 beyond the actual group count silently returns ''.", + "No-match, non-participating groups, and over-count in-range groups all return '' (never NULL) — '' from regexp_extract is ambiguous between no-match and empty capture.", + "Options 'm'/'n'/'p' (newline-sensitive) do NOT enable multiline ^/$ anchors in 1.5.5; inline (?m) does. Option 's' does enable dot-matches-newline.", + "In the options string the LAST of conflicting flags wins: 'ci' is case-insensitive, 'ic' is case-sensitive.", + "Alternation is leftmost-FIRST, not leftmost-longest: 'a|aa' on 'aaa' extracts 'a'.", + "\\d, \\w, [0-9] are ASCII-only (RE2): no match on Arabic-Indic '٣٤'; (\\w+) on 'héllo' captures just 'h'. But '.' is codepoint-aware (matches a full emoji as one char).", + "Case-insensitive 'i' does not fold 'İ' (U+0130) to 'i'.", + "Name-list form: fewer names than groups is allowed (takes first N); more names, empty list, duplicate names, NULL names are bind errors; empty-string name yields an unnamed STRUCT field (typeof STRUCT(VARCHAR, VARCHAR)); NULL pattern is a bind error here (vs NULL result in scalar form).", + "Pattern may be a per-row column, but group index and options must be constants (runtime InvalidInputException otherwise); BIGINT group index is NOT implicitly cast to INTEGER (binder error).", + "DuckDB 1.5.5 does not support U&'...' literals ('Parser Error: unicode_to_utf8 NOT IMPLEMENTED') — use direct UTF-8 literals." + ], + "recommendation": "Engine scope: implement the scalar forms (2-arg, 3-arg INTEGER group, 4-arg options) returning VARCHAR; classify the name-list STRUCT forms (VARCHAR[] 3rd arg, +options) as clean out-of-scope non-scalar. Semantics: unanchored leftmost-first RE2 search; '' for no-match, non-participating groups, and any in-range (0..9) group index above the pattern's group count; NULL only for NULL subject or NULL pattern; NULL group index -> '' and NULL options -> error; constant group index required, range-checked 0..9 with verbatim 'Group index must be between 0 and 9!'; constant options required; options c/i/l/m/n/p/s with last-flag-wins conflicts, 'g' rejected — and m/n/p must NOT enable multiline anchors while inline (?m) does. Match RE2 exactly: ASCII-only \\d/\\w classes, codepoint-aware '.', simple case folding (no İ->i)." +} \ No newline at end of file diff --git a/docs/superpowers/specs/pins-waveB/matches-operators.json b/docs/superpowers/specs/pins-waveB/matches-operators.json new file mode 100644 index 0000000..a37cfdf --- /dev/null +++ b/docs/superpowers/specs/pins-waveB/matches-operators.json @@ -0,0 +1,178 @@ +{ + "area": "matches-operators", + "duckdb_version": "v1.5.5", + "setup": "CREATE TABLE t(id INT, s VARCHAR); INSERT INTO t VALUES (1,'hello'),(2,'hello world'),(3,''),(4,NULL),(5,'h\u00e9llo'),(6,'HELLO')", + "pins": [ + { + "claim": "regexp_matches(s, p) is a SEARCH (unanchored, contains-style): regexp_matches('hello','ell') is true; regexp_full_match('hello','ell') is false; the ~ operator agrees with regexp_full_match (FULL), not regexp_matches", + "query": "SELECT id, s, regexp_matches(s, 'ell'), regexp_full_match(s, 'ell'), s ~ 'ell', s !~ 'ell' FROM t ORDER BY id", + "observed": "[(1, 'hello', True, False, False, True), (2, 'hello world', True, False, False, True), (3, '', False, False, False, True), (4, None, None, None, None, None), (5, 'h\\xe9llo', False, False, False, True), (6, 'HELLO', False, False, False, True)]" + }, + { + "claim": "With pattern 'hello': regexp_matches true for 'hello' AND 'hello world' (search); regexp_full_match and ~ true ONLY for exactly 'hello'; case-sensitive by default ('HELLO' false); accent-sensitive ('h\u00e9llo' false)", + "query": "SELECT id, s, regexp_matches(s, 'hello'), regexp_full_match(s, 'hello'), s ~ 'hello', s !~ 'hello' FROM t ORDER BY id", + "observed": "[(1, 'hello', True, True, True, False), (2, 'hello world', True, False, False, True), (3, '', False, False, False, True), (4, None, None, None, None, None), (5, 'h\\xe9llo', False, False, False, True), (6, 'HELLO', False, False, False, True)]" + }, + { + "claim": "Empty pattern '': regexp_matches(s,'') is TRUE for every non-NULL string (including '' itself); regexp_full_match(s,'') and s ~ '' are true ONLY for the empty string", + "query": "SELECT id, s, regexp_matches(s, ''), regexp_full_match(s, ''), s ~ '' FROM t ORDER BY id", + "observed": "[(1, 'hello', True, False, False), (2, 'hello world', True, False, False), (3, '', True, True, True), (4, None, None, None, None), (5, 'h\\xe9llo', True, False, False), (6, 'HELLO', True, False, False)]" + }, + { + "claim": "The ~ operator is FULL match: on 'hello', ~ 'ell' is false (regexp_matches says true, regexp_full_match says false; ~ sides with full match); on exact 'ell' all three are true", + "query": "SELECT v ~ 'ell' AS op, regexp_matches(v, 'ell') AS srch, regexp_full_match(v, 'ell') AS fullm FROM (VALUES ('hello'), ('ell')) AS x(v)", + "observed": "[(False, True, False), (True, True, True)]" + }, + { + "claim": "!~ is exactly NOT(~): elementwise identical to NOT (s ~ p) on every row, and NULL input yields NULL (not true/false)", + "query": "SELECT id, s, s !~ 'ell', NOT (s ~ 'ell') FROM t ORDER BY id", + "observed": "[(1, 'hello', True, True), (2, 'hello world', True, True), (3, '', True, True), (4, None, None, None), (5, 'h\\xe9llo', True, True), (6, 'HELLO', True, True)]" + }, + { + "claim": "Three-valued logic on the operators: NULL ~ 'x', 'x' ~ NULL, NULL !~ 'x', 'x' !~ NULL are all NULL", + "query": "SELECT NULL ~ 'x', 'x' ~ NULL, NULL !~ 'x', 'x' !~ NULL", + "observed": "[(None, None, None, None)]" + }, + { + "claim": "NULL string or NULL pattern makes regexp_matches and regexp_full_match return NULL (no error)", + "query": "SELECT regexp_matches(NULL, 'x'), regexp_matches('x', NULL), regexp_full_match(NULL, 'x'), regexp_full_match('x', NULL)", + "observed": "[(None, None, None, None)]" + }, + { + "claim": "SIMILAR TO with NULL on either side is NULL; NOT SIMILAR TO likewise", + "query": "SELECT NULL SIMILAR TO 'x', 'x' SIMILAR TO NULL, NULL NOT SIMILAR TO 'x'", + "observed": "[(None, None, None)]" + }, + { + "claim": "Result type of regexp_matches, regexp_full_match, ~, and SIMILAR TO is BOOLEAN", + "query": "SELECT typeof(regexp_matches('hello','ell')), typeof(regexp_full_match('hello','ell')), typeof('a' ~ 'b'), typeof('a' SIMILAR TO 'b')", + "observed": "[('BOOLEAN', 'BOOLEAN', 'BOOLEAN', 'BOOLEAN')]" + }, + { + "claim": "DESCRIBE confirms BOOLEAN (nullable) for all three forms", + "query": "DESCRIBE SELECT regexp_matches('hello','ell') AS m, 'a' ~ 'b' AS o, 'a' SIMILAR TO 'b' AS st", + "observed": "[('m', 'BOOLEAN', 'YES', None, None, None), ('o', 'BOOLEAN', 'YES', None, None, None), ('st', 'BOOLEAN', 'YES', None, None, None)]" + }, + { + "claim": "SIMILAR TO does NOT translate SQL wildcards: 'hello' SIMILAR TO 'h%o' is FALSE ('%' is passed to RE2 as a literal percent, not .*); '%ell%' and bare 'ell' also false; only the raw-regex-equal pattern 'hello' matches", + "query": "SELECT id, s, s SIMILAR TO 'h%o', s SIMILAR TO 'ell', s SIMILAR TO '%ell%', s SIMILAR TO 'hello', s NOT SIMILAR TO 'h%o' FROM t ORDER BY id", + "observed": "[(1, 'hello', False, False, False, True, True), (2, 'hello world', False, False, False, False, True), (3, '', False, False, False, False, True), (4, None, None, None, None, None, None), (5, 'h\\xe9llo', False, False, False, False, True), (6, 'HELLO', False, False, False, False, True)]" + }, + { + "claim": "SIMILAR TO does not translate '_' either: 'hello' SIMILAR TO 'h_llo' is false for every row", + "query": "SELECT id, s, s SIMILAR TO 'h_llo' FROM t ORDER BY id", + "observed": "[(1, 'hello', False), (2, 'hello world', False), (3, '', False), (4, None, None), (5, 'h\\xe9llo', False), (6, 'HELLO', False)]" + }, + { + "claim": "SIMILAR TO treats the pattern as a raw RE2 regex, full-match anchored: 'h.llo' matches 'hello' AND 'h\u00e9llo' (regex dot), not 'hello world' (anchored); identical column of results to regexp_full_match and ~", + "query": "SELECT id, s, regexp_matches(s, 'h.llo'), regexp_full_match(s, 'h.llo'), s ~ 'h.llo', s SIMILAR TO 'h.llo' FROM t ORDER BY id", + "observed": "[(1, 'hello', True, True, True, True), (2, 'hello world', True, False, False, False), (3, '', False, False, False, False), (4, None, None, None, None, None), (5, 'h\\xe9llo', True, True, True, True), (6, 'HELLO', False, False, False, False)]" + }, + { + "claim": "A '%' in SIMILAR TO is a literal percent: 'h%o' SIMILAR TO 'h%o' is true", + "query": "SELECT 'h%o' SIMILAR TO 'h%o'", + "observed": "[(True,)]" + }, + { + "claim": "SIMILAR TO ... ESCAPE is NOT implemented: raises Not implemented Error: Custom escape in SIMILAR TO", + "query": "SELECT '100%' SIMILAR TO '100\\%' ESCAPE '\\'", + "observed": "ERROR NotImplementedException; str(e) verbatim: Not implemented Error: Custom escape in SIMILAR TO" + }, + { + "claim": "'' SIMILAR TO '' is true; 'x' SIMILAR TO '' is false (full-match semantics on empty pattern)", + "query": "SELECT '' SIMILAR TO '', 'x' SIMILAR TO ''", + "observed": "[(True, False)]" + }, + { + "claim": "regexp_matches third options arg 'i' = case-insensitive: matches 'HELLO' against 'hello'; regexp_full_match accepts the same options arg", + "query": "SELECT regexp_matches('HELLO', 'hello', 'i'), regexp_full_match('HELLO', 'hello', 'i')", + "observed": "[(True, True)]" + }, + { + "claim": "Option 'i' case-folds non-ASCII Latin-1: 'H\u00c9LLO' ~i~ 'h\u00e9llo' true in both directions", + "query": "SELECT regexp_matches('H\u00c9LLO', 'h\u00e9llo', 'i'), regexp_matches('h\u00e9llo', 'H\u00c9LLO', 'i')", + "observed": "[(True, True)]" + }, + { + "claim": "Option 'i' does NOT fold Turkish dotted capital I (U+0130): regexp_matches('\u0130stanbul','istanbul','i') is FALSE, and 'istanbul' vs '\u0130STANBUL' with 'i' is also FALSE (RE2 simple folding, no special Unicode mappings)", + "query": "SELECT regexp_matches('\u0130stanbul', 'istanbul', 'i'), regexp_matches('istanbul', '\u0130STANBUL', 'i')", + "observed": "[(False, False)]" + }, + { + "claim": "Option 'c' = case-sensitive (the default): 'Hello' vs 'hello' with 'c' is false, exact case true", + "query": "SELECT regexp_matches('Hello', 'hello', 'c'), regexp_matches('hello', 'hello', 'c')", + "observed": "[(False, True)]" + }, + { + "claim": "Unknown option letter is an execution error, verbatim: Invalid Input Error: Unrecognized Regex option z", + "query": "SELECT regexp_matches('HELLO', 'hello', 'z')", + "observed": "ERROR InvalidInputException; str(e) verbatim: Invalid Input Error: Unrecognized Regex option z" + }, + { + "claim": "NULL options argument is an ERROR, not NULL: Invalid Input Error: Regex options field must not be NULL", + "query": "SELECT regexp_matches('HELLO', 'hello', NULL)", + "observed": "ERROR InvalidInputException; str(e) verbatim: Invalid Input Error: Regex options field must not be NULL" + }, + { + "claim": "Binder error for regexp_matches(123, 'x') verbatim, listing the two overloads (VARCHAR,VARCHAR) and (VARCHAR,VARCHAR,VARCHAR)", + "query": "SELECT regexp_matches(123, 'x')", + "observed": "ERROR BinderException; str(e) verbatim: Binder Error: No function matches the given name and argument types 'regexp_matches(INTEGER_LITERAL, STRING_LITERAL)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tregexp_matches(VARCHAR, VARCHAR) -> BOOLEAN\n\tregexp_matches(VARCHAR, VARCHAR, VARCHAR) -> BOOLEAN\n\n\nLINE 1: SELECT regexp_matches(123, 'x')\n ^" + }, + { + "claim": "Binder error for s ~ 123 names regexp_full_match (proof the ~ operator is bound to regexp_full_match), verbatim", + "query": "SELECT s ~ 123 FROM t", + "observed": "ERROR BinderException; str(e) verbatim: Binder Error: No function matches the given name and argument types 'regexp_full_match(VARCHAR, INTEGER_LITERAL)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tregexp_full_match(VARCHAR, VARCHAR) -> BOOLEAN\n\tregexp_full_match(VARCHAR, VARCHAR, VARCHAR) -> BOOLEAN\n\n\nLINE 1: SELECT s ~ 123 FROM t\n ^" + }, + { + "claim": "Binder error for 123 SIMILAR TO '1%' ALSO names regexp_full_match (SIMILAR TO desugars to regexp_full_match too; no implicit INTEGER->VARCHAR cast)", + "query": "SELECT 123 SIMILAR TO '1%'", + "observed": "ERROR BinderException; str(e) verbatim: Binder Error: No function matches the given name and argument types 'regexp_full_match(INTEGER_LITERAL, STRING_LITERAL)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tregexp_full_match(VARCHAR, VARCHAR) -> BOOLEAN\n\tregexp_full_match(VARCHAR, VARCHAR, VARCHAR) -> BOOLEAN\n\n\nLINE 1: SELECT 123 SIMILAR TO '1%'\n ^" + }, + { + "claim": "Invalid regex '(' gives the same RE2 error through regexp_matches: Invalid Input Error: missing ): (", + "query": "SELECT regexp_matches('x', '(')", + "observed": "ERROR InvalidInputException; str(e) verbatim: Invalid Input Error: missing ): (" + }, + { + "claim": "Invalid regex '(' through the ~ operator: same verbatim error", + "query": "SELECT 'x' ~ '('", + "observed": "ERROR InvalidInputException; str(e) verbatim: Invalid Input Error: missing ): (" + }, + { + "claim": "Invalid regex '(' through SIMILAR TO: same verbatim error", + "query": "SELECT 'x' SIMILAR TO '('", + "observed": "ERROR InvalidInputException; str(e) verbatim: Invalid Input Error: missing ): (" + }, + { + "claim": "Invalid constant regex errors even when the row is filtered out (WHERE 1=0) => pattern compiled at bind/prepare time, not lazily per row", + "query": "SELECT 'x' ~ '(' WHERE 1=0", + "observed": "ERROR InvalidInputException; str(e) verbatim: Invalid Input Error: missing ): (" + }, + { + "claim": "Regex '.' matches ONE full Unicode codepoint, not a byte: 'a\\U0001F600b' (emoji) full-matches 'a.b' via regexp_full_match, regexp_matches and ~", + "query": "SELECT regexp_matches('a\ud83d\ude00b', 'a.b'), regexp_full_match('a\ud83d\ude00b', 'a.b'), 'a\ud83d\ude00b' ~ 'a.b'", + "observed": "[(True, True, True)]" + }, + { + "claim": "Arabic-Indic digits '\u0663\u0664': [0-9] false, \\d false (RE2 \\d is ASCII-only), \\p{Nd} true (Unicode property classes work)", + "query": "SELECT regexp_matches('\u0663\u0664', '[0-9]'), regexp_matches('\u0663\u0664', '\\d'), regexp_matches('\u0663\u0664', '\\p{Nd}')", + "observed": "[(False, False, True)]" + }, + { + "claim": "Explicit anchors work inside regexp_matches patterns: '^hello$' does not search-match 'hello world', '^hello' does; regexp_full_match('hello world','hello.*') is true", + "query": "SELECT regexp_matches('hello world', '^hello$'), regexp_matches('hello world', '^hello'), regexp_full_match('hello world', 'hello.*')", + "observed": "[(False, True, True)]" + } + ], + "surprises": [ + "The ~ operator is FULL match, not search: DuckDB binds s ~ p to regexp_full_match (binder error for s ~ 123 names regexp_full_match). This DIVERGES from PostgreSQL, where ~ is an unanchored search.", + "SIMILAR TO does NOT translate SQL wildcards % and _: the pattern goes to RE2 raw ('hello' SIMILAR TO 'h%o' is false, 'h%o' SIMILAR TO 'h%o' is true) and regex metachars like '.' are live. It is exactly regexp_full_match(s, pattern).", + "SIMILAR TO ... ESCAPE raises NotImplementedException 'Not implemented Error: Custom escape in SIMILAR TO'.", + "A NULL third options argument to regexp_matches is an execution ERROR ('Regex options field must not be NULL'), not a NULL result \u2014 unlike NULL string/pattern which propagate NULL.", + "Empty pattern asymmetry: regexp_matches(s,'') is true for every non-NULL string, while regexp_full_match(s,'')/s ~ '' are true only for ''.", + "Option 'i' folds \u00c9<->\u00e9 but NOT the Turkish dotted capital \u0130 (U+0130) \u2014 RE2 simple case folding only.", + "\\d is ASCII-only in RE2: Arabic-Indic '\u0663\u0664' does not match \\d or [0-9], but does match \\p{Nd}.", + "Invalid constant patterns error even under WHERE 1=0 \u2014 compiled at bind time." + ], + "recommendation": "Implement ~, !~, and SIMILAR TO all as regexp_full_match(s, raw_pattern) with RE2 semantics (no %/_ translation, full-string anchoring); implement regexp_matches as unanchored RE2 search. !~ and NOT SIMILAR TO are plain NOT with NULL propagation. NULL in string or pattern -> NULL; NULL in the options arg -> error. Support options 'i'/'c' (RE2 simple case folding, ASCII-only \\d, \\p{...} classes available, '.' = one codepoint). Reject SIMILAR TO ... ESCAPE as not-implemented. Reproduce binder errors mentioning regexp_full_match for the operator forms." +} \ No newline at end of file diff --git a/docs/superpowers/specs/pins-waveB/options-errors.json b/docs/superpowers/specs/pins-waveB/options-errors.json new file mode 100644 index 0000000..8323c01 --- /dev/null +++ b/docs/superpowers/specs/pins-waveB/options-errors.json @@ -0,0 +1,218 @@ +{ + "area": "options-errors", + "duckdb_version": "v1.5.5", + "pins": [ + { + "claim": "Accepted option letters are c i l m n p s (all regexp functions) plus g (regexp_replace only); any other character is rejected. Bogus 'q' errors identically in regexp_matches/regexp_extract/regexp_replace.", + "query": "SELECT regexp_matches('ABC', 'abc', 'q')\nSELECT regexp_extract('a', 'a', 0, 'q')\nSELECT regexp_replace('a', 'a', 'x', 'q')", + "observed": "ERROR InvalidInputException: Invalid Input Error: Unrecognized Regex option q\nERROR InvalidInputException: Invalid Input Error: Unrecognized Regex option q\nERROR InvalidInputException: Invalid Input Error: Unrecognized Regex option q" + }, + { + "claim": "'x' (extended/free-spacing) is NOT a valid option letter, and option letters are case-sensitive (uppercase 'I' rejected) — same 'Unrecognized Regex option' error shape for each.", + "query": "SELECT regexp_matches('ABC', 'abc', 'x')\nSELECT regexp_matches('ABC', 'abc', 'I')", + "observed": "ERROR InvalidInputException: Invalid Input Error: Unrecognized Regex option x\nERROR InvalidInputException: Invalid Input Error: Unrecognized Regex option I" + }, + { + "claim": "'c' = case-sensitive matching (the default): 'ABC' vs 'abc' with 'c' is false.", + "query": "SELECT regexp_matches('ABC', 'abc', 'c')", + "observed": "OK [(False,)]" + }, + { + "claim": "'i' = case-insensitive matching: 'ABC' vs 'abc' with 'i' is true; works in extract (returns 'ABC') and replace (replaces first 'A').", + "query": "SELECT regexp_matches('ABC', 'abc', 'i')\nSELECT regexp_extract('ABC', '(abc)', 1, 'i')\nSELECT regexp_replace('AaAa', 'a', 'x', 'i')", + "observed": "OK [(True,)]\nOK [('ABC',)]\nOK [('xaAa',)]" + }, + { + "claim": "'l' = literal matching: metacharacters lose meaning. 'a.c' with 'l' matches only the literal string 'a.c', not 'abc' (which the same pattern matches without 'l').", + "query": "SELECT regexp_matches('a.c', 'a.c', 'l'), regexp_matches('abc', 'a.c', 'l'), regexp_matches('abc', 'a.c')", + "observed": "OK [(True, False, True)]" + }, + { + "claim": "'l' composes with 'i' ('il' = literal case-insensitive: 'A.C' vs pattern 'a.c' is true), and under 'l' inline-flag syntax is literal text (pattern '(?i)abc' with 'l' matches the literal string '(?i)abc').", + "query": "SELECT regexp_matches('A.C', 'a.c', 'il')\nSELECT regexp_matches('(?i)abc', '(?i)abc', 'l')", + "observed": "OK [(True,)]\nOK [(True,)]" + }, + { + "claim": "'s' = dot matches newline (dotall): 'a'||chr(10)||'b' vs 'a.b' is false by default and true with 's'.", + "query": "SELECT regexp_matches('a' || chr(10) || 'b', 'a.b')\nSELECT regexp_matches('a' || chr(10) || 'b', 'a.b', 's')", + "observed": "OK [(False,)]\nOK [(True,)]" + }, + { + "claim": "'m', 'n', 'p' are all accepted and all mean newline-sensitive matching, which is already the default: dot still does not match chr(10) under any of them.", + "query": "SELECT regexp_matches('a' || chr(10) || 'b', 'a.b', 'm')\nSELECT regexp_matches('a' || chr(10) || 'b', 'a.b', 'n')\nSELECT regexp_matches('a' || chr(10) || 'b', 'a.b', 'p')", + "observed": "OK [(False,)]\nOK [(False,)]\nOK [(False,)]" + }, + { + "claim": "SURPRISE: 'm'/'n'/'p' do NOT turn ^/$ into per-line anchors. '^b$' (and even '^a$') against 'a'||chr(10)||'b' is false with 'm', 'n', and 'p'.", + "query": "SELECT regexp_matches('a' || chr(10) || 'b', '^b$', 'm')\nSELECT regexp_matches('a' || chr(10) || 'b', '^b$', 'n')\nSELECT regexp_matches('a' || chr(10) || 'b', '^b$', 'p')\nSELECT regexp_matches('a' || chr(10) || 'b', '^a$', 'm')", + "observed": "OK [(False,)]\nOK [(False,)]\nOK [(False,)]\nOK [(False,)]" + }, + { + "claim": "Multiline anchors come ONLY from the inline flag: '(?m)^b$' matches 'a'||chr(10)||'b' (true), including when options 'n' or 's' are also passed.", + "query": "SELECT regexp_matches('a' || chr(10) || 'b', '(?m)^b$')\nSELECT regexp_matches('a' || chr(10) || 'b', '(?m)^b$', 'n')\nSELECT regexp_matches('a' || chr(10) || 'b', '(?m)^b$', 's')", + "observed": "OK [(True,)]\nOK [(True,)]\nOK [(True,)]" + }, + { + "claim": "'n' is NOT RE2 never_nl: a pattern containing a literal newline, the escape \\n, or [\\s\\S] still matches across chr(10) with 'n'.", + "query": "SELECT regexp_matches('a' || chr(10) || 'b', 'a' || chr(10) || 'b', 'n')\nSELECT regexp_matches('a' || chr(10) || 'b', 'a\\nb', 'n')\nSELECT regexp_matches('a' || chr(10) || 'b', 'a[\\s\\S]b', 'n')\nSELECT regexp_matches('a' || chr(10) || 'b', 'a[\\s\\S]b')", + "observed": "OK [(True,)]\nOK [(True,)]\nOK [(True,)]\nOK [(True,)]" + }, + { + "claim": "'g' is only valid in regexp_replace; in regexp_matches, regexp_extract, and regexp_extract_all it raises the same verbatim error.", + "query": "SELECT regexp_matches('ABC', 'abc', 'g')\nSELECT regexp_extract('ABC', '(abc)', 1, 'g')\nSELECT regexp_extract_all('aAbA', 'a', 0, 'g')", + "observed": "ERROR InvalidInputException: Invalid Input Error: Option 'g' (global replace) is only valid for regexp_replace\nERROR InvalidInputException: Invalid Input Error: Option 'g' (global replace) is only valid for regexp_replace\nERROR InvalidInputException: Invalid Input Error: Option 'g' (global replace) is only valid for regexp_replace" + }, + { + "claim": "regexp_replace replaces only the FIRST match by default; 'g' makes it global. Combined 'gi'/'ig' both give global case-insensitive.", + "query": "SELECT regexp_replace('AaAa', 'a', 'x')\nSELECT regexp_replace('AaAa', 'a', 'x', 'g')\nSELECT regexp_replace('AaAa', 'a', 'x', 'gi')\nSELECT regexp_replace('AaAa', 'a', 'x', 'ig')", + "observed": "OK [('AxAa',)]\nOK [('AxAx',)]\nOK [('xxxx',)]\nOK [('xxxx',)]" + }, + { + "claim": "Empty options string '' is identical to omitting the argument: no error, default (case-sensitive, non-dotall) behavior.", + "query": "SELECT regexp_matches('ABC', 'abc', '')\nSELECT regexp_matches('a' || chr(10) || 'b', 'a.b', '')\nSELECT regexp_extract('ABC', '(abc)', 1, '')\nSELECT regexp_replace('AaAa', 'a', 'x', '')", + "observed": "OK [(False,)]\nOK [(False,)]\nOK [('',)]\nOK [('AxAa',)]" + }, + { + "claim": "Whitespace inside the options string is silently ignored (space, tab, newline all fine, including an options string that is only a space); any other separator such as ',' is an unrecognized option.", + "query": "SELECT regexp_matches('ABC', 'abc', 'i s')\nSELECT regexp_matches('ABC', 'abc', 'i' || chr(9) || 's')\nSELECT regexp_matches('ABC', 'abc', 'i' || chr(10) || 's')\nSELECT regexp_matches('ABC', 'abc', ' ')\nSELECT regexp_matches('ABC', 'abc', 'i,s')", + "observed": "OK [(True,)]\nOK [(True,)]\nOK [(True,)]\nOK [(False,)]\nERROR InvalidInputException: Invalid Input Error: Unrecognized Regex option ," + }, + { + "claim": "SURPRISE: conflicting options do NOT error — the options string is processed left-to-right and the LAST occurrence wins: 'ci' is case-insensitive (true), 'ic' is case-sensitive (false), 'ici' insensitive again.", + "query": "SELECT regexp_matches('ABC', 'abc', 'ci')\nSELECT regexp_matches('ABC', 'abc', 'ic')\nSELECT regexp_matches('ABC', 'abc', 'ici')", + "observed": "OK [(True,)]\nOK [(False,)]\nOK [(True,)]" + }, + { + "claim": "Same last-wins rule for newline options: 'sn' behaves like 'n' (dot does not cross newline, false), 'ns' behaves like 's' (true).", + "query": "SELECT regexp_matches('a' || chr(10) || 'b', 'a.b', 'sn')\nSELECT regexp_matches('a' || chr(10) || 'b', 'a.b', 'ns')", + "observed": "OK [(False,)]\nOK [(True,)]" + }, + { + "claim": "Duplicate option letters are harmless: 'ii' and 'gg' work.", + "query": "SELECT regexp_matches('ABC', 'abc', 'ii')\nSELECT regexp_replace('aaa', 'a', 'x', 'gg')", + "observed": "OK [(True,)]\nOK [('xxx',)]" + }, + { + "claim": "NULL options is an ERROR (not a NULL result): 'Regex options field must not be NULL'.", + "query": "SELECT regexp_matches('ABC', 'abc', NULL)", + "observed": "ERROR InvalidInputException: Invalid Input Error: Regex options field must not be NULL" + }, + { + "claim": "NULL pattern or NULL subject returns NULL (no error).", + "query": "SELECT regexp_matches('ABC', NULL)\nSELECT regexp_matches(NULL, 'abc')", + "observed": "OK [(None,)]\nOK [(None,)]" + }, + { + "claim": "The options argument must be a CONSTANT: sourcing it from a column errors 'Regex options field must be a constant' even when the value is a valid letter and even under WHERE 1=0 (bind-time check).", + "query": "SELECT regexp_matches('ABC', 'abc', o) FROM (SELECT 'i' AS o) t\nSELECT regexp_matches('a', 'a', o) FROM (SELECT 'q' AS o) WHERE 1=0", + "observed": "ERROR InvalidInputException: Invalid Input Error: Regex options field must be a constant\nERROR InvalidInputException: Invalid Input Error: Regex options field must be a constant" + }, + { + "claim": "Empty pattern matches everything including the empty string; regexp_matches is a contains-style search, not a full match.", + "query": "SELECT regexp_matches('abc', ''), regexp_matches('', '')\nSELECT regexp_matches('xabcx', 'abc')", + "observed": "OK [(True, True)]\nOK [(True,)]" + }, + { + "claim": "Inline flags (?i), (?s), (?m) are all accepted by RE2 inside DuckDB patterns.", + "query": "SELECT regexp_matches('ABC', '(?i)abc')\nSELECT regexp_matches('a' || chr(10) || 'b', '(?s)a.b')\nSELECT regexp_matches('a' || chr(10) || 'b', '(?m)^b$')", + "observed": "OK [(True,)]\nOK [(True,)]\nOK [(True,)]" + }, + { + "claim": "(?U) (ungreedy) is accepted: '(?U)a+' extracts the shortest match 'a' from 'aaa'.", + "query": "SELECT regexp_extract('aaa', '(?U)a+')", + "observed": "OK [('a',)]" + }, + { + "claim": "(?x) (free-spacing) is NOT supported: 'invalid perl operator: (?x'.", + "query": "SELECT regexp_matches('abc', '(?x)a b c')", + "observed": "ERROR InvalidInputException: Invalid Input Error: invalid perl operator: (?x" + }, + { + "claim": "Inline flags compose with and OVERRIDE the options argument: '(?i)abc' with options 'c' still matches 'ABC' (true); '(?-i)abc' with options 'i' does not (false); '(?s)a.b' with options 'n' still crosses the newline (true).", + "query": "SELECT regexp_matches('ABC', '(?i)abc', 'c')\nSELECT regexp_matches('ABC', '(?-i)abc', 'i')\nSELECT regexp_matches('a' || chr(10) || 'b', '(?s)a.b', 'n')\nSELECT regexp_replace('AaAa', '(?i)a', 'x', 'g')", + "observed": "OK [(True,)]\nOK [(False,)]\nOK [(True,)]\nOK [('xxxx',)]" + }, + { + "claim": "Pattern '[' (unclosed class): verbatim error 'Invalid Input Error: missing ]: ['. All pattern-compile and option errors are duckdb.InvalidInputException whose message is 'Invalid Input Error: ' + RE2's own error text.", + "query": "SELECT regexp_matches('aab', '[')", + "observed": "ERROR InvalidInputException: Invalid Input Error: missing ]: [" + }, + { + "claim": "Pattern '(' (unclosed group): verbatim error 'Invalid Input Error: missing ): ('.", + "query": "SELECT regexp_matches('aab', '(')", + "observed": "ERROR InvalidInputException: Invalid Input Error: missing ): (" + }, + { + "claim": "Pattern 'a{2,1}' (maxcase_sensitive, i->case_insensitive, l->literal, s->dot_nl(true), m/n/p->dot_nl(false), g->global (regexp_replace only, else error \"Option 'g' (global replace) is only valid for regexp_replace\"); any other char -> \"Unrecognized Regex option \". No conflict detection, last write wins. Options arg must be constant and non-NULL (distinct bind-time error texts). Do NOT map m/n/p to multiline anchors or never_nl. Compile constant patterns at bind (errors surface in EXPLAIN/PREPARE/WHERE false), non-constant patterns per-row at execution, both with error text 'Invalid Input Error: ' + RE2 message verbatim, as InvalidInputException." +} \ No newline at end of file diff --git a/docs/superpowers/specs/pins-waveB/re2-vs-rust-regex.json b/docs/superpowers/specs/pins-waveB/re2-vs-rust-regex.json new file mode 100644 index 0000000..411312f --- /dev/null +++ b/docs/superpowers/specs/pins-waveB/re2-vs-rust-regex.json @@ -0,0 +1,131 @@ +{ + "area": "re2-vs-rust-regex", + "regex_crate_version": "1.13.1", + "duckdb_version": "1.5.5", + "method": "98-entry battery run on identical (pattern, input) pairs: DuckDB 1.5.5 regexp_matches / regexp_extract(0..3) / regexp_replace (first + 'g') vs rust regex 1.13.1 is_match / find / captures / replace / replace_all with marker '<>'. Probes additionally ran rust with rewritten patterns or RegexBuilder settings against the original DuckDB construct to verify which divergences a bind-time rewrite/config eliminates. Runners + raw outputs: scratchpad/run_duckdb.py, scratchpad/re2diff/, scratchpad/{battery,duckdb_out,rust_out}.json, scratchpad/compare.py.", + "agreements": [ + "(?i) case folding identical WITH NO CONFIG: both do Unicode simple folding. (?i)k matches K and KELVIN SIGN U+212A (and (?i)U+212A matches k); (?i)\u00df matches \u1e9e but NOT 'SS'; (?i)ss does not match \u00df; (?i)i does not match \u0130; (?i)\u03c3 matches final sigma \u03c2; (?i)stra\u00dfe does not match STRASSE. No full case folding on either side.", + "\\p{L}, \\p{Greek}+, \\pN, \\p{Lu}+, \\P{L} all identical — Unicode property classes are Unicode-aware in BOTH engines (\\pN matches U+0663). The unicode gap is ONLY in the Perl classes \\d \\w \\s \\b.", + "POSIX classes ASCII in both: [[:alpha:]]+ on 'a\u00e9b' finds 'a' in both.", + "Leftmost-first (NOT leftmost-longest) alternation in both: 'a|ab' on 'ab' finds 'a', replace_all '<>b'; 'samwise|sam' finds 'samwise'; 'sam|samwise' finds 'sam'. DuckDB does not enable RE2 longest_match.", + "Greedy/lazy identical: 'a.*b' on 'aXbXb' -> 'aXbXb'; 'a.*?b' -> 'aXb'. (?U) swap-greed supported and identical. (?s) supported; '.' excludes \\n by default in both; '.' consumes a full astral codepoint (U+1F600) in both (rune/char, never bytes... on valid UTF-8).", + "Empty-match iteration identical: replace_all 'a*' on 'bab' -> '<>b<>b<>' both; empty pattern '' on 'abc' -> '<>a<>b<>c<>' both (same skip-after-empty-match rule); '(|a)' on 'ab' -> '<>a<>b<>' both.", + "(a*)* on 'aaaa': matches 'aaaa', group1 'aaaa', both; both engines are linear-time automata, no catastrophic backtracking risk on either side.", + "Character-class edge cases identical (surprise: BOTH accept POSIX-style ']' first): '[]a]' matches 'a' in both (no error!); '[^]]' matches 'a'; '[a-]' matches '-'; '[\\]a]' matches ']'.", + "Inline flags identical including mid-pattern: '(?i)abc', 'a(?i)bc' on 'aBC', '(?i:x)y' on 'Xy', '(?i)a(?-i)b' on 'Ab' — all same.", + "(?P...) named groups work in both (captures identical).", + "Anchors: \\A and \\z identical; 'abc$' does NOT match 'abc\\n' in either (no PCRE before-trailing-newline rule); (?m)^ identical. \\Z is a compile ERROR in both (duckdb: 'Invalid Input Error: invalid escape sequence: \\\\Z'; rust: 'unrecognized escape sequence').", + "Both reject backrefs and all lookaround, different texts, same outcome: '(a)\\1' duckdb 'Invalid Input Error: invalid escape sequence: \\\\1' / rust 'backreferences are not supported'; '(?=a)b' and '(?!a)b' duckdb 'invalid perl operator: (?=' resp '(?!' / rust 'look-around, including look-ahead and look-behind, is not supported'; '(?<=a)b' duckdb 'invalid perl operator: (?<' / rust same look-around error.", + "'a{2,1}' rejected by both (duckdb 'invalid repetition size: {2,1}', rust 'invalid repetition count range'); 'a{1000}' compiles and matches in both (1000 is exactly RE2's limit).", + "Hex escapes '\\x41' and '\\x{263A}' identical.", + "'\\bh\u00e9llo\\b' on exactly 'h\u00e9llo' agrees (string start/end are boundaries in both) — but interior \\b positions around non-ASCII diverge, see divergences.", + "Word-boundary probe: rust '(?-u:\\b)' reproduced DuckDB '\\b' replace_all on 'h\u00e9llo w\u00f6rld' byte-for-byte ('<>h<>\u00e9<>llo<> <>w<>\u00f6<>rld<>')." + ], + "divergences": [ + { + "construct": "\\d / \\D (Perl digit class, Unicode-ness)", + "pattern": "\\d and \\d+ and \\D", + "input": "'\u0663' (U+0663), 'a\u06635b'", + "duckdb": "\\d is ASCII [0-9]: no match on U+0663; '\\d+' on 'a\u06635b' finds '5'; \\D matches U+0663", + "rust_regex": "\\d is Unicode \\p{Nd}: matches U+0663; '\\d+' finds '\u06635'; \\D does not match U+0663", + "guardable_at_bind": "yes, ELIMINATED by rewrite (verified identical): \\d -> (?-u:\\d) (== [0-9]), \\D -> [^0-9] (unicode negated class; NOT (?-u:\\D)). Detect via regex-syntax AST ClassPerl{kind:Digit}, including occurrences inside bracket classes ([\\d] and [^\\d] diverge identically)." + }, + { + "construct": "\\w / \\W (Perl word class, Unicode-ness)", + "pattern": "\\w, \\w+, \\W", + "input": "'\u00e9', '\u00df', '\u6f22', 'h\u00e9llo', 'w\u00f6rld'", + "duckdb": "\\w is ASCII [0-9A-Za-z_]: no match on \u00e9/\u00df/\u6f22; '\\w+' on 'h\u00e9llo' finds 'h' (replace_all '<>\u00e9<>'); \\W matches \u00e9", + "rust_regex": "\\w is Unicode: matches all of them; '\\w+' finds 'h\u00e9llo'; \\W does not match \u00e9", + "guardable_at_bind": "yes, ELIMINATED by rewrite (verified identical): \\w -> [0-9A-Za-z_] (or (?-u:\\w)), \\W -> [^0-9A-Za-z_]. WARNING: (?-u:\\W) itself does NOT compile for str regexes ('error: pattern can match invalid UTF-8') — negated forms must become unicode-mode negated classes." + }, + { + "construct": "\\s / \\S (Perl space class: Unicode-ness AND vertical-tab membership)", + "pattern": "\\s, \\S", + "input": "NBSP U+00A0, VT U+000B", + "duckdb": "RE2 \\s is exactly [\\t\\n\\f\\r ]: matches NEITHER NBSP nor U+000B; \\S matches both", + "rust_regex": "\\s is Unicode White_Space: matches BOTH NBSP and U+000B; \\S matches neither", + "guardable_at_bind": "yes, ELIMINATED by rewrite (verified identical): \\s -> [\\t\\n\\f\\r ], \\S -> [^\\t\\n\\f\\r ]. NOTE: (?-u:\\s) is the WRONG rewrite — rust ASCII \\s is [\\t\\n\\v\\f\\r ] which still includes VT that RE2 excludes; spell the class out." + }, + { + "construct": "\\b interior word boundaries around non-ASCII letters", + "pattern": "\\b (replace_all marker)", + "input": "'h\u00e9llo w\u00f6rld'", + "duckdb": "ASCII boundaries — \u00e9 is a non-word char, so boundaries on both sides of it: '<>h<>\u00e9<>llo<> <>w<>\u00f6<>rld<>'", + "rust_regex": "Unicode boundaries: '<>h\u00e9llo<> <>w\u00f6rld<>'", + "guardable_at_bind": "yes, ELIMINATED by rewrite (verified byte-for-byte identical): \\b -> (?-u:\\b). ASCII \\b cannot split UTF-8 so str regexes allow it." + }, + { + "construct": "\\B non-word-boundary on non-ASCII text (DuckDB itself is broken here)", + "pattern": "\\B", + "input": "'h\u00e9llo'", + "duckdb": "is_match true, but regexp_replace(..., '<>'[, 'g']) throws RUNTIME 'Invalid Input Error: Invalid unicode (byte sequence mismatch) detected in value construction' — RE2's ASCII \\B matches at byte offsets INSIDE the 2-byte \u00e9 and DuckDB builds invalid UTF-8", + "rust_regex": "Unicode \\B: replace_all 'h<>\u00e9<>l<>l<>o'; the ASCII rewrite (?-u:\\B) compiles but only matches at char boundaries: 'h\u00e9l<>l<>o' — neither reproduces DuckDB's byte-level positions (which DuckDB can't serve anyway)", + "guardable_at_bind": "yes — REJECT patterns containing \\B at bind (regex-syntax AST WordBoundary::AsciiNegate after (?-u) rewrite / Assertion NotWordBoundary). No rewrite can match DuckDB because DuckDB's own behavior is a data-dependent runtime error on non-ASCII input; on guaranteed-ASCII columns (?-u:\\B) agrees, but rejecting is the safe default." + }, + { + "construct": "(?...) angle-bracket named group", + "pattern": "(?a)", + "input": "'a'", + "duckdb": "compile ERROR 'Invalid Input Error: invalid perl operator: (?<' (DuckDB's bundled RE2 only accepts (?P...))", + "rust_regex": "accepted, matches 'a' with named group", + "guardable_at_bind": "yes — reject (?...) so the engine is not a superset of DuckDB (pattern DuckDB rejects must not silently serve). Detect: regex-syntax ast Group CaptureName with starts_with_p == false." + }, + { + "construct": "duplicate capture group names", + "pattern": "(?Pa)(?Pb)", + "input": "'ab'", + "duckdb": "ACCEPTED (modern RE2 allows duplicate names); positional captures work: group0 'ab', group1 'a', group2 'b'", + "rust_regex": "compile ERROR 'regex parse error: ... error: duplicate capture group name'", + "guardable_at_bind": "yes — detect duplicate names while walking the AST capture groups and reject as unsupported (or auto-rename n -> n_1, n_2 and use positional capture indices, since DuckDB group semantics here are positional anyway)." + }, + { + "construct": "counted repetition above RE2's 1000 limit", + "pattern": "a{1001}, a{100000}", + "input": "any", + "duckdb": "compile ERROR 'Invalid Input Error: invalid repetition size: {1001}' (limit is exactly 1000: a{1000} works)", + "rust_regex": "compiles and matches fine (only bounded by size_limit)", + "guardable_at_bind": "yes — reject any {n}/{n,}/{n,m} with a bound > 1000 (AST Repetition Range). Keeps the engine from accepting patterns the DuckDB oracle rejects." + }, + { + "construct": "possessive-looking / stacked quantifiers", + "pattern": "a*+", + "input": "'aa'", + "duckdb": "compile ERROR 'Invalid Input Error: bad repetition operator: *+'", + "rust_regex": "accepted — parsed as nested repetition (a*)+, matches 'aa'", + "guardable_at_bind": "yes — reject a repetition whose operand is itself a bare repetition (AST Repetition directly wrapping Repetition without an explicit group)." + }, + { + "construct": "octal escapes", + "pattern": "\\101", + "input": "'A'", + "duckdb": "matches 'A' (RE2 supports octal escapes)", + "rust_regex": "default compile ERROR 'regex parse error: ... error: backreferences are not supported' (\\1 read as backref)", + "guardable_at_bind": "yes, ELIMINATED by config (verified identical): regex::RegexBuilder::octal(true). Also improves error parity: with octal(false) a real backref '(a)\\1' reports 'backreferences are not supported' matching intent anyway, so octal(true) costs nothing." + }, + { + "construct": "\\Q...\\E literal quoting", + "pattern": "\\Qa.b\\E", + "input": "'a.b'", + "duckdb": "supported: matches literal 'a.b' (the '.' is literal)", + "rust_regex": "compile ERROR 'regex parse error: ... error: unrecognized escape sequence'", + "guardable_at_bind": "yes — lexical pre-pass: either reject \\Q, or expand \\Q...\\E by escaping the enclosed span (regex::escape) before handing to the crate; expansion is exact and trivial since \\Q..\\E does not nest." + }, + { + "construct": "\\uXXXX / \\UXXXXXXXX codepoint escapes", + "pattern": "\\u0041", + "input": "'A'", + "duckdb": "compile ERROR 'Invalid Input Error: invalid escape sequence: \\\\u' (RE2 only has \\x{...} and \\xNN)", + "rust_regex": "accepted, matches 'A'", + "guardable_at_bind": "yes — reject \\u and \\U escapes (lexical scan for backslash-u/backslash-U outside classes and inside; RE2-portable spelling \\x{...} works in both)." + }, + { + "construct": "replacement-string syntax (API-level, not a pattern construct)", + "pattern": "(a)(b) replacing in 'ab'", + "input": "replacement '\\2\\1' vs '$2$1'", + "duckdb": "RE2 rewrite syntax: '\\2\\1' -> 'ba'; '$2$1' is LITERAL '$2$1'", + "rust_regex": "rust syntax: '$2$1' -> 'ba'; '\\2\\1' is LITERAL '\\2\\1'", + "guardable_at_bind": "yes — translate the replacement template at bind: \\N -> ${N}, \\\\ -> \\, escape every '$' as '$$', and reject \\N with N>9 semantics differences (RE2 rewrite only supports \\0-\\9). Never pass user replacement strings through untranslated." + } + ], + "recommendation": "VERDICT: regex 1.13.1 can serve DuckDB(RE2) semantics exactly, but only with a bind-time translation layer; raw pattern pass-through is wrong on any pattern containing a Perl class. CONFIG (eliminates divergences, no guards needed): (1) regex::RegexBuilder::octal(true) — fixes \\101-style octal escapes, verified identical; (2) do NOT set unicode(false): default Unicode mode is what makes (?i) folding (incl. KELVIN U+212A), \\p classes, and '.' agree — unicode(false) was measured to BREAK (?i)k vs U+212A parity. REWRITE at bind (AST walk with regex-syntax; each verified identical incl. inside bracket classes): \\d->(?-u:\\d), \\D->[^0-9], \\w->[0-9A-Za-z_], \\W->[^0-9A-Za-z_] ((?-u:\\W) does not compile for str), \\s->[\\t\\n\\f\\r ] (NOT (?-u:\\s): rust ASCII \\s wrongly includes VT U+000B), \\S->[^\\t\\n\\f\\r ], \\b->(?-u:\\b); optionally expand \\Q...\\E via regex::escape. REJECT at bind (engine must not accept what DuckDB rejects, and vice versa): \\B (DuckDB itself throws a runtime invalid-UTF8 error on non-ASCII input — unservable), (?...) angle-form named groups (DuckDB rejects), duplicate group names (rust rejects; or auto-rename + positional captures), repetition bounds > 1000 (DuckDB rejects; a{1000} is the max), stacked quantifiers like a*+ (DuckDB rejects, rust silently reinterprets as (a*)+ — the one construct where pass-through would serve WRONG ANSWERS instead of an error), and \\u/\\U escapes (DuckDB rejects). REPLACEMENT templates must be translated: RE2 \\N rewrite -> rust ${N}, '$' -> '$$'. Everything else — leftmost-first alternation, greedy/lazy, (?U)/(?s)/(?m)/inline (?i) placement, empty-match iteration and replace_all adjacency-skip, \\A/\\z/$ semantics, char-class edge parsing ('[]a]' legal in both), \\p classes, POSIX classes, \\x escapes, backref/lookaround rejection, a{2,1} rejection, \\Z rejection — agreed byte-for-byte with zero configuration." +} diff --git a/docs/superpowers/specs/pins-waveB/replace.json b/docs/superpowers/specs/pins-waveB/replace.json new file mode 100644 index 0000000..d247532 --- /dev/null +++ b/docs/superpowers/specs/pins-waveB/replace.json @@ -0,0 +1,286 @@ +{ + "area": "replace", + "duckdb_version": "v1.5.5 (python pkg 1.5.5)", + "pins": [ + { + "claim": "Default (no options) replaces FIRST match only: 'aaa' with a->X gives 'Xaa'", + "query": "SELECT regexp_replace('aaa', 'a', 'X')", + "observed": "[('Xaa',)]" + }, + { + "claim": "Default first-match-only over table t rows (hello world, aaa, '', NULL, a1b2c3) with a->X", + "query": "SELECT s, regexp_replace(s, 'a', 'X') FROM t", + "observed": "[('hello world', 'hello world'), ('aaa', 'Xaa'), ('', ''), (None, None), ('a1b2c3', 'X1b2c3')]" + }, + { + "claim": "Return type of regexp_replace is VARCHAR", + "query": "SELECT typeof(regexp_replace('aaa', 'a', 'X'))", + "observed": "[('VARCHAR',)]" + }, + { + "claim": "'g' option (4th arg) replaces ALL matches: 'aaa' -> 'XXX'", + "query": "SELECT regexp_replace('aaa', 'a', 'X', 'g')", + "observed": "[('XXX',)]" + }, + { + "claim": "Options combine: 'gi' on 'AAA' with pattern 'a' -> 'XXX'", + "query": "SELECT regexp_replace('AAA', 'a', 'X', 'gi')", + "observed": "[('XXX',)]" + }, + { + "claim": "Option order irrelevant: 'ig' same as 'gi'", + "query": "SELECT regexp_replace('AAA', 'a', 'X', 'ig')", + "observed": "[('XXX',)]" + }, + { + "claim": "Empty options string '' behaves like no options (first match only)", + "query": "SELECT regexp_replace('aaa', 'a', 'X', '')", + "observed": "[('Xaa',)]" + }, + { + "claim": "NULL options argument -> NULL result (not treated as no-options)", + "query": "SELECT regexp_replace('aaa', 'a', 'X', NULL)", + "observed": "[(None,)]" + }, + { + "claim": "Global replace over table t with a->X and 'g'", + "query": "SELECT s, regexp_replace(s, 'a', 'X', 'g') FROM t", + "observed": "[('hello world', 'hello world'), ('aaa', 'XXX'), ('', ''), (None, None), ('a1b2c3', 'X1b2c3')]" + }, + { + "claim": "Backslash style backref \\1 in replacement works (SQL literal '[\\1]')", + "query": "SELECT regexp_replace('hello world', '(hello)', '[\\1]')", + "observed": "[('[hello] world',)]" + }, + { + "claim": "Dollar style $1 is LITERAL text in the replacement (no $-backrefs): produces '[$1] world'", + "query": "SELECT regexp_replace('hello world', '(hello)', '[$1]')", + "observed": "[('[$1] world',)]" + }, + { + "claim": "\\0 in replacement = the WHOLE match", + "query": "SELECT regexp_replace('hello world', 'l+', '[\\0]')", + "observed": "[('he[ll]o world',)]" + }, + { + "claim": "$0 in replacement is LITERAL '$0', not the match", + "query": "SELECT regexp_replace('hello world', 'l+', '[$0]')", + "observed": "[('he[$0]o world',)]" + }, + { + "claim": "& in replacement is LITERAL '&', not the match (no sed-style ampersand)", + "query": "SELECT regexp_replace('hello world', 'l+', '[&]')", + "observed": "[('he[&]o world',)]" + }, + { + "claim": "Two backrefs swap: (h)(e) -> \\2\\1 on 'hello'", + "query": "SELECT regexp_replace('hello', '(h)(e)', '\\2\\1')", + "observed": "[('ehllo',)]" + }, + { + "claim": "Out-of-range backref \\2 (pattern has 1 group): replacement is SILENTLY SKIPPED entirely — original string returned, no error", + "query": "SELECT regexp_replace('hello', '(h)', '\\2')", + "observed": "[('hello',)]" + }, + { + "claim": "Disambiguation: out-of-range backref '[\\2]' with '(h)' — result is unchanged 'hello' (silent no-op), NOT '[]ello' (empty-substitution)", + "query": "SELECT regexp_replace('hello', '(h)', '[\\2]')", + "observed": "[('hello',)]" + }, + { + "claim": "Out-of-range backref with 'g': still silent no-op, original returned", + "query": "SELECT regexp_replace('aaa', '(a)', '\\2', 'g')", + "observed": "[('aaa',)]" + }, + { + "claim": "Literal backslash in replacement: SQL '\\\\' (two backslash chars) as rewrite", + "query": "SELECT regexp_replace('hello', 'h', '\\\\')", + "observed": "[('\\\\ello',)]" + }, + { + "claim": "Single trailing backslash alone as entire replacement: invalid rewrite -> silent no-op, original returned", + "query": "SELECT regexp_replace('hello', 'h', '\\')", + "observed": "[('hello',)]" + }, + { + "claim": "Backslash before non-digit in replacement ('\\x'): INVALID rewrite -> silent no-op, original returned (not literal 'x', not error)", + "query": "SELECT regexp_replace('hello', 'h', '\\x')", + "observed": "[('hello',)]" + }, + { + "claim": "DIVERGENCE with 'g': bad escape '\\x' rewrite under 'g' does NOT no-op — the match is consumed and the (failed) rewrite emits nothing: 'hello' -> 'ello'", + "query": "SELECT regexp_replace('hello', 'h', '\\x', 'g')", + "observed": "[('ello',)]" + }, + { + "claim": "Partial emission under 'g': rewrite 'A\\xB' emits text up to the bad escape then stops ('A'), match consumed, trailing 'B' dropped", + "query": "SELECT regexp_replace('hello', 'h', 'A\\xB', 'g')", + "observed": "[('Aello',)]" + }, + { + "claim": "Same partial emission WITHOUT 'g'? rewrite 'A\\xB' non-global", + "query": "SELECT regexp_replace('hello', 'h', 'A\\xB')", + "observed": "[('hello',)]" + }, + { + "claim": "Bad escape rewrite '\\x' with 'g' on 'aaa': every match deleted -> ''", + "query": "SELECT regexp_replace('aaa', 'a', '\\x', 'g')", + "observed": "[('',)]" + }, + { + "claim": "Trailing lone backslash rewrite 'Q\\' with 'g': emits 'Q' then stops at bad trailing backslash", + "query": "SELECT regexp_replace('hello', 'h', 'Q\\', 'g')", + "observed": "[('Qello',)]" + }, + { + "claim": "Out-of-range backref mixed with literal under 'g': 'A\\2B' with '(a)' -> pre-check no-op (contrast with bad-escape partial emission)", + "query": "SELECT regexp_replace('aaa', '(a)', 'A\\2B', 'g')", + "observed": "[('aaa',)]" + }, + { + "claim": "Backref followed by digit: '\\10' parses as \\1 then literal '0' (single-digit backrefs only)", + "query": "SELECT regexp_replace('ab', '(a)', '\\10')", + "observed": "[('a0b',)]" + }, + { + "claim": "Literal dollar sign in replacement", + "query": "SELECT regexp_replace('hello', 'h', '$')", + "observed": "[('$ello',)]" + }, + { + "claim": "Wrapping with 'g': a1b2c3 with (\\d) -> [\\1] globally", + "query": "SELECT regexp_replace('a1b2c3', '(\\d)', '[\\1]', 'g')", + "observed": "[('a[1]b[2]c[3]',)]" + }, + { + "claim": "Empty pattern '' with 'g' on 'abc': where do empty matches insert; terminates", + "query": "SELECT regexp_replace('abc', '', 'X', 'g')", + "observed": "[('XaXbXcX',)]" + }, + { + "claim": "Empty pattern '' without 'g' on 'abc'", + "query": "SELECT regexp_replace('abc', '', 'X')", + "observed": "[('Xabc',)]" + }, + { + "claim": "Pattern 'a*' with 'g' on 'bab': empty + nonempty match interleaving", + "query": "SELECT regexp_replace('bab', 'a*', 'X', 'g')", + "observed": "[('XbXbX',)]" + }, + { + "claim": "Pattern 'a*' with 'g' on empty string ''", + "query": "SELECT regexp_replace('', 'a*', 'X', 'g')", + "observed": "[('X',)]" + }, + { + "claim": "Empty pattern on empty string, no options", + "query": "SELECT regexp_replace('', '', 'X')", + "observed": "[('X',)]" + }, + { + "claim": "Empty pattern '' with 'g' on 'a😀b' inserts at CODEPOINT boundaries (never inside the emoji's UTF-8 bytes): 'XaX😀XbX'", + "query": "SELECT regexp_replace('a😀b', '', 'X', 'g')", + "observed": "[('XaX😀XbX',)]" + }, + { + "claim": "No match returns original string unchanged", + "query": "SELECT regexp_replace('hello world', 'z', 'X')", + "observed": "[('hello world',)]" + }, + { + "claim": "NULL subject string -> NULL", + "query": "SELECT regexp_replace(NULL, 'a', 'X')", + "observed": "[(None,)]" + }, + { + "claim": "NULL pattern -> NULL", + "query": "SELECT regexp_replace('aaa', NULL, 'X')", + "observed": "[(None,)]" + }, + { + "claim": "NULL replacement -> NULL", + "query": "SELECT regexp_replace('aaa', 'a', NULL)", + "observed": "[(None,)]" + }, + { + "claim": "Non-participating group backref: '(a)|(b)' on 'a', replacement '[\\2]'", + "query": "SELECT regexp_replace('a', '(a)|(b)', '[\\2]')", + "observed": "[('[]',)]" + }, + { + "claim": "Optional group that did not match: '(a)(x)?(b)' on 'ab', replacement '\\1\\2\\3' -> 'ab' (a genuine rewrite that happens to equal input; \\2 -> empty)", + "query": "SELECT regexp_replace('ab', '(a)(x)?(b)', '\\1\\2\\3')", + "observed": "[('ab',)]" + }, + { + "claim": "Disambiguation: '(a)(x)?(b)' on 'ab' with '[\\1][\\2][\\3]' -> '[a][][b]' proves in-range non-participating group substitutes EMPTY (contrast with out-of-range no-op)", + "query": "SELECT regexp_replace('ab', '(a)(x)?(b)', '[\\1][\\2][\\3]')", + "observed": "[('[a][][b]',)]" + }, + { + "claim": "'i' option: case-insensitive match 'hello' on 'HELLO'", + "query": "SELECT regexp_replace('HELLO', 'hello', 'X', 'i')", + "observed": "[('X',)]" + }, + { + "claim": "'i' option does NOT fold 'İ' (U+0130) to 'i': 'İstanbul' unchanged (RE2 simple case folding)", + "query": "SELECT regexp_replace('İstanbul', 'i', 'X', 'i')", + "observed": "[('İstanbul',)]" + }, + { + "claim": "'i' option simple-folds ASCII in Unicode text: 'S' matches 's' in 'İstanbul'", + "query": "SELECT regexp_replace('İstanbul', 'S', 'X', 'i')", + "observed": "[('İXtanbul',)]" + }, + { + "claim": "Invalid option letter 'z' -> verbatim error", + "query": "SELECT regexp_replace('aaa', 'a', 'X', 'z')", + "observed": "ERROR [InvalidInputException]: Invalid Input Error: Unrecognized Regex option z" + }, + { + "claim": "Invalid regex pattern '(' -> verbatim error", + "query": "SELECT regexp_replace('a', '(', 'X')", + "observed": "ERROR [InvalidInputException]: Invalid Input Error: missing ): (" + }, + { + "claim": "Accented char replace: 'héllo' é->E", + "query": "SELECT regexp_replace('héllo', 'é', 'E')", + "observed": "[('hEllo',)]" + }, + { + "claim": "Dot '.' with 'g' on 'a😀b': emoji is ONE match (UTF-8 aware) -> 'XXX'", + "query": "SELECT regexp_replace('a😀b', '.', 'X', 'g')", + "observed": "[('XXX',)]" + }, + { + "claim": "Whole-match backref with emoji: '😀' -> '[\\0]'", + "query": "SELECT regexp_replace('a😀b', '😀', '[\\0]')", + "observed": "[('a[😀]b',)]" + }, + { + "claim": "\\d does NOT match Arabic-Indic digits '٣٤' (RE2 \\d is ASCII)", + "query": "SELECT regexp_replace('٣٤', '\\d', 'X', 'g')", + "observed": "[('٣٤',)]" + }, + { + "claim": "[0-9] does not match Arabic-Indic digits either", + "query": "SELECT regexp_replace('٣٤', '[0-9]', 'X', 'g')", + "observed": "[('٣٤',)]" + }, + { + "claim": "Direct replace of 'İ' (U+0130) with 'I'", + "query": "SELECT regexp_replace('İstanbul', 'İ', 'I')", + "observed": "[('Istanbul',)]" + } + ], + "surprises": [ + "Invalid rewrite strings NEVER error, but non-global and global handle them DIFFERENTLY. Non-global: ANY invalid rewrite (out-of-range backref \\2, bad escape \\x, trailing lone backslash) silently skips the whole replacement — original returned. Global 'g': out-of-range backref is still a full no-op (RE2 GlobalReplace pre-checks MaxSubmatch), BUT a bad escape (backslash before non-digit, or trailing lone backslash) still CONSUMES each match and emits the rewrite only up to the bad escape — regexp_replace('hello','h','\\x','g') = 'ello', ('hello','h','A\\xB','g') = 'Aello', ('aaa','a','\\x','g') = ''.", + "In-range non-participating groups behave differently from out-of-range ones: '(a)|(b)' \\2 and '(a)(x)?(b)' \\2 substitute EMPTY string (replacement still happens), while out-of-range \\2 aborts the whole replacement.", + "NULL options argument makes the result NULL (any NULL argument -> NULL), it is not treated as 'no options'.", + "Backrefs are backslash-style ONLY: \\1-\\9 plus \\0 for the whole match; $1/$0/& are literal text. \\10 parses as \\1 followed by literal '0'.", + "Empty pattern with 'g' inserts the replacement at every codepoint boundary INCLUDING both ends ('abc' -> 'XaXbXcX') and terminates; on 'a😀b' boundaries are codepoints, never inside the emoji's UTF-8 bytes.", + "'a*' with 'g' on 'bab' gives 'XbXbX' — no extra empty-match insertion immediately after a nonempty match.", + "'i' does not fold 'İ' (U+0130) to 'i' (RE2 simple folding); \\d and [0-9] do not match Arabic-Indic digits '٣٤'." + ], + "recommendation": "Implement as a thin wrapper matching RE2 Replace (non-global) and GlobalReplace ('g') semantics exactly. Order of behavior: any NULL argument (incl. options) -> NULL; validate option letters ('Invalid Input Error: Unrecognized Regex option '); compile pattern ('Invalid Input Error: missing ): (' style); then run. Rewrite grammar: \\0 = whole match, \\1-\\9 = groups (single digit only, \\10 = \\1 + '0'), \\\\ = literal backslash; $1/$0/& are plain literals. Invalid-rewrite handling must copy RE2's asymmetry, never error: non-global -> return input unchanged for ANY invalid rewrite; global -> out-of-range backref returns input unchanged (MaxSubmatch pre-check), bad escape (\\ before non-digit/non-backslash, incl. trailing \\) consumes each match and emits the rewrite prefix up to the bad escape. In-range non-participating groups substitute empty. Empty-match semantics per RE2 GlobalReplace: replacement inserted at every codepoint boundary including string start and end; no empty match immediately after a nonempty match; codepoint boundaries, never UTF-8 byte interiors. Matching is RE2: \\d and [0-9] are ASCII-only, 'i' is simple case folding (no İ->i). If the engine can't reproduce the global bad-escape partial-emission quirk cheaply, raising a compile-time error on invalid rewrites is a DELIBERATE divergence to flag, not silently differ." +} \ No newline at end of file diff --git a/docs/superpowers/specs/pins-waveB/star-similar-columns.json b/docs/superpowers/specs/pins-waveB/star-similar-columns.json new file mode 100644 index 0000000..e9035a1 --- /dev/null +++ b/docs/superpowers/specs/pins-waveB/star-similar-columns.json @@ -0,0 +1,243 @@ +{ + "area": "star-similar-columns", + "duckdb_version": "1.5.5 (SELECT version() -> 'v1.5.5', executed this session)", + "schema_note": "t(abc INT, abd INT, xyz VARCHAR, \"Weird Name\" INT) rows (1,2,'hello',10),(-3,NULL,'héllo',20),(NULL,2147483647,'a😀b',30); nx(\"héllo\" INT, \"a😀b\" INT, \"٣٤\" INT, \"İstanbul\" INT) row (1,2,3,4); t1(id,abc,name) rows (1,10,'x'),(2,20,'y'); t2(id,abd,name) rows (1,100,'xx'),(2,200,'yy'). All queries run via python duckdb con.sql() unless noted; error-text LINE-context nuance pinned separately.", + "pins": [ + { + "claim": "RE-VERIFIED: positive * SIMILAR TO is an UNANCHORED regex SEARCH over column names: pattern 'c' keeps only abc (substring hit).", + "query": "SELECT * SIMILAR TO 'c' FROM t", + "observed": "cols=[\"abc\"] types=[\"INTEGER\"] rows=[[\"1\"], [\"-3\"], [\"None\"]]" + }, + { + "claim": "Written anchors are respected by the positive form: '^a' keeps abc, abd only ('Weird Name' contains 'a' in 'Name' but not at start; the ^ anchor excludes it).", + "query": "SELECT * SIMILAR TO '^a' FROM t", + "observed": "cols=[\"abc\", \"abd\"] types=[\"INTEGER\", \"INTEGER\"] rows=[[\"1\", \"2\"], [\"-3\", \"None\"], [\"None\", \"2147483647\"]]" + }, + { + "claim": "Unanchored search means mid-name hits count: 'a.*' keeps abc, abd AND \"Weird Name\" (the 'a' inside 'Name' matches).", + "query": "SELECT * SIMILAR TO 'a.*' FROM t", + "observed": "cols=[\"abc\", \"abd\", \"Weird Name\"] types=[\"INTEGER\", \"INTEGER\", \"INTEGER\"] rows=[[\"1\", \"2\", \"10\"], [\"-3\", \"None\", \"20\"], [\"None\", \"2147483647\", \"30\"]]" + }, + { + "claim": "* SIMILAR TO '.*' keeps all four columns in declaration order.", + "query": "SELECT * SIMILAR TO '.*' FROM t", + "observed": "cols=[\"abc\", \"abd\", \"xyz\", \"Weird Name\"] types=[\"INTEGER\", \"INTEGER\", \"VARCHAR\", \"INTEGER\"] rows=[[\"1\", \"2\", \"'hello'\", \"10\"], [\"-3\", \"None\", \"'héllo'\", \"20\"], [\"None\", \"2147483647\", \"'a😀b'\", \"30\"]]" + }, + { + "claim": "RE-VERIFIED asymmetry: * NOT SIMILAR TO 'c' keeps ALL FOUR columns (NOT form is NOT-regexp_full_match; 'c' full-matches no name), even though positive 'c' selected abc — the two forms are NOT complements.", + "query": "SELECT * NOT SIMILAR TO 'c' FROM t", + "observed": "cols=[\"abc\", \"abd\", \"xyz\", \"Weird Name\"] types=[\"INTEGER\", \"INTEGER\", \"VARCHAR\", \"INTEGER\"] rows=[[\"1\", \"2\", \"'hello'\", \"10\"], [\"-3\", \"None\", \"'héllo'\", \"20\"], [\"None\", \"2147483647\", \"'a😀b'\", \"30\"]]" + }, + { + "claim": "* NOT SIMILAR TO '^a' also keeps ALL FOUR columns: '^a' full-matches no whole name (full match of 'abc' against '^a' fails), so nothing is removed — positive '^a' selected {abc,abd} yet NOT '^a' removes nothing.", + "query": "SELECT * NOT SIMILAR TO '^a' FROM t", + "observed": "cols=[\"abc\", \"abd\", \"xyz\", \"Weird Name\"] types=[\"INTEGER\", \"INTEGER\", \"VARCHAR\", \"INTEGER\"] rows=[[\"1\", \"2\", \"'hello'\", \"10\"], [\"-3\", \"None\", \"'héllo'\", \"20\"], [\"None\", \"2147483647\", \"'a😀b'\", \"30\"]]" + }, + { + "claim": "* NOT SIMILAR TO 'a.*' keeps xyz and \"Weird Name\": abc/abd are FULL matches of 'a.*' and are removed; 'Weird Name' is not a full match (doesn't start with a) and survives. NOTE 'Weird Name' appears in BOTH the positive 'a.*' result and the NOT 'a.*' result — proof positive=unanchored-search, negative=NOT-full-match.", + "query": "SELECT * NOT SIMILAR TO 'a.*' FROM t", + "observed": "cols=[\"xyz\", \"Weird Name\"] types=[\"VARCHAR\", \"INTEGER\"] rows=[[\"'hello'\", \"10\"], [\"'héllo'\", \"20\"], [\"'a😀b'\", \"30\"]]" + }, + { + "claim": "* NOT SIMILAR TO '.*' removes everything -> empty-set BinderException whose text embeds the desugaring COLUMNS(list_filter([...4 names incl 'Weird Name'...], lambda: NOT regexp_full_match(...))); identical text from both con.execute and con.sql.", + "query": "SELECT * NOT SIMILAR TO '.*' FROM t", + "observed": "ERROR (BinderException): Binder Error: Star expression \"COLUMNS(list_filter(['abc', 'abd', 'xyz', 'Weird Name'], (lambda __lambda_col: (NOT regexp_full_match(__lambda_col, '.*')))))\" resulted in an empty set of columns" + }, + { + "claim": "Empty pattern: * SIMILAR TO '' keeps ALL columns (empty regex search matches every name).", + "query": "SELECT * SIMILAR TO '' FROM t", + "observed": "cols=[\"abc\", \"abd\", \"xyz\", \"Weird Name\"] types=[\"INTEGER\", \"INTEGER\", \"VARCHAR\", \"INTEGER\"] rows=[[\"1\", \"2\", \"'hello'\", \"10\"], [\"-3\", \"None\", \"'héllo'\", \"20\"], [\"None\", \"2147483647\", \"'a😀b'\", \"30\"]]" + }, + { + "claim": "* NOT SIMILAR TO '' ALSO keeps all columns ('' full-matches only the empty name, which doesn't exist) — both empty-pattern forms are total, another non-complement pair.", + "query": "SELECT * NOT SIMILAR TO '' FROM t", + "observed": "cols=[\"abc\", \"abd\", \"xyz\", \"Weird Name\"] types=[\"INTEGER\", \"INTEGER\", \"VARCHAR\", \"INTEGER\"] rows=[[\"1\", \"2\", \"'hello'\", \"10\"], [\"-3\", \"None\", \"'héllo'\", \"20\"], [\"None\", \"2147483647\", \"'a😀b'\", \"30\"]]" + }, + { + "claim": "Zero-match * SIMILAR TO error verbatim: 'No matching columns found that match regex' + Did-you-mean listing ALL columns quoted, comma-separated (\"abc\", \"abd\", \"Weird Name\", \"xyz\"); NO LINE-context in either python API (con.execute or con.sql).", + "query": "SELECT * SIMILAR TO 'zz' FROM t", + "observed": "ERROR (BinderException): Binder Error: No matching columns found that match regex \"zz\"\n\nDid you mean: \"abc\", \"abd\", \"Weird Name\", \"xyz\"" + }, + { + "claim": "* NOT SIMILAR TO 'zz' raises NO error — all four columns survive (the NOT form only errors when the pattern full-matches EVERY name; a nowhere-matching pattern keeps everything).", + "query": "SELECT * NOT SIMILAR TO 'zz' FROM t", + "observed": "cols=[\"abc\", \"abd\", \"xyz\", \"Weird Name\"] types=[\"INTEGER\", \"INTEGER\", \"VARCHAR\", \"INTEGER\"] rows=[[\"1\", \"2\", \"'hello'\", \"10\"], [\"-3\", \"None\", \"'héllo'\", \"20\"], [\"None\", \"2147483647\", \"'a😀b'\", \"30\"]]" + }, + { + "claim": "Invalid regex in POSITIVE * SIMILAR TO is a BinderException 'Failed to compile regex \"[\": missing ]: [' (no LINE context via either API).", + "query": "SELECT * SIMILAR TO '[' FROM t", + "observed": "ERROR (BinderException): Binder Error: Failed to compile regex \"[\": missing ]: [" + }, + { + "claim": "Invalid regex in NEGATIVE * NOT SIMILAR TO is a DIFFERENT exception class AND text: InvalidInputException 'Invalid Input Error: missing ]: [' (comes from regexp_full_match, not the star binder).", + "query": "SELECT * NOT SIMILAR TO '[' FROM t", + "observed": "ERROR (InvalidInputException): Invalid Input Error: missing ]: [" + }, + { + "claim": "Non-constant pattern (column ref) rejected: 'Pattern applied to a star expression must be a constant'; same text for the NOT form.", + "query": "SELECT * SIMILAR TO xyz FROM t -- and: SELECT * NOT SIMILAR TO xyz FROM t", + "observed": "ERROR (BinderException): Binder Error: Pattern applied to a star expression must be a constant (both forms identical)" + }, + { + "claim": "Even a constant-FOLDABLE expression is rejected — the pattern must be a bare literal, no folding: ('ab' || 'c') gives the same 'must be a constant' error.", + "query": "SELECT * SIMILAR TO ('ab' || 'c') FROM t", + "observed": "ERROR (BinderException): Binder Error: Pattern applied to a star expression must be a constant" + }, + { + "claim": "A prepared-statement parameter is also rejected as non-constant, with LINE context pointing at the star.", + "query": "python: con.execute(\"SELECT * SIMILAR TO ? FROM t\", ['ab.'])", + "observed": "ERROR (BinderException): Binder Error: Pattern applied to a star expression must be a constant\n\nLINE 1: SELECT * SIMILAR TO ? FROM t\n ^" + }, + { + "claim": "* SIMILAR TO NULL is NOT the LIKE-family empty-set error: it raises the COLUMNS argument-type error 'COLUMNS expects either a VARCHAR argument (regex) or a LIST of VARCHAR (list of columns)'.", + "query": "SELECT * SIMILAR TO NULL FROM t", + "observed": "ERROR (BinderException): Binder Error: COLUMNS expects either a VARCHAR argument (regex) or a LIST of VARCHAR (list of columns)" + }, + { + "claim": "RE2 inline flags work in * SIMILAR TO: '(?i)AB.' keeps abc, abd (bare 'AB.' would match nothing — matching is case-sensitive by default).", + "query": "SELECT * SIMILAR TO '(?i)AB.' FROM t", + "observed": "cols=[\"abc\", \"abd\"] types=[\"INTEGER\", \"INTEGER\"] rows=[[\"1\", \"2\"], [\"-3\", \"None\"], [\"None\", \"2147483647\"]]" + }, + { + "claim": "Names with spaces participate normally: 'Weird' (substring) selects \"Weird Name\"; * NOT SIMILAR TO 'Weird Name' (exact full match) removes ONLY that column.", + "query": "SELECT * SIMILAR TO 'Weird' FROM t -- and: SELECT * NOT SIMILAR TO 'Weird Name' FROM t", + "observed": "positive: cols=[\"Weird Name\"] rows=[[\"10\"],[\"20\"],[\"30\"]]; negative: cols=[\"abc\", \"abd\", \"xyz\"] rows=[[\"1\",\"2\",\"'hello'\"],[\"-3\",\"None\",\"'héllo'\"],[\"None\",\"2147483647\",\"'a😀b'\"]]" + }, + { + "claim": "ESCAPE clause on a star SIMILAR TO parses but raises NotImplementedException: 'Not implemented Error: Custom escape in SIMILAR TO'.", + "query": "SELECT * SIMILAR TO 'a' ESCAPE '\\' FROM t", + "observed": "ERROR (NotImplementedException): Not implemented Error: Custom escape in SIMILAR TO" + }, + { + "claim": "EXCLUDE composes BEFORE the filter, same as LIKE-family: * EXCLUDE (abc) SIMILAR TO 'ab.' yields just abd; the reverse order is a parser error at or near \"EXCLUDE\".", + "query": "SELECT * EXCLUDE (abc) SIMILAR TO 'ab.' FROM t -- and: SELECT * SIMILAR TO 'ab.' EXCLUDE (abc) FROM t", + "observed": "ok form: cols=[\"abd\"] rows=[[\"2\"],[\"None\"],[\"2147483647\"]]; reverse: ERROR (ParserException): Parser Error: syntax error at or near \"EXCLUDE\"\n\nLINE 1: SELECT * SIMILAR TO 'ab.' EXCLUDE (abc) FROM t\n ^" + }, + { + "claim": "EXCLUDE of the quoted spacey name composes with the filter: * EXCLUDE (\"Weird Name\") SIMILAR TO '.*' keeps abc, abd, xyz.", + "query": "SELECT * EXCLUDE (\"Weird Name\") SIMILAR TO '.*' FROM t", + "observed": "cols=[\"abc\", \"abd\", \"xyz\"] types=[\"INTEGER\", \"INTEGER\", \"VARCHAR\"] rows=[[\"1\", \"2\", \"'hello'\"], [\"-3\", \"None\", \"'héllo'\"], [\"None\", \"2147483647\", \"'a😀b'\"]]" + }, + { + "claim": "An AS alias on a star SIMILAR TO is LEGAL and is copied to EVERY surviving column, producing duplicate output names (x, x).", + "query": "SELECT * SIMILAR TO 'ab.' AS x FROM t WHERE abc = 1", + "observed": "cols=[\"x\", \"x\"] types=[\"INTEGER\", \"INTEGER\"] rows=[[\"1\", \"2\"]]" + }, + { + "claim": "Qualified stars support both forms: t.* SIMILAR TO 'ab.' -> {abc,abd}; t.* NOT SIMILAR TO 'ab.' -> {xyz, Weird Name}; on a join, t1.* SIMILAR TO 'a' filters only t1's columns (abc, name).", + "query": "SELECT t.* SIMILAR TO 'ab.' FROM t -- and t.* NOT SIMILAR TO 'ab.'; and SELECT t1.* SIMILAR TO 'a' FROM t1 JOIN t2 ON t1.id = t2.id", + "observed": "t.* SIMILAR: cols=[\"abc\",\"abd\"]; t.* NOT SIMILAR: cols=[\"xyz\",\"Weird Name\"]; join: cols=[\"abc\",\"name\"] rows=[[\"10\",\"'x'\"],[\"20\",\"'y'\"]]" + }, + { + "claim": "COLUMNS('ab.') keeps name-matching columns in TABLE DECLARATION order (abc, abd), values/types untouched.", + "query": "SELECT COLUMNS('ab.') FROM t", + "observed": "cols=[\"abc\", \"abd\"] types=[\"INTEGER\", \"INTEGER\"] rows=[[\"1\", \"2\"], [\"-3\", \"None\"], [\"None\", \"2147483647\"]]" + }, + { + "claim": "Expansion order is DECLARATION order, not regex/alternation order: COLUMNS('(xyz|abc)') yields abc THEN xyz; COLUMNS('d|c') yields abc, abd, \"Weird Name\" (the 'd' in 'Weird' matches).", + "query": "SELECT COLUMNS('(xyz|abc)') FROM t -- and: SELECT COLUMNS('d|c') FROM t", + "observed": "'(xyz|abc)': cols=[\"abc\", \"xyz\"]; 'd|c': cols=[\"abc\", \"abd\", \"Weird Name\"]" + }, + { + "claim": "An alias on a COLUMNS expression is applied to EVERY expansion producing duplicate output names: COLUMNS('ab.') + 1 AS x -> cols [x, x] with per-column values (abc+1, abd+1).", + "query": "SELECT COLUMNS('ab.') + 1 AS x FROM t WHERE abc = 1", + "observed": "cols=[\"x\", \"x\"] types=[\"INTEGER\", \"INTEGER\"] rows=[[\"2\", \"3\"]]" + }, + { + "claim": "WITHOUT an alias, a COLUMNS expression names outputs by the BARE column name — not the expression text: COLUMNS('ab.') * 2 -> cols [abc, abd] holding the DOUBLED values.", + "query": "SELECT COLUMNS('ab.') * 2 FROM t WHERE abc = 1", + "observed": "cols=[\"abc\", \"abd\"] types=[\"INTEGER\", \"INTEGER\"] rows=[[\"2\", \"4\"]]" + }, + { + "claim": "An alias may collide with a real column name (AS abc) — duplicates allowed, no error; COLUMNS(*) AS x aliases all four expansions to x.", + "query": "SELECT COLUMNS('ab.') AS abc FROM t -- and: SELECT COLUMNS(*) AS x FROM t WHERE abc = 1", + "observed": "AS abc: cols=[\"abc\", \"abc\"]; COLUMNS(*) AS x: cols=[\"x\", \"x\", \"x\", \"x\"] rows=[[\"1\", \"2\", \"'hello'\", \"10\"]]" + }, + { + "claim": "COLUMNS('(?i)AB.') matches case-insensitively (abc, abd); bare COLUMNS regex is case-sensitive.", + "query": "SELECT COLUMNS('(?i)AB.') FROM t", + "observed": "cols=[\"abc\", \"abd\"] types=[\"INTEGER\", \"INTEGER\"] rows=[[\"1\", \"2\"], [\"-3\", \"None\"], [\"None\", \"2147483647\"]]" + }, + { + "claim": "COLUMNS matches quoted/spacey names as plain strings: 'Weird' and 'rd Na' (regex containing a space) both select \"Weird Name\".", + "query": "SELECT COLUMNS('Weird') FROM t -- and: SELECT COLUMNS('rd Na') FROM t", + "observed": "both: cols=[\"Weird Name\"] types=[\"INTEGER\"] rows=[[\"10\"], [\"20\"], [\"30\"]]" + }, + { + "claim": "COLUMNS(*) == plain * == COLUMNS('.*') == COLUMNS('') — all four expand to every column in declaration order with bare names.", + "query": "SELECT COLUMNS(*) FROM t -- vs SELECT * FROM t, SELECT COLUMNS('.*') FROM t, SELECT COLUMNS('') FROM t", + "observed": "all four queries: cols=[\"abc\", \"abd\", \"xyz\", \"Weird Name\"] types=[\"INTEGER\", \"INTEGER\", \"VARCHAR\", \"INTEGER\"], identical rows" + }, + { + "claim": "COLUMNS zero-match error verbatim — same 'No matching columns' + Did-you-mean text as * SIMILAR TO, but WITH LINE context when run via con.execute (con.sql omits the LINE block; * SIMILAR TO never includes it in either API).", + "query": "SELECT COLUMNS('zz') FROM t (via con.execute)", + "observed": "ERROR (BinderException): Binder Error: No matching columns found that match regex \"zz\"\n\nDid you mean: \"abc\", \"abd\", \"Weird Name\", \"xyz\"\n\nLINE 1: SELECT COLUMNS('zz') FROM t\n ^" + }, + { + "claim": "COLUMNS('[') invalid-regex error verbatim (con.execute includes LINE context; con.sql gives the same first line without it).", + "query": "SELECT COLUMNS('[') FROM t (via con.execute)", + "observed": "ERROR (BinderException): Binder Error: Failed to compile regex \"[\": missing ]: [\n\nLINE 1: SELECT COLUMNS('[') FROM t\n ^" + }, + { + "claim": "COLUMNS() is not a constant-pattern error but a lambda-binding error: 'Failed to bind \"xyz\" - COLUMNS expression can only contain lambda parameters'; COLUMNS(123) and COLUMNS(NULL) both give 'COLUMNS expects either a VARCHAR argument (regex) or a LIST of VARCHAR (list of columns)'.", + "query": "SELECT COLUMNS(xyz) FROM t -- and COLUMNS(123), COLUMNS(NULL)", + "observed": "COLUMNS(xyz): ERROR (BinderException): Binder Error: Failed to bind \"xyz\" - COLUMNS expression can only contain lambda parameters; COLUMNS(123)/COLUMNS(NULL): ERROR (BinderException): Binder Error: COLUMNS expects either a VARCHAR argument (regex) or a LIST of VARCHAR (list of columns)" + }, + { + "claim": "There is NO qualified COLUMNS form: t.COLUMNS('ab.') binds as a scalar function lookup and fails with CatalogException 'Scalar Function with name columns does not exist!' + Did-you-mean \"main.count\".", + "query": "SELECT t.COLUMNS('ab.') FROM t", + "observed": "ERROR (CatalogException): Catalog Error: Scalar Function with name columns does not exist!\nDid you mean \"main.count\"?" + }, + { + "claim": "COLUMNS list-form: COLUMNS(['abc','xyz']) and COLUMNS(['xyz','abc']) BOTH output in declaration order (abc, xyz) — list order is ignored; duplicate list entries dedupe silently; unknown name errors 'Column \"nope\" was selected but was not found in the FROM clause'; empty list [] gives the VARCHAR/LIST argument error.", + "query": "SELECT COLUMNS(['xyz','abc']) FROM t -- and ['abc','abc'], ['nope'], []", + "observed": "['xyz','abc']: cols=[\"abc\", \"xyz\"]; ['abc','abc']: cols=[\"abc\"]; ['nope']: ERROR (BinderException): Binder Error: Column \"nope\" was selected but was not found in the FROM clause; []: ERROR (BinderException): Binder Error: COLUMNS expects either a VARCHAR argument (regex) or a LIST of VARCHAR (list of columns)" + }, + { + "claim": "The same COLUMNS may appear multiple times in one expression and zips per-column (abc+abc, abd+abd), still named by bare column name; two DIFFERENT star/COLUMNS sets in one expression error: 'Multiple different STAR/COLUMNS in the same expression are not supported'. min(COLUMNS('ab.')) is one aggregate per column, named abc/abd.", + "query": "SELECT COLUMNS('ab.') + COLUMNS('ab.') FROM t WHERE abc = 1 -- and + COLUMNS('xy.'); and SELECT min(COLUMNS('ab.')) FROM t", + "observed": "same-regex: cols=[\"abc\", \"abd\"] rows=[[\"2\", \"4\"]]; different: ERROR (BinderException): Binder Error: Multiple different STAR/COLUMNS in the same expression are not supported; min: cols=[\"abc\", \"abd\"] rows=[[\"-3\", \"2\"]]" + }, + { + "claim": "COLUMNS coexists with ordinary select-list items (SELECT abc, COLUMNS('x.z')) and works across a join's combined column set (COLUMNS('ab.') over t1 JOIN t2 picks abc from t1 and abd from t2).", + "query": "SELECT abc, COLUMNS('x.z') FROM t -- and: SELECT COLUMNS('ab.') FROM t1 JOIN t2 ON t1.id = t2.id", + "observed": "mix: cols=[\"abc\", \"xyz\"]; join: cols=[\"abc\", \"abd\"] rows=[[\"10\", \"100\"], [\"20\", \"200\"]]" + }, + { + "claim": "Non-ASCII names match by codepoint: COLUMNS('é') -> \"héllo\"; COLUMNS('😀') -> \"a😀b\"; COLUMNS('٣') -> \"٣٤\"; * SIMILAR TO 'é' and '٣٤' behave identically; * NOT SIMILAR TO '٣٤' removes only the full-matching \"٣٤\".", + "query": "SELECT COLUMNS('é') FROM nx -- plus '😀', '٣'; SELECT * SIMILAR TO '٣٤' FROM nx; SELECT * NOT SIMILAR TO '٣٤' FROM nx", + "observed": "COLUMNS('é'): cols=[\"héllo\"]; COLUMNS('😀'): cols=[\"a😀b\"]; COLUMNS('٣'): cols=[\"٣٤\"]; * SIMILAR TO '٣٤': cols=[\"٣٤\"]; NOT: cols=[\"héllo\", \"a😀b\", \"İstanbul\"] rows=[[\"1\", \"2\", \"4\"]]" + }, + { + "claim": "(?i) uses RE2 simple case folding, NOT Turkish/locale folding: COLUMNS('(?i)istanbul') does NOT match \"İstanbul\" (zero-match error, Did-you-mean \"İstanbul\"), while COLUMNS('(?i)İSTANBUL') DOES match it (ASCII letters fold; İ only matches İ).", + "query": "SELECT COLUMNS('(?i)istanbul') FROM nx -- and: SELECT COLUMNS('(?i)İSTANBUL') FROM nx", + "observed": "'(?i)istanbul': ERROR (BinderException): Binder Error: No matching columns found that match regex \"(?i)istanbul\"\n\nDid you mean: \"İstanbul\"; '(?i)İSTANBUL': cols=[\"İstanbul\"] rows=[[\"4\"]]" + }, + { + "claim": "COLUMNS in a WHERE clause expands to an AND-conjunction across matched columns: WHERE COLUMNS('ab.') IS NOT NULL keeps only the row where abc AND abd are both non-NULL.", + "query": "SELECT * FROM t WHERE COLUMNS('ab.') IS NOT NULL", + "observed": "cols=[\"abc\", \"abd\", \"xyz\", \"Weird Name\"] rows=[[\"1\", \"2\", \"'hello'\", \"10\"]] (rows 2 and 3 eliminated: each has one NULL among abc/abd)" + }, + { + "claim": "Lambda form: COLUMNS(c -> c SIMILAR TO 'ab.') uses SCALAR SIMILAR TO semantics (anchored full match) on names and keeps abc, abd — consistent with the star NOT-form being full-match-based.", + "query": "SELECT COLUMNS(c -> c SIMILAR TO 'ab.') FROM t", + "observed": "cols=[\"abc\", \"abd\"] types=[\"INTEGER\", \"INTEGER\"] rows=[[\"1\", \"2\"], [\"-3\", \"None\"], [\"None\", \"2147483647\"]]" + } + ], + "surprises": [ + "* NOT SIMILAR TO '^a' keeps ALL columns including abc/abd: because the NOT form is NOT-regexp_full_match, a '^'-anchored prefix pattern that full-matches nothing removes nothing — positive and negative forms disagree on the SAME anchored pattern.", + "\"Weird Name\" appears in BOTH * SIMILAR TO 'a.*' (unanchored search finds 'a' in 'Name') and * NOT SIMILAR TO 'a.*' (not a full match) — the crispest single-table proof of the search-vs-full-match asymmetry.", + "Invalid regex errors differ BY EXCEPTION CLASS between the forms: positive -> BinderException 'Failed to compile regex \"[\": missing ]: ['; negative -> InvalidInputException 'Invalid Input Error: missing ]: [' (bubbles out of regexp_full_match).", + "* SIMILAR TO NULL raises the COLUMNS argument-type error ('COLUMNS expects either a VARCHAR argument (regex) or a LIST of VARCHAR'), unlike * LIKE NULL which raises the empty-set desugaring error — NULL takes the COLUMNS path only for SIMILAR TO.", + "The star pattern must be a BARE literal: even constant-foldable ('ab' || 'c') and prepared-statement parameters raise 'Pattern applied to a star expression must be a constant'.", + "* SIMILAR TO 'x' ESCAPE 'y' parses but hits NotImplementedException 'Custom escape in SIMILAR TO'.", + "An alias is legal on a star filter (* SIMILAR TO 'ab.' AS x) and on COLUMNS, and is copied to EVERY expansion producing duplicate output names; without an alias, COLUMNS('ab.') * 2 still names outputs by the BARE column name (abc, abd), not the expression text.", + "COLUMNS list-form ignores the list order — COLUMNS(['xyz','abc']) outputs abc,xyz in table declaration order — and silently dedupes duplicate entries; unknown entries error with 'was selected but was not found in the FROM clause' (a different text from EXCLUDE/REPLACE errors).", + "LINE-context in error text depends on the python API for COLUMNS errors (con.execute includes 'LINE 1: ...', con.sql drops it) but * SIMILAR TO zero-match/compile errors NEVER carry LINE context in either API.", + "Zero-match Did-you-mean lists EVERY column of the table quoted and comma-separated (\"abc\", \"abd\", \"Weird Name\", \"xyz\"), not just the closest name.", + "The NOT form cannot produce the zero-match 'No matching columns' error: * NOT SIMILAR TO 'zz' succeeds with all columns; the only NOT-form error is the empty-set desugaring error when the pattern full-matches every name.", + "(?i) case folding is RE2 simple folding: ASCII letters fold, but 'İ' (U+0130) does not fold to 'i' — '(?i)istanbul' misses \"İstanbul\" while '(?i)İSTANBUL' hits it.", + "COLUMNS in WHERE expands as an AND-conjunction over the matched columns (documented DuckDB behavior, confirmed)." + ], + "recommendation": "Implement both forms over one bind-time name-selection pass with two distinct predicates: positive * SIMILAR TO p / COLUMNS(p) = RE2 partial (unanchored) search of p over declared-case names; * NOT SIMILAR TO p = NOT regexp_full_match(name, p). Never derive one from the other — they are not complements (pin the '^a' and 'a.*' cases as regression tests). Pattern must be a bare string literal (reject column refs, foldable exprs, and parameters with 'Pattern applied to a star expression must be a constant'; NULL routes to the COLUMNS VARCHAR/LIST argument error). Selection preserves table declaration order everywhere, including COLUMNS(list) which also dedupes. Error surface to reproduce: (1) positive zero-match -> 'No matching columns found that match regex \"p\"' + Did-you-mean listing ALL candidate columns; (2) NOT-form all-eliminated -> empty-set error embedding COLUMNS(list_filter([names...], (lambda __lambda_col: (NOT regexp_full_match(__lambda_col, 'p'))))); (3) positive bad regex -> BinderException 'Failed to compile regex ...'; NOT-form bad regex -> InvalidInputException with the raw RE2 message — different class, keep them distinct; (4) ESCAPE -> NotImplementedException 'Custom escape in SIMILAR TO'. Grammar: * [EXCLUDE (...)] SIMILAR TO 'p' only (filter last; reverse order is a parse error); alias after the filter is legal and stamps every expansion (duplicate output names allowed). Aliases on COLUMNS likewise stamp every expansion; un-aliased COLUMNS expressions name outputs by the bare column name. Support qualified t.* SIMILAR TO with per-table candidate sets; there is no qualified COLUMNS. Use RE2 semantics exactly (byte/codepoint matching, (?i) simple folding — no locale folding)." +} From 023b1e77d547ae65461dda493ddf6fce88d919ee Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Mon, 27 Jul 2026 02:32:09 +0200 Subject: [PATCH 3/5] =?UTF-8?q?feat(specializer):=20wave-B=20regexp=20fami?= =?UTF-8?q?ly=20=E2=80=94=20rust-regex=20behind=20an=20RE2=20translation?= =?UTF-8?q?=20layer,=20corpus=20484=20->=20505=20(TASK-53)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine is regex 1.x with the measured parity recipe (pins-waveB/ re2-vs-rust-regex.json): RegexBuilder::octal(true), default Unicode mode (unicode(false) BREAKS (?i) folding parity — measured), and a bind-time pattern rewrite in retrans.rs closing the whole Perl-class gap (\d -> (?-u:\d) etc, in-class variants included) plus a reject list for the irreconcilables (\B, (?), duplicate group names, bounds > 1000, stacked quantifiers — the last silently WRONG in rust, not just error-shaped different). Replacement templates translate \N -> ${N}, $ -> $$, with RE2's invalid-rewrite quirks resolved at bind (out-of- range backref = identity; global bad escape = consume-with-prefix). Program grows a prepare-time regex table (print/parse round-trips it); ReMatch/ReExtract/ReReplace run on both backends. Frontend serves regexp_matches (SEARCH) / regexp_full_match / regexp_extract (''-on-miss, flat 0..9 group check, NULL-group -> '') / regexp_replace (backslash backrefs, 'g', NULL options -> NULL — the pinned asymmetry); ~ / !~ are FULL match (NOT the Postgres search — measured, the DuckDB binder error names regexp_full_match); SIMILAR TO passes the RAW pattern (no %% translation) full-anchored; star * SIMILAR TO filters names by unanchored search while NOT SIMILAR TO negates a full match (independent predicates — pinned asymmetry); bare COLUMNS('re')/COLUMNS(*) expand in declared order with alias stamping. List-valued forms stay clean-unsupported. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 39 +++ Cargo.toml | 1 + src/specializer/exec/cranelift.rs | 110 +++++++ src/specializer/exec/interp.rs | 77 ++++- src/specializer/exec/tests.rs | 9 +- src/specializer/fold.rs | 14 + src/specializer/frontend.rs | 476 ++++++++++++++++++++++++++++-- src/specializer/ir/gen.rs | 1 + src/specializer/ir/mod.rs | 42 +++ src/specializer/ir/parse.rs | 92 ++++++ src/specializer/ir/print.rs | 25 +- src/specializer/ir/tests.rs | 3 + src/specializer/ir/verify.rs | 23 +- src/specializer/lower.rs | 46 ++- src/specializer/mod.rs | 5 +- src/specializer/plan.rs | 18 ++ src/specializer/retrans.rs | 333 +++++++++++++++++++++ src/specializer/rewrite.rs | 13 + src/specializer/tests.rs | 131 +++++++- tests/test_duckdb_waveB_regexp.py | 73 +++++ 20 files changed, 1494 insertions(+), 37 deletions(-) create mode 100644 src/specializer/retrans.rs create mode 100644 tests/test_duckdb_waveB_regexp.py diff --git a/Cargo.lock b/Cargo.lock index b30379c..be7f12f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -451,6 +460,35 @@ dependencies = [ "smallvec", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "region" version = "3.0.2" @@ -519,6 +557,7 @@ dependencies = [ "cranelift-jit", "cranelift-module", "pyo3", + "regex", "sqlparser", "target-lexicon", ] diff --git a/Cargo.toml b/Cargo.toml index 31e93f0..62b984d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,5 +16,6 @@ cranelift-frontend = "0.126" cranelift-jit = "0.126" cranelift-module = "0.126" pyo3 = { version = "0.29", features = ["abi3-py314"] } +regex = "1" sqlparser = "0.62" target-lexicon = "0.13.5" diff --git a/src/specializer/exec/cranelift.rs b/src/specializer/exec/cranelift.rs index 01128c3..1946ba8 100644 --- a/src/specializer/exec/cranelift.rs +++ b/src/specializer/exec/cranelift.rs @@ -57,9 +57,18 @@ struct Cx { input: *const Batch, statics: *const PreparedStatic, statics_len: usize, + regexes: *const CompiledRe, + regexes_len: usize, trap: Option, } +/// One program regex, compiled, with its replace template (if any) — owned +/// by the [`CraneliftFn`], read by the h_re* helpers through [`Cx`]. +pub(super) struct CompiledRe { + rx: std::rc::Rc, + template: Option, +} + impl Cx { fn arena(&mut self) -> &mut Arena { unsafe { &mut *self.arena } @@ -591,6 +600,63 @@ extern "C" fn h_srepeat(p: *mut Cx, ao: i64, al: i64, n: i64, len_out: *mut i64) } } +extern "C" fn h_rematch(p: *mut Cx, re: i64, ao: i64, al: i64) -> u8 { + let c = unsafe { cx(p) }; + let rx = &unsafe { std::slice::from_raw_parts(c.regexes, c.regexes_len) }[re as usize]; + let arena = unsafe { &*c.arena }; + rx.rx.is_match(arena.get(span(ao, al))) as u8 +} + +extern "C" fn h_reextract( + p: *mut Cx, + re: i64, + group: i64, + ao: i64, + al: i64, + len_out: *mut i64, +) -> i64 { + let c = unsafe { cx(p) }; + let rx = &unsafe { std::slice::from_raw_parts(c.regexes, c.regexes_len) }[re as usize]; + let arena = unsafe { &mut *c.arena }; + // No match / non-participating group -> '' (wave-B pins). + let out = { + let s = arena.get(span(ao, al)); + rx.rx + .captures(s) + .and_then(|caps| caps.get(group as usize)) + .map(|m| m.as_str().to_string()) + .unwrap_or_default() + }; + let r = arena.push_str(&out); + unsafe { *len_out = r.len as i64 }; + r.off as i64 +} + +extern "C" fn h_rereplace( + p: *mut Cx, + re: i64, + global: i64, + ao: i64, + al: i64, + len_out: *mut i64, +) -> i64 { + let c = unsafe { cx(p) }; + let rx = &unsafe { std::slice::from_raw_parts(c.regexes, c.regexes_len) }[re as usize]; + let arena = unsafe { &mut *c.arena }; + let template = rx.template.as_deref().unwrap_or_default(); + let out = { + let s = arena.get(span(ao, al)); + if global != 0 { + rx.rx.replace_all(s, template).into_owned() + } else { + rx.rx.replace(s, template).into_owned() + } + }; + let r = arena.push_str(&out); + unsafe { *len_out = r.len as i64 }; + r.off as i64 +} + extern "C" fn h_ishl(p: *mut Cx, x: i64, y: i64) -> i64 { match interp::duck_shl(x, y) { Ok(v) => v, @@ -836,6 +902,8 @@ pub struct CraneliftFn { row_fn: RowFn, interp: InterpFn, trap_msgs: Vec, + /// Program regexes compiled once, read by helpers through Cx. + regexes: Vec, /// Owned backing for absolute addresses baked into the code. _const_strs: Vec>, _probe_descs: Vec>, @@ -1084,11 +1152,21 @@ pub fn compile(p: &Program, statics: Vec) -> Result { + let (ao, al) = vals[&a.0].str2(); + let rev = icon(b, *re as i64); + let v = call_h(b, module, "h_rematch", &[cxp, rev, ao, al]).unwrap(); + vals.insert(dst.0, V::S(v)); + } + Inst::ReExtract { re, group, dst, a } => { + let (ao, al) = vals[&a.0].str2(); + let rev = icon(b, *re as i64); + let gv = icon(b, *group as i64); + let lp = b.ins().stack_addr(types::I64, slot_out, 0); + let off = call_h(b, module, "h_reextract", &[cxp, rev, gv, ao, al, lp]).unwrap(); + let len = b.ins().stack_load(types::I64, slot_out, 0); + vals.insert(dst.0, V::Str(off, len)); + } + Inst::ReReplace { re, global, dst, a } => { + let (ao, al) = vals[&a.0].str2(); + let rev = icon(b, *re as i64); + let gv = icon(b, *global as i64); + let lp = b.ins().stack_addr(types::I64, slot_out, 0); + let off = call_h(b, module, "h_rereplace", &[cxp, rev, gv, ao, al, lp]).unwrap(); + let len = b.ins().stack_load(types::I64, slot_out, 0); + vals.insert(dst.0, V::Str(off, len)); + } Inst::Sord { empty_zero, dst, @@ -1850,6 +1954,9 @@ const HELPERS: &[(&str, *const u8)] = &[ ("h_sjaccard", h_sjaccard as *const u8), ("h_str3", h_str3 as *const u8), ("h_srepeat", h_srepeat as *const u8), + ("h_rematch", h_rematch as *const u8), + ("h_reextract", h_reextract as *const u8), + ("h_rereplace", h_rereplace as *const u8), ("h_ishl", h_ishl as *const u8), ("h_sextract", h_sextract as *const u8), ("h_spad", h_spad as *const u8), @@ -1887,6 +1994,9 @@ fn helper_sig(name: &str, sig: &mut cranelift_codegen::ir::Signature, ptr: types "h_sjaccard" => (&[ptr, I64, I64, I64, I64], Some(F64)), "h_str3" => (&[ptr, I64, I64, I64, I64, I64, I64, I64, I64], Some(I64)), "h_srepeat" => (&[ptr, I64, I64, I64, I64], Some(I64)), + "h_rematch" => (&[ptr, I64, I64, I64], Some(I8)), + "h_reextract" => (&[ptr, I64, I64, I64, I64, I64], Some(I64)), + "h_rereplace" => (&[ptr, I64, I64, I64, I64, I64], Some(I64)), "h_ishl" => (&[ptr, I64, I64], Some(I64)), "h_sextract" => (&[ptr, I64, I64, I64, I64], Some(I64)), "h_spad" => (&[ptr, I64, I64, I64, I64, I64, I64, I64], Some(I64)), diff --git a/src/specializer/exec/interp.rs b/src/specializer/exec/interp.rs index 9c6ae8f..5b5bd2b 100644 --- a/src/specializer/exec/interp.rs +++ b/src/specializer/exec/interp.rs @@ -41,6 +41,9 @@ pub enum CompileError { Verify(Vec), /// The static data does not match the program's static declarations. Static(String), + /// A ReSpec pattern failed to compile — the frontend validates patterns + /// at bind, so this only fires on hand-written IR. + Regex(String), } impl std::fmt::Display for CompileError { @@ -54,10 +57,28 @@ impl std::fmt::Display for CompileError { Ok(()) } CompileError::Static(msg) => write!(f, "static data mismatch: {msg}"), + CompileError::Regex(msg) => write!(f, "regex table entry failed to compile: {msg}"), } } } +/// Compile every [`ir::ReSpec`] in the program with the wave-B builder +/// settings (octal(true), default Unicode — see retrans.rs). +pub(super) fn compile_regexes(p: &Program) -> Result>, String> { + p.regexes + .iter() + .map(|r| { + regex::RegexBuilder::new(&r.pattern) + .case_insensitive(r.ci) + .dot_matches_new_line(r.dotall) + .octal(true) + .build() + .map(std::rc::Rc::new) + .map_err(|e| e.to_string()) + }) + .collect() +} + /// One compiled instruction: reads registers/input/statics, writes registers /// or output builders. `Ok(())` or a call-aborting trap. type InstFn = Box Fn(&mut Ctx<'a>) -> Result<(), Trap>>; @@ -120,6 +141,7 @@ pub struct InterpFn { pub fn compile(p: &Program, statics: Vec) -> Result { verify(p).map_err(CompileError::Verify)?; let prepared = prepare_statics(p, statics)?; + let regexes = compile_regexes(p).map_err(CompileError::Regex)?; // Register slots are assigned densely in definition order, decoupling // the frame size from raw value ids — a verified program with sparse ids @@ -143,7 +165,7 @@ pub fn compile(p: &Program, statics: Vec) -> Result = Vec::with_capacity(b.insts.len()); for inst in &b.insts { - insts.push(compile_inst(p, inst, &slots)); + insts.push(compile_inst(p, inst, &slots, ®exes)); } blocks.push(CBlock { insts, @@ -1432,8 +1454,59 @@ fn sl(slots: &HashMap, v: Value) -> usize { slots[&v.0] as usize } -fn compile_inst(p: &Program, inst: &Inst, slots: &HashMap) -> InstFn { +fn compile_inst( + p: &Program, + inst: &Inst, + slots: &HashMap, + regexes: &[std::rc::Rc], +) -> InstFn { match inst.clone() { + Inst::ReMatch { re, dst, a } => { + let rx = regexes[re as usize].clone(); + let (dst, a) = (sl(slots, dst), sl(slots, a)); + Box::new(move |ctx| { + let s = ctx.arena.get(as_str(ctx.regs[a])); + let m = rx.is_match(s); + ctx.regs[dst] = RegVal::I1(m); + Ok(()) + }) + } + Inst::ReExtract { re, group, dst, a } => { + let rx = regexes[re as usize].clone(); + let (dst, a) = (sl(slots, dst), sl(slots, a)); + Box::new(move |ctx| { + // No match / non-participating group -> '' (wave-B pins). + let out = { + let s = ctx.arena.get(as_str(ctx.regs[a])); + rx.captures(s) + .and_then(|c| c.get(group as usize)) + .map(|m| m.as_str().to_string()) + .unwrap_or_default() + }; + ctx.regs[dst] = RegVal::Str(ctx.arena.push_str(&out)); + Ok(()) + }) + } + Inst::ReReplace { re, global, dst, a } => { + let rx = regexes[re as usize].clone(); + let template = p.regexes[re as usize] + .rewrite + .clone() + .expect("verified: rereplace has a template"); + let (dst, a) = (sl(slots, dst), sl(slots, a)); + Box::new(move |ctx| { + let out = { + let s = ctx.arena.get(as_str(ctx.regs[a])); + if global { + rx.replace_all(s, template.as_str()).into_owned() + } else { + rx.replace(s, template.as_str()).into_owned() + } + }; + ctx.regs[dst] = RegVal::Str(ctx.arena.push_str(&out)); + Ok(()) + }) + } Inst::Const { dst, lit } => { let dst = sl(slots, dst); match lit { diff --git a/src/specializer/exec/tests.rs b/src/specializer/exec/tests.rs index 45a98d4..7988624 100644 --- a/src/specializer/exec/tests.rs +++ b/src/specializer/exec/tests.rs @@ -188,7 +188,9 @@ entry: ); match compile(&p, vec![]) { Err(CompileError::Verify(errs)) => assert!(!errs.is_empty()), - Err(CompileError::Static(m)) => panic!("wrong error kind: {m}"), + Err(CompileError::Static(m)) | Err(CompileError::Regex(m)) => { + panic!("wrong error kind: {m}") + } Ok(_) => panic!("compile accepted an unverified program"), } } @@ -224,7 +226,9 @@ fn rejects_mismatched_statics() { Err(CompileError::Static(msg)) => { assert!(msg.contains(needle), "expected '{needle}' in '{msg}'") } - Err(CompileError::Verify(_)) => panic!("fixture failed verify?"), + Err(CompileError::Verify(_)) | Err(CompileError::Regex(_)) => { + panic!("fixture failed verify?") + } Ok(_) => panic!("compile accepted bad statics (wanted '{needle}')"), } } @@ -451,6 +455,7 @@ fn sparse_value_ids_use_dense_register_slots() { use super::super::ir::{Block, Col, ColTy, Inst, Lit, Program, Term, Ty, Value}; let p = Program { statics: vec![], + regexes: vec![], name: "sparse".into(), in_cols: vec![], out_cols: vec![Col { diff --git a/src/specializer/fold.rs b/src/specializer/fold.rs index 4a4b051..d1036a6 100644 --- a/src/specializer/fold.rs +++ b/src/specializer/fold.rs @@ -81,6 +81,20 @@ pub fn fold(e: SExpr) -> SExpr { a: Box::new(a), }) } + SKind::ReMatch { re, a } => e(SKind::ReMatch { + re, + a: Box::new(fold(*a)), + }), + SKind::ReExtract { re, group, a } => e(SKind::ReExtract { + re, + group, + a: Box::new(fold(*a)), + }), + SKind::ReReplace { re, global, a } => e(SKind::ReReplace { + re, + global, + a: Box::new(fold(*a)), + }), SKind::Like { ci, a, p, esc } => { let a = fold(*a); let p = fold(*p); diff --git a/src/specializer/frontend.rs b/src/specializer/frontend.rs index 4ca2719..428f066 100644 --- a/src/specializer/frontend.rs +++ b/src/specializer/frontend.rs @@ -67,7 +67,7 @@ pub fn frontend( this_name: &str, in_cols: &[Col], statics: &[StaticTable], -) -> Result<(Rel, Vec, Vec), PrepareError> { +) -> Result<(Rel, Vec, Vec, Vec), PrepareError> { // GenericDialect, not DuckDbDialect: measured as a strict superset for // the forms we serve (adds ^@, * ILIKE, * RENAME) and matches the oracle // path in datafusion/plan.rs — see pins-wave5/sqlparser-spike.json. @@ -147,13 +147,31 @@ pub fn frontend( }; for item in &select.projection { match item { - SelectItem::UnnamedExpr(e) => push_item( - &mut out_cols, - &mut exprs, - default_name(e), - fold(binder.expr(e)?), - )?, + SelectItem::UnnamedExpr(e) => { + // COLUMNS('re') expands like a filtered star, keeping the + // bare column names (wave-B pins). + if let Some(cols) = binder.expand_columns_item(e)? { + for (name, ex) in cols { + push_item(&mut out_cols, &mut exprs, name, ex)?; + } + } else { + push_item( + &mut out_cols, + &mut exprs, + default_name(e), + fold(binder.expr(e)?), + )? + } + } SelectItem::ExprWithAlias { expr, alias } => { + // An alias on COLUMNS stamps EVERY expansion (duplicates + // feed the dedup rename — measured). + if let Some(cols) = binder.expand_columns_item(expr)? { + for (_, ex) in cols { + push_item(&mut out_cols, &mut exprs, alias.value.clone(), ex)?; + } + continue; + } let e = fold(binder.expr(expr)?); // Lateral aliases (wave-5 pins): later items and WHERE may // reference this alias; the real column still wins. @@ -212,6 +230,7 @@ pub fn frontend( }, joins, out_cols, + binder.regexes.into_inner(), )) } @@ -291,6 +310,7 @@ fn bind_from<'a>( }) .collect(), bound_aliases: std::cell::RefCell::new(Vec::new()), + regexes: std::cell::RefCell::new(Vec::new()), }; let mut specs: Vec = Vec::new(); @@ -841,6 +861,45 @@ struct Binder<'a> { /// Aliases already bound this pass, in SELECT order (frontend() fills /// this as it walks the projection; RefCell keeps `expr(&self)` intact). bound_aliases: std::cell::RefCell>, + /// Program regex table under construction (wave-B); indices are baked + /// into ReMatch/ReExtract/ReReplace nodes. + regexes: std::cell::RefCell>, +} + +/// The bound subject of a regex op must be VARCHAR (no implicit casts — +/// wave-B pins; DuckDB binder errors name the function). +fn str_only(name: &str, e: SExpr) -> Result { + if e.ty != Ty::Str { + return Err(PrepareError::Bind(format!( + "no function matches {name}({})", + e.ty.name() + ))); + } + Ok(e) +} + +/// `''` for non-NULL subjects, NULL for NULL ones (the pinned NULL-group +/// result of regexp_extract). +fn empty_for_nonnull(subject: SExpr) -> SExpr { + if !subject.nullable { + return lit_str(""); + } + let is_null = SExpr { + kind: SKind::IsNull { + negated: false, + inner: Box::new(subject), + }, + ty: Ty::I1, + nullable: false, + }; + SExpr { + kind: SKind::Case { + arms: vec![(is_null, null_of(Ty::Str))], + default: Some(Box::new(lit_str(""))), + }, + ty: Ty::Str, + nullable: true, + } } fn math1_node(op: NumOp1, inner: SExpr) -> SExpr { @@ -923,6 +982,9 @@ fn like_match(s: &str, p: &str, ci: bool) -> bool { enum StarFilter { Like { ci: bool, neg: bool }, Glob, + /// Wave-B pins: positive = unanchored RE2 SEARCH over names; NOT = + /// NOT full-match — independent predicates, never complements. + Similar { neg: bool }, } /// Decoded star filter + any EXCLUDE entries the rewrite absorbed into the @@ -940,6 +1002,8 @@ fn decode_star_filter(pattern: &str) -> DecodedFilter { ("NL:", StarFilter::Like { ci: false, neg: true }), ("NI:", StarFilter::Like { ci: true, neg: true }), ("G:", StarFilter::Glob), + ("S:", StarFilter::Similar { neg: false }), + ("NS:", StarFilter::Similar { neg: true }), ] { let Some(body) = rest.strip_prefix(code) else { continue; @@ -1072,13 +1136,13 @@ impl Binder<'_> { /// f64 side promotes all sides. Numeric-with-string/bool mixing has /// exec-time cast semantics we don't model — clean-unsupported. fn unify_family(&self, exprs: &[&SqlExpr]) -> Result, PrepareError> { - let (mut any_f64, mut any_num, mut any_other) = (false, false, false); + let (mut any_f64, mut any_num) = (false, false); for e in exprs { if let Some(b) = self.expr_or_null(e)? { match b.ty { Ty::F64 => (any_f64, any_num) = (true, true), Ty::I64 => any_num = true, - Ty::Str | Ty::I1 => any_other = true, + Ty::Str | Ty::I1 => {} } } } @@ -1394,9 +1458,18 @@ impl Binder<'_> { // Name filter LAST in our pipeline but semantically after EXCLUDE // only (REPLACE/RENAME + filter were rejected above). if let Some(DecodedFilter { op, pat, .. }) = &filter { - cols.retain(|(_, cn, _)| match op { - StarFilter::Like { ci, neg } => like_match(cn, pat, *ci) != *neg, - StarFilter::Glob => super::exec::interp::duck_glob(cn, pat), + let similar = match op { + // Positive SIMILAR TO searches names UNANCHORED; the NOT + // form negates a FULL match — measured to not be + // complements ('a.*': "Weird Name" is in BOTH results). + StarFilter::Similar { neg } => Some(self.name_regex(pat, *neg)?), + _ => None, + }; + cols.retain(|(_, cn, _)| match (op, &similar) { + (StarFilter::Like { ci, neg }, _) => like_match(cn, pat, *ci) != *neg, + (StarFilter::Glob, _) => super::exec::interp::duck_glob(cn, pat), + (StarFilter::Similar { neg }, Some(rx)) => rx.is_match(cn) != *neg, + (StarFilter::Similar { .. }, None) => unreachable!(), }); if cols.is_empty() { return Err(PrepareError::Bind(format!( @@ -1439,6 +1512,18 @@ impl Binder<'_> { // Re-associate: in-order traversal of the parsed run recovers // source order, then left-fold. User parens are Nested nodes, // which the flatten treats as leaves. + // ~ / !~ are FULL regex match in DuckDB (measured: the binder + // error names regexp_full_match — NOT the Postgres search). + SqlExpr::BinaryOp { + left, + op: BinaryOperator::PGRegexMatch, + right, + } => self.regex_full_predicate("~", left, right, false), + SqlExpr::BinaryOp { + left, + op: BinaryOperator::PGRegexNotMatch, + right, + } => self.regex_full_predicate("!~", left, right, true), SqlExpr::BinaryOp { op, .. } if is_flat_bitop(op) => { let (mut ops, mut operands) = (Vec::new(), Vec::new()); flatten_bitops(e, &mut ops, &mut operands); @@ -1702,9 +1787,20 @@ impl Binder<'_> { like }) } - SqlExpr::SimilarTo { .. } => Err(unsup( - "SIMILAR TO (DuckDB binds it to regexp_full_match, not SQL wildcards)", - )), + // SIMILAR TO on VALUES is exactly regexp_full_match on the RAW + // pattern — DuckDB translates NO wildcards ('h%o' is literal %, + // 'h.llo' is a live regex dot). Wave-B pins. + SqlExpr::SimilarTo { + negated, + expr, + pattern, + escape_char, + } => { + if escape_char.is_some() { + return Err(unsup("Custom escape in SIMILAR TO (DuckDB: not implemented)")); + } + self.regex_full_predicate("SIMILAR TO", expr, pattern, *negated) + } // Bracket syntax s[i] / s[a:b] — exactly array_extract / // array_slice in DuckDB (one shared implementation, measured: // pins-wave5/{subscripts-extended,slices}.json). @@ -1748,6 +1844,218 @@ impl Binder<'_> { } } + /// Compile a column-NAME regex for star filters / COLUMNS: unanchored + /// for the positive search, `\A..\z`-wrapped for the NOT-full-match + /// form. Bind-time only — never reaches the exec regex table. + fn name_regex(&self, pat: &str, full: bool) -> Result { + let translated = super::retrans::translate_pattern(pat)?; + let pattern = if full { + format!("\\A(?:{translated})\\z") + } else { + translated + }; + regex::RegexBuilder::new(&pattern) + .octal(true) + .build() + .map_err(|e| { + PrepareError::Bind(format!("Failed to compile regex \"{pat}\": {e}")) + }) + } + + /// Expand a `COLUMNS('re')` / `COLUMNS(*)` SELECT item (wave-B): + /// unanchored RE2 search over declared-case names, table-declaration + /// order. Returns None when `e` is not a COLUMNS call; expression + /// forms (COLUMNS(..) + 1) stay unsupported upstream. + fn expand_columns_item( + &self, + e: &SqlExpr, + ) -> Result>, PrepareError> { + use sqlparser::ast::{FunctionArg, FunctionArgExpr, FunctionArguments}; + let SqlExpr::Function(f) = e else { + return Ok(None); + }; + if !f.name.to_string().eq_ignore_ascii_case("columns") { + return Ok(None); + } + let FunctionArguments::List(list) = &f.args else { + return Ok(None); + }; + let all = self.expand_star(None, &sqlparser::ast::WildcardAdditionalOptions::default())?; + match &list.args[..] { + [FunctionArg::Unnamed(FunctionArgExpr::Wildcard)] => Ok(Some(all)), + [FunctionArg::Unnamed(FunctionArgExpr::Expr(p))] => { + let Some(bp) = self.expr_or_null(p)? else { + return Err(PrepareError::Bind( + "COLUMNS requires a constant pattern".into(), + )); + }; + let SKind::Lit(Lit::Str(pat)) = bp.kind else { + return Err(unsup("COLUMNS with a non-constant or list argument")); + }; + let rx = self.name_regex(&pat, false)?; + let cols: Vec<(String, SExpr)> = + all.into_iter().filter(|(n, _)| rx.is_match(n)).collect(); + if cols.is_empty() { + return Err(PrepareError::Bind(format!( + "No matching columns found that match regex \"{pat}\"" + ))); + } + Ok(Some(cols)) + } + _ => Err(unsup("COLUMNS argument form")), + } + } + + /// `~` / `!~` / SIMILAR TO: FULL match on the raw pattern (measured — + /// `~` is regexp_full_match in DuckDB, NOT the Postgres search). + fn regex_full_predicate( + &self, + name: &str, + subject: &SqlExpr, + pattern: &SqlExpr, + negated: bool, + ) -> Result { + let Some(bs) = self.expr_or_null(subject)? else { + return Ok(null_of(Ty::I1)); + }; + let bs = str_only(name, bs)?; + let Some(re) = self.regex_pattern(pattern, super::retrans::ReOptions::default(), true)? + else { + return Ok(null_of(Ty::I1)); + }; + let nullable = bs.nullable; + let m = SExpr { + kind: SKind::ReMatch { + re, + a: Box::new(bs), + }, + ty: Ty::I1, + nullable, + }; + Ok(if negated { + SExpr { + kind: SKind::Not(Box::new(m)), + ty: Ty::I1, + nullable, + } + } else { + m + }) + } + + /// Bind a regex options argument: absent -> defaults; otherwise a + /// non-NULL constant string ("must not be NULL" / "must be a constant" + /// are the pinned texts). + fn regex_options( + &self, + opts: Option<&SqlExpr>, + allow_g: bool, + ) -> Result { + match self.regex_options_raw(opts, allow_g)? { + Some(o) => Ok(o), + None => Err(PrepareError::Bind( + "Regex options field must not be NULL".into(), + )), + } + } + + /// regexp_replace's variant: a NULL options argument makes the whole + /// RESULT NULL (pinned asymmetry) — None here means "return NULL". + fn regex_options_nullable( + &self, + opts: Option<&SqlExpr>, + ) -> Result, PrepareError> { + self.regex_options_raw(opts, true) + } + + fn regex_options_raw( + &self, + opts: Option<&SqlExpr>, + allow_g: bool, + ) -> Result, PrepareError> { + let Some(o) = opts else { + return Ok(Some(super::retrans::ReOptions::default())); + }; + match self.expr_or_null(o)? { + None => Ok(None), + Some(b) => match b.kind { + SKind::Lit(Lit::Str(s)) => { + super::retrans::parse_options(&s, allow_g).map(Some) + } + _ => Err(PrepareError::Bind( + "Regex options field must be a constant".into(), + )), + }, + } + } + + /// Bind a constant regex pattern into the program regex table: + /// translate (retrans), optionally full-match anchor, and COMPILE NOW + /// so invalid patterns error at prepare (pinned bind-time eagerness). + /// `Ok(None)` = the pattern was a NULL literal (result is NULL). + fn regex_pattern( + &self, + p: &SqlExpr, + o: super::retrans::ReOptions, + full: bool, + ) -> Result, PrepareError> { + Ok(self.regex_pattern_counted(p, o, full)?.map(|(re, _)| re)) + } + + fn regex_pattern_counted( + &self, + p: &SqlExpr, + o: super::retrans::ReOptions, + full: bool, + ) -> Result, PrepareError> { + let Some(bp) = self.expr_or_null(p)? else { + return Ok(None); + }; + let SKind::Lit(Lit::Str(raw)) = bp.kind else { + if bp.ty != Ty::Str { + return Err(PrepareError::Bind(format!( + "no function matches a regex with a {} pattern", + bp.ty.name() + ))); + } + // Column patterns compile per row in DuckDB; the engine model + // is prepare-time compilation only. + return Err(unsup("non-constant regex pattern (compiled at prepare in v0)")); + }; + let translated = if o.literal { + regex::escape(&raw) + } else { + super::retrans::translate_pattern(&raw)? + }; + let pattern = if full { + format!("\\A(?:{translated})\\z") + } else { + translated + }; + let rx = regex::RegexBuilder::new(&pattern) + .case_insensitive(o.case_insensitive) + .dot_matches_new_line(o.dotall) + .octal(true) + .build() + .map_err(|e| PrepareError::Bind(format!("Invalid Input Error: {e}")))?; + let group_count = rx.captures_len() - 1; + let spec = super::ir::ReSpec { + pattern, + ci: o.case_insensitive, + dotall: o.dotall, + rewrite: None, + }; + let mut v = self.regexes.borrow_mut(); + // Reuse identical rewrite-less entries (star filters + repeated + // predicates); replace ops mutate `rewrite` after, so only share + // entries that still have none. + if let Some(i) = v.iter().position(|r| *r == spec) { + return Ok(Some((i as u32, group_count))); + } + v.push(spec); + Ok(Some(((v.len() - 1) as u32, group_count))) + } + /// s[i] / array_extract / list_extract on a bound VARCHAR subject: /// exec handles negatives (len+1+i), 0/out-of-range -> '' and the /// runtime +-2^32 offset trap (pins-wave5/subscripts-extended.json). @@ -1802,7 +2110,7 @@ impl Binder<'_> { bs.ty.name() ))); } - let mut bind_bound = |e: Option<&SqlExpr>, open: i64| -> Result, PrepareError> { + let bind_bound = |e: Option<&SqlExpr>, open: i64| -> Result, PrepareError> { match e { None => Ok(Some(lit_i64(open))), Some(e) => self.expr_or_null(e), @@ -2281,11 +2589,7 @@ impl Binder<'_> { // guard with a CASE unless the divisor is a provably non-zero // literal. The idiv/irem traps stay reachable only for MIN op -1, // where DuckDB traps too. Float % is IEEE (x % 0.0 = NaN), no guard. - let needs_guard = match (op, ty) { - (ArithOp::Rem, Ty::I64) => true, - (ArithOp::IDiv, _) => true, - _ => false, - }; + let needs_guard = matches!((op, ty), (ArithOp::Rem, Ty::I64) | (ArithOp::IDiv, _)); let nonzero_lit = matches!(b.kind, SKind::Lit(Lit::I64(n)) if n != 0) || matches!(b.kind, SKind::Lit(Lit::F64(x)) if x != 0.0); if needs_guard && !nonzero_lit { @@ -3253,9 +3557,133 @@ impl Binder<'_> { | "first" | "last" | "any_value" => Err(unsup(format!( "aggregate function {name} (no aggregation in v0)" ))), - "regexp_matches" | "regexp_extract" | "regexp_full_match" | "regexp_replace" - | "regexp_split_to_array" => Err(unsup(format!( - "function {name} (RE2 regex semantics, not in v0)" + // Wave-B regexp family (pins: 2026-07-27-waveB-regexp-pins.md). + "regexp_matches" | "regexp_full_match" => { + let (s, p, opts) = match args[..] { + [s, p] => (s, p, None), + [s, p, o] => (s, p, Some(o)), + _ => { + return Err(PrepareError::Bind(format!( + "{name} takes 2 or 3 arguments" + ))) + } + }; + let o = self.regex_options(opts, false)?; + let Some(bs) = self.expr_or_null(s)? else { + return Ok(null_of(Ty::I1)); + }; + let full = name == "regexp_full_match"; + let bs = str_only(&name, bs)?; + match self.regex_pattern(p, o, full)? { + None => Ok(null_of(Ty::I1)), + Some(re) => Ok(SExpr { + nullable: bs.nullable, + kind: SKind::ReMatch { + re, + a: Box::new(bs), + }, + ty: Ty::I1, + }), + } + } + "regexp_extract" => { + let (s, p, group, opts) = match args[..] { + [s, p] => (s, p, None, None), + [s, p, g] => (s, p, Some(g), None), + [s, p, g, o] => (s, p, Some(g), Some(o)), + _ => { + return Err(PrepareError::Bind(format!( + "{name} takes 2 to 4 arguments" + ))) + } + }; + let o = self.regex_options(opts, false)?; + let Some(bs) = self.expr_or_null(s)? else { + return Ok(null_of(Ty::Str)); + }; + let bs = str_only(&name, bs)?; + // Group index: constant, flat 0..9 range check unrelated to + // the pattern; NULL group -> '' for non-NULL subjects. + let group = match group { + None => 0u32, + Some(g) => match self.expr_or_null(g)? { + None => return Ok(empty_for_nonnull(bs)), + Some(bg) => match bg.kind { + SKind::Lit(Lit::I64(n)) if (0..=9).contains(&n) => n as u32, + SKind::Lit(Lit::I64(_)) => { + return Err(PrepareError::Bind( + "Group index must be between 0 and 9!".into(), + )) + } + _ => { + return Err(unsup( + "non-constant regexp_extract group index", + )) + } + }, + }, + }; + match self.regex_pattern(p, o, false)? { + None => Ok(null_of(Ty::Str)), + Some(re) => Ok(SExpr { + nullable: bs.nullable, + kind: SKind::ReExtract { + re, + group, + a: Box::new(bs), + }, + ty: Ty::Str, + }), + } + } + "regexp_replace" => { + let (s, p, r, opts) = match args[..] { + [s, p, r] => (s, p, r, None), + [s, p, r, o] => (s, p, r, Some(o)), + _ => { + return Err(PrepareError::Bind(format!( + "{name} takes 3 or 4 arguments" + ))) + } + }; + // Pinned asymmetry: for regexp_replace ANY NULL argument + // (including the options string) -> NULL result. + let Some(o) = self.regex_options_nullable(opts)? else { + return Ok(null_of(Ty::Str)); + }; + let Some(bs) = self.expr_or_null(s)? else { + return Ok(null_of(Ty::Str)); + }; + let bs = str_only(&name, bs)?; + let Some(br) = self.expr_or_null(r)? else { + return Ok(null_of(Ty::Str)); + }; + let SKind::Lit(Lit::Str(rw)) = br.kind else { + return Err(unsup("non-constant regexp_replace replacement")); + }; + let Some((re, group_count)) = self.regex_pattern_counted(p, o, false)? else { + return Ok(null_of(Ty::Str)); + }; + match super::retrans::translate_rewrite(&rw, group_count, o.global) { + // Invalid rewrites never error (measured RE2 quirks). + super::retrans::Rewrite::Identity => Ok(bs), + super::retrans::Rewrite::Template(t) + | super::retrans::Rewrite::ConsumeWithPrefix(t) => { + self.regexes.borrow_mut()[re as usize].rewrite = Some(t); + Ok(SExpr { + nullable: bs.nullable, + kind: SKind::ReReplace { + re, + global: o.global, + a: Box::new(bs), + }, + ty: Ty::Str, + }) + } + } + } + "regexp_split_to_array" | "regexp_extract_all" => Err(unsup(format!( + "function {name} (list-valued — non-scalar in v0)" ))), "reverse" => Err(unsup( "function reverse (grapheme-cluster semantics — measured UAX-29 \ diff --git a/src/specializer/ir/gen.rs b/src/specializer/ir/gen.rs index e9b8df4..0a4874c 100644 --- a/src/specializer/ir/gen.rs +++ b/src/specializer/ir/gen.rs @@ -754,6 +754,7 @@ pub fn gen_program(seed: u64) -> Program { Program { statics, + regexes: Vec::new(), name: "fuzzed".to_string(), in_cols, out_cols, diff --git a/src/specializer/ir/mod.rs b/src/specializer/ir/mod.rs index db9a32d..e9af065 100644 --- a/src/specializer/ir/mod.rs +++ b/src/specializer/ir/mod.rs @@ -767,6 +767,29 @@ pub enum Inst { flag: Value, dst: Value, }, + /// Match `a` against program regex `re` (full-match forms are + /// pre-anchored in the ReSpec pattern at bind) -> I1. TOTAL. + ReMatch { + re: u32, + dst: Value, + a: Value, + }, + /// Leftmost-search extract of capture `group` (0 = whole match); no + /// match / non-participating group -> '' (never NULL; wave-B pins). + ReExtract { + re: u32, + group: u32, + dst: Value, + a: Value, + }, + /// First-match (or `global`) replace using the ReSpec's rewrite + /// template. TOTAL — invalid-rewrite quirks were resolved at bind. + ReReplace { + re: u32, + global: bool, + dst: Value, + a: Value, + }, } #[derive(Clone, PartialEq, Debug)] @@ -802,12 +825,25 @@ pub struct Block { #[derive(Clone, PartialEq, Debug)] pub struct Program { pub statics: Vec, + /// Prepare-time-compiled regexes (wave-B): patterns already translated + /// to rust-regex syntax (retrans.rs), full-match forms pre-anchored. + pub regexes: Vec, pub name: String, pub in_cols: Vec, pub out_cols: Vec, pub blocks: Vec, } +/// One entry of [`Program::regexes`]; `rewrite` is a rust replacement +/// template, present only for replace ops. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct ReSpec { + pub pattern: String, + pub ci: bool, + pub dotall: bool, + pub rewrite: Option, +} + impl Inst { /// Values this instruction defines, in definition order. pub fn dsts(&self) -> Vec { @@ -837,6 +873,9 @@ impl Inst { | Inst::Ssubstr { dst, .. } | Inst::Num1 { dst, .. } | Inst::Load { dst, .. } + | Inst::ReMatch { dst, .. } + | Inst::ReExtract { dst, .. } + | Inst::ReReplace { dst, .. } | Inst::Sload { dst, .. } => vec![*dst], Inst::StoiOpt { flag, dst, .. } | Inst::StofOpt { flag, dst, .. } @@ -887,6 +926,9 @@ impl Inst { | Inst::SLen { dst, a, .. } | Inst::Sord { dst, a, .. } | Inst::Num1 { dst, a, .. } + | Inst::ReMatch { dst, a, .. } + | Inst::ReExtract { dst, a, .. } + | Inst::ReReplace { dst, a, .. } | Inst::Not { dst, a } => { *dst = m(*dst); *a = m(*a); diff --git a/src/specializer/ir/parse.rs b/src/specializer/ir/parse.rs index d4e1150..e9f1390 100644 --- a/src/specializer/ir/parse.rs +++ b/src/specializer/ir/parse.rs @@ -475,6 +475,57 @@ impl Parser { self.expect(Tok::Colon)?; statics.push(self.static_ty()?); } + let mut regexes = Vec::new(); + while matches!(self.peek(), Tok::Ident(s) if s == "regex") { + self.bump(); + self.expect(Tok::At)?; + let idx = self.int_literal("regex index")?; + if idx != regexes.len() as i64 { + return Err(self.err(format!( + "regex ids must be dense and in order: expected @{}, found @{idx}", + regexes.len() + ))); + } + self.expect(Tok::Colon)?; + let pattern = match self.bump() { + Tok::Str(s) => s, + other => { + return Err(self.err(format!( + "expected a regex pattern string, found {}", + other.show() + ))) + } + }; + let (mut ci, mut dotall, mut rewrite) = (false, false, None); + loop { + if matches!(self.peek(), Tok::Ident(s) if s == "ci") { + self.bump(); + ci = true; + } else if matches!(self.peek(), Tok::Ident(s) if s == "dotall") { + self.bump(); + dotall = true; + } else if matches!(self.peek(), Tok::Ident(s) if s == "rewrite") { + self.bump(); + rewrite = Some(match self.bump() { + Tok::Str(s) => s, + other => { + return Err(self.err(format!( + "expected a rewrite template string, found {}", + other.show() + ))) + } + }); + } else { + break; + } + } + regexes.push(super::ReSpec { + pattern, + ci, + dotall, + rewrite, + }); + } self.keyword("fn")?; let name = self.ident("function name")?; @@ -548,6 +599,7 @@ impl Parser { Ok(Program { statics, + regexes, name, in_cols, out_cols, @@ -1276,6 +1328,46 @@ impl Parser { dst: def!(1), } } + "rematch" => { + want_dsts(1, self)?; + self.expect(Tok::At)?; + let re = self.int_literal("regex index")? as u32; + self.expect(Tok::Comma)?; + let a = self.use_value()?; + Inst::ReMatch { + re, + dst: def!(0), + a, + } + } + "reextract" => { + want_dsts(1, self)?; + self.expect(Tok::At)?; + let re = self.int_literal("regex index")? as u32; + self.expect(Tok::Comma)?; + let group = self.int_literal("group index")? as u32; + self.expect(Tok::Comma)?; + let a = self.use_value()?; + Inst::ReExtract { + re, + group, + dst: def!(0), + a, + } + } + "rereplace" | "rereplace.g" => { + want_dsts(1, self)?; + self.expect(Tok::At)?; + let re = self.int_literal("regex index")? as u32; + self.expect(Tok::Comma)?; + let a = self.use_value()?; + Inst::ReReplace { + re, + global: opcode == "rereplace.g", + dst: def!(0), + a, + } + } other => return Err(self.err(format!("unknown opcode '{other}'"))), }; Ok(inst) diff --git a/src/specializer/ir/print.rs b/src/specializer/ir/print.rs index 0a9d2ae..ff140d9 100644 --- a/src/specializer/ir/print.rs +++ b/src/specializer/ir/print.rs @@ -21,7 +21,20 @@ pub fn print(p: &Program) -> String { } } } - if !p.statics.is_empty() { + for (i, re) in p.regexes.iter().enumerate() { + let _ = write!(s, "regex @{i}: {}", quote(&re.pattern)); + if re.ci { + s.push_str(" ci"); + } + if re.dotall { + s.push_str(" dotall"); + } + if let Some(rw) = &re.rewrite { + let _ = write!(s, " rewrite {}", quote(rw)); + } + s.push('\n'); + } + if !p.statics.is_empty() || !p.regexes.is_empty() { s.push('\n'); } let _ = writeln!( @@ -206,6 +219,16 @@ fn print_inst(s: &mut String, p: &Program, inst: &Inst) { Inst::SloadOpt { static_id, .. } => { let _ = write!(s, "sload.opt @{static_id}"); } + Inst::ReMatch { re, a, .. } => { + let _ = write!(s, "rematch @{re}, {}", val(*a)); + } + Inst::ReExtract { re, group, a, .. } => { + let _ = write!(s, "reextract @{re}, {group}, {}", val(*a)); + } + Inst::ReReplace { re, global, a, .. } => { + let opcode = if *global { "rereplace.g" } else { "rereplace" }; + let _ = write!(s, "{opcode} @{re}, {}", val(*a)); + } } } diff --git a/src/specializer/ir/tests.rs b/src/specializer/ir/tests.rs index 29662b3..9d26f67 100644 --- a/src/specializer/ir/tests.rs +++ b/src/specializer/ir/tests.rs @@ -150,6 +150,7 @@ entry: use super::{BinOp, Block, Col, ColTy, Inst, Program, Term, Ty, Value}; let p = Program { statics: vec![], + regexes: vec![], name: "f".into(), in_cols: vec![Col { name: "a".into(), @@ -537,6 +538,7 @@ fn api_program(statics: Vec, name: &str, blocks: Vec Vec<(Value, Ty)> { Inst::Num1 { op, dst, .. } => vec![(*dst, op.sig())], Inst::Str2 { op, dst, .. } => vec![(*dst, op.result_ty())], Inst::SLen { dst, .. } | Inst::Sord { dst, .. } => vec![(*dst, Ty::I64)], - Inst::Slike { dst, .. } => vec![(*dst, Ty::I1)], + Inst::Slike { dst, .. } | Inst::ReMatch { dst, .. } => vec![(*dst, Ty::I1)], + Inst::ReExtract { dst, .. } | Inst::ReReplace { dst, .. } => vec![(*dst, Ty::Str)], Inst::Round2f { dst, .. } => vec![(*dst, Ty::F64)], Inst::Round2i { dst, .. } => vec![(*dst, Ty::I64)], Inst::StoiOpt { flag, dst, .. } => vec![(*flag, Ty::I1), (*dst, Ty::I64)], @@ -366,6 +367,26 @@ fn check_block( Inst::StoiOpt { a, .. } | Inst::StofOpt { a, .. } => { want(&in_scope, def_types, *a, Ty::Str, "operand", bi, i, errs) } + Inst::ReMatch { re, a, .. } + | Inst::ReExtract { re, a, .. } + | Inst::ReReplace { re, a, .. } => { + want(&in_scope, def_types, *a, Ty::Str, "operand", bi, i, errs); + if *re as usize >= p.regexes.len() { + err(errs, Some(bi), i, format!("regex @{re} out of range")); + } + if matches!(inst, Inst::ReReplace { .. }) + && p.regexes + .get(*re as usize) + .is_some_and(|r| r.rewrite.is_none()) + { + err( + errs, + Some(bi), + i, + format!("rereplace on regex @{re} without a rewrite template"), + ); + } + } Inst::Sconcat { a, b: rhs, .. } | Inst::Strim { a, chars: rhs, .. } | Inst::Str2 { a, b: rhs, .. } => { diff --git a/src/specializer/lower.rs b/src/specializer/lower.rs index afe8195..8ab5814 100644 --- a/src/specializer/lower.rs +++ b/src/specializer/lower.rs @@ -39,6 +39,7 @@ pub fn lower( catalog: &[StaticTable], in_cols: &[Col], out_cols: Vec, + regexes: Vec, name: &str, ) -> Result { let (exprs, filter_pred) = match rel { @@ -143,7 +144,7 @@ pub fn lower( }) .collect(); - fb.finish(name, statics, in_cols, out_cols) + fb.finish(name, statics, in_cols, out_cols, regexes) } /// A value in the null-lane representation: payload + optional validity. @@ -238,6 +239,7 @@ impl<'a> FB<'a> { statics: Vec, in_cols: &[Col], out_cols: Vec, + regexes: Vec, ) -> Result { let mut blocks = Vec::with_capacity(self.blocks.len()); for (i, pb) in self.blocks.into_iter().enumerate() { @@ -252,6 +254,7 @@ impl<'a> FB<'a> { } Ok(Program { statics, + regexes, name: name.to_string(), in_cols: in_cols.to_vec(), out_cols, @@ -890,6 +893,47 @@ impl<'a> FB<'a> { val: dst, }) } + SKind::ReMatch { re, a } => { + let l = self.emit(a, live)?; + let dst = self.fresh(); + self.inst(Inst::ReMatch { + re: *re, + dst, + a: l.val, + }); + Ok(Lane { + flag: l.flag, + val: dst, + }) + } + SKind::ReExtract { re, group, a } => { + let l = self.emit(a, live)?; + let dst = self.fresh(); + self.inst(Inst::ReExtract { + re: *re, + group: *group, + dst, + a: l.val, + }); + Ok(Lane { + flag: l.flag, + val: dst, + }) + } + SKind::ReReplace { re, global, a } => { + let l = self.emit(a, live)?; + let dst = self.fresh(); + self.inst(Inst::ReReplace { + re: *re, + global: *global, + dst, + a: l.val, + }); + Ok(Lane { + flag: l.flag, + val: dst, + }) + } SKind::Concat { a, b } => { let la = self.emit(a, live)?; live.push((la, Ty::Str)); diff --git a/src/specializer/mod.rs b/src/specializer/mod.rs index 2986b60..78fb16b 100644 --- a/src/specializer/mod.rs +++ b/src/specializer/mod.rs @@ -14,6 +14,7 @@ pub mod frontend; pub mod ir; pub mod lower; pub mod plan; +mod retrans; mod rewrite; #[cfg(test)] @@ -49,8 +50,8 @@ pub fn prepare( in_cols: &[ir::Col], statics: &[plan::StaticTable], ) -> Result { - let (rel, joins, out_cols) = frontend::frontend(sql, this_name, in_cols, statics)?; - let mut program = lower::lower(&rel, &joins, statics, in_cols, out_cols, "run")?; + let (rel, joins, out_cols, regexes) = frontend::frontend(sql, this_name, in_cols, statics)?; + let mut program = lower::lower(&rel, &joins, statics, in_cols, out_cols, regexes, "run")?; // Block-splitting lowerings mint ids out of text order; renumber so // every prepared program is exactly canonical (parse(print(p)) == p). ir::canonicalize(&mut program); diff --git a/src/specializer/plan.rs b/src/specializer/plan.rs index 4aecf57..8fd8ad9 100644 --- a/src/specializer/plan.rs +++ b/src/specializer/plan.rs @@ -179,6 +179,24 @@ pub enum SKind { bytes: bool, a: Box, }, + /// Regex match against program regex `re` (wave-B; full-match forms + /// pre-anchored in the ReSpec pattern at bind) -> I1. + ReMatch { + re: u32, + a: Box, + }, + /// Leftmost-search capture extract; no match -> '' (wave-B pins). + ReExtract { + re: u32, + group: u32, + a: Box, + }, + /// Regex replace using the ReSpec's rewrite template. + ReReplace { + re: u32, + global: bool, + a: Box, + }, /// LIKE/ILIKE (negation handled by a Not wrapper at bind). Like { ci: bool, diff --git a/src/specializer/retrans.rs b/src/specializer/retrans.rs new file mode 100644 index 0000000..96aee57 --- /dev/null +++ b/src/specializer/retrans.rs @@ -0,0 +1,333 @@ +//! DuckDB-RE2 -> rust-regex translation (wave-B pins, +//! pins-waveB/re2-vs-rust-regex.json). Raw pass-through is WRONG for any +//! pattern with a Perl class: RE2's \d \w \s \b are ASCII while rust-regex +//! defaults Unicode. The measured fix is this bind-time rewrite (verified +//! byte-for-byte on the differential battery) plus a reject list for +//! constructs that differ irreconcilably. Keep default Unicode mode — +//! `unicode(false)` was measured to BREAK (?i) folding parity. + +use super::frontend::PrepareError; + +fn unsup(what: impl Into) -> PrepareError { + PrepareError::Unsupported(what.into()) +} + +/// Parsed DuckDB regex options string (wave-B pins: parsed left-to-right, +/// whitespace skipped, LAST conflicting letter wins; `m`/`n`/`p` are +/// functional no-ops — multiline anchors exist only via inline (?m)). +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +pub struct ReOptions { + pub case_insensitive: bool, + pub dotall: bool, + pub literal: bool, + pub global: bool, +} + +/// `allow_g`: only regexp_replace accepts 'g' (pinned error otherwise). +pub fn parse_options(opts: &str, allow_g: bool) -> Result { + let mut o = ReOptions::default(); + for c in opts.chars() { + match c { + 'c' => o.case_insensitive = false, + 'i' => o.case_insensitive = true, + 'l' => o.literal = true, + 's' => o.dotall = true, + // Measured no-ops in 1.5.5 (do NOT wire to multiline/never_nl). + 'm' | 'n' | 'p' => o.dotall = false, + 'g' if allow_g => o.global = true, + 'g' => { + return Err(PrepareError::Bind( + "Option 'g' (global replace) is only valid for regexp_replace".into(), + )) + } + w if w.is_whitespace() => {} + other => { + return Err(PrepareError::Bind(format!( + "Unrecognized Regex option {other}" + ))) + } + } + } + Ok(o) +} + +/// Rewrite a DuckDB/RE2 pattern into a rust-regex pattern with identical +/// semantics, or reject constructs on the measured divergence list. +pub fn translate_pattern(p: &str) -> Result { + let b = p.as_bytes(); + let mut out = String::with_capacity(p.len() + 8); + let mut i = 0; + let mut in_class = false; + while i < b.len() { + let c = b[i]; + match c { + b'\\' => { + let Some(&n) = b.get(i + 1) else { + // Trailing backslash: both engines reject; pass through + // so the compile error surfaces uniformly. + out.push('\\'); + i += 1; + continue; + }; + match n { + // Perl classes: RE2 is ASCII, rust is Unicode (measured + // rewrites; negated forms must be Unicode-mode classes, + // and (?-u:..) cannot appear inside [...]). + b'd' => out.push_str(if in_class { "0-9" } else { "(?-u:\\d)" }), + b'w' => out.push_str(if in_class { + "0-9A-Za-z_" + } else { + "[0-9A-Za-z_]" + }), + b's' => out.push_str(if in_class { + "\\t\\n\\x0c\\r " + } else { + "[\\t\\n\\x0c\\r ]" + }), + b'D' | b'W' | b'S' if in_class => { + return Err(unsup( + "negated Perl class inside a character class (RE2/rust-regex \ + semantics differ; not translatable in place)", + )) + } + b'D' => out.push_str("[^0-9]"), + b'W' => out.push_str("[^0-9A-Za-z_]"), + b'S' => out.push_str("[^\\t\\n\\x0c\\r ]"), + b'b' if !in_class => out.push_str("(?-u:\\b)"), + // \B: DuckDB itself traps at runtime on non-ASCII (RE2's + // ASCII \B matches inside multibyte chars) — unservable. + b'B' => return Err(unsup("\\B in a regex (unservable RE2 byte semantics)")), + // \uXXXX: rust-only extension, DuckDB rejects. + b'u' | b'U' => { + return Err(unsup("\\u escape in a regex (not RE2 syntax)")) + } + // \Q...\E literal quoting: RE2-only, rust rejects. + b'Q' => return Err(unsup("\\Q...\\E literal quoting in a regex")), + other => { + out.push('\\'); + out.push(other as char); + } + } + i += 2; + } + b'[' if !in_class => { + in_class = true; + out.push('['); + i += 1; + // Consume the class-leading forms so a literal ']' first + // member ('[]a]' — legal, same meaning in both) survives. + if b.get(i) == Some(&b'^') { + out.push('^'); + i += 1; + } + if b.get(i) == Some(&b']') { + out.push(']'); + i += 1; + } + } + b']' if in_class => { + in_class = false; + out.push(']'); + i += 1; + } + b'(' if !in_class => { + // (?...) angle-bracket groups: DuckDB rejects them. + if p[i..].starts_with("(?<") && !p[i..].starts_with("(?<=") { + return Err(unsup("(?...) capture group (not RE2 syntax)")); + } + out.push('('); + i += 1; + } + b'{' if !in_class => { + // Repetition bound > 1000 is a DuckDB error; rust allows + // larger — reject past the pinned cap. + let close = b[i..].iter().position(|&x| x == b'}'); + if let Some(off) = close { + let body = &p[i + 1..i + off]; + let too_big = body + .split(',') + .filter_map(|s| s.trim().parse::().ok()) + .any(|n| n > 1000); + if too_big { + return Err(unsup(format!( + "regex repetition bound over 1000 ({{{body}}})" + ))); + } + } + out.push('{'); + i += 1; + } + b'*' | b'+' if !in_class => { + out.push(c as char); + i += 1; + // Stacked quantifiers (a*+, a++): DuckDB errors while rust + // silently reinterprets — a wrong-answer risk, reject. + if matches!(b.get(i), Some(b'*') | Some(b'+')) { + return Err(unsup("stacked regex quantifiers (RE2 rejects them)")); + } + } + _ => { + // Preserve the raw byte run (UTF-8 safe: we only split at + // ASCII metacharacters above). + let start = i; + i += 1; + while i < b.len() && !p.is_char_boundary(i) { + i += 1; + } + out.push_str(&p[start..i]); + } + } + } + // Duplicate group names: DuckDB's RE2 accepts them, rust rejects — + // rust's own compile error would misclassify, so pre-reject. + let mut names: Vec<&str> = Vec::new(); + let mut j = 0; + while let Some(pos) = p[j..].find("(?P<") { + let start = j + pos + 4; + if let Some(end) = p[start..].find('>') { + let name = &p[start..start + end]; + if names.contains(&name) { + return Err(unsup(format!( + "duplicate regex capture group name '{name}' (rust-regex rejects)" + ))); + } + names.push(name); + j = start + end; + } else { + break; + } + } + Ok(out) +} + +/// Translate a DuckDB replacement template (`\N` backrefs, literal `$`) +/// into a rust-regex template (`$N` refs, literal `\`), resolving the +/// pinned invalid-rewrite quirks at bind time. +pub enum Rewrite { + /// A valid rust template. + Template(String), + /// Out-of-range backref (or non-global bad escape): the WHOLE replace + /// is a silent no-op — bind the subject expression unchanged. + Identity, + /// Global bad escape: each match is CONSUMED and only the template + /// prefix before the bad escape is emitted (measured RE2 quirk). + ConsumeWithPrefix(String), +} + +pub fn translate_rewrite(r: &str, group_count: usize, global: bool) -> Rewrite { + let b = r.as_bytes(); + let mut out = String::with_capacity(r.len() + 4); + let mut i = 0; + while i < b.len() { + match b[i] { + b'$' => { + out.push_str("$$"); + i += 1; + } + b'\\' => match b.get(i + 1) { + Some(&d) if d.is_ascii_digit() => { + let n = (d - b'0') as usize; + if n > group_count { + // MaxSubmatch pre-check: full no-op, global or not. + return Rewrite::Identity; + } + out.push_str(&format!("${{{n}}}")); + i += 2; + } + Some(b'\\') => { + out.push('\\'); + i += 2; + } + _ => { + // Bad escape (incl. trailing lone backslash). + return if global { + Rewrite::ConsumeWithPrefix(out) + } else { + Rewrite::Identity + }; + } + }, + _ => { + let start = i; + i += 1; + while i < b.len() && !r.is_char_boundary(i) { + i += 1; + } + out.push_str(&r[start..i]); + } + } + } + Rewrite::Template(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn perl_classes_rewrite_in_and_out_of_classes() { + assert_eq!(translate_pattern(r"\d+").unwrap(), r"(?-u:\d)+"); + assert_eq!(translate_pattern(r"[\d]").unwrap(), "[0-9]"); + assert_eq!(translate_pattern(r"[a\w]").unwrap(), "[a0-9A-Za-z_]"); + assert_eq!(translate_pattern(r"\s").unwrap(), "[\\t\\n\\x0c\\r ]"); + assert_eq!(translate_pattern(r"\S").unwrap(), "[^\\t\\n\\x0c\\r ]"); + assert_eq!(translate_pattern(r"a\bb").unwrap(), r"a(?-u:\b)b"); + // ']' literal-if-first survives; '\D' inside a class rejects. + assert_eq!(translate_pattern("[]a]").unwrap(), "[]a]"); + assert!(translate_pattern(r"[a\D]").is_err()); + } + + #[test] + fn reject_list() { + for p in [r"a\B", "(?a)", "a{1001}", "a*+", r"\u0041", r"\Qab\E"] { + assert!(translate_pattern(p).is_err(), "{p} should reject"); + } + assert!(translate_pattern("(?Pa)(?Pb)").is_err()); + // Lookbehind (?<=..) is NOT the angle-group reject — it passes + // through and fails to compile in both engines. + assert!(translate_pattern("(?<=a)b").is_ok()); + assert!(translate_pattern("a{1000}").is_ok()); + } + + #[test] + fn options_alphabet() { + let o = parse_options("gi", true).unwrap(); + assert!(o.global && o.case_insensitive); + // Last conflicting letter wins. + assert!(!parse_options("ic", true).unwrap().case_insensitive); + assert!(parse_options("ns", false).unwrap().dotall); + assert!(!parse_options("sn", false).unwrap().dotall); + assert!(parse_options("g", false).is_err()); + assert!(parse_options("q", false).is_err()); + assert!(parse_options(" i ", false).unwrap().case_insensitive); + } + + #[test] + fn rewrite_translation_and_quirks() { + assert!(matches!( + translate_rewrite(r"[\1]", 1, false), + Rewrite::Template(t) if t == "[${1}]" + )); + assert!(matches!( + translate_rewrite("a$b", 0, false), + Rewrite::Template(t) if t == "a$$b" + )); + // Out-of-range backref: full no-op both modes. + assert!(matches!(translate_rewrite(r"\2", 1, true), Rewrite::Identity)); + // Bad escape: no-op non-global, consume-with-prefix global. + assert!(matches!(translate_rewrite(r"A\xB", 0, false), Rewrite::Identity)); + assert!(matches!( + translate_rewrite(r"A\xB", 0, true), + Rewrite::ConsumeWithPrefix(p) if p == "A" + )); + // \10 = \1 then literal '0'; \0 = whole match. + assert!(matches!( + translate_rewrite(r"\10", 1, false), + Rewrite::Template(t) if t == "${1}0" + )); + assert!(matches!( + translate_rewrite(r"\0", 0, false), + Rewrite::Template(t) if t == "${0}" + )); + } +} diff --git a/src/specializer/rewrite.rs b/src/specializer/rewrite.rs index 2b03578..f42a5da 100644 --- a/src/specializer/rewrite.rs +++ b/src/specializer/rewrite.rs @@ -117,6 +117,19 @@ pub fn rewrite_star_filters(tokens: Vec) -> Vec { Some(Token::Word(w)) if w.value.eq_ignore_ascii_case("glob") && !negated => { Some("G") } + // `* [NOT] SIMILAR TO 're'` (wave-B): consume the TO here. + Some(Token::Word(w)) if w.keyword == Keyword::SIMILAR => { + j += 1; + while matches!(tokens.get(j), Some(Token::Whitespace(_))) { + j += 1; + } + match tokens.get(j) { + Some(Token::Word(w2)) if w2.keyword == Keyword::TO => { + Some(if negated { "NS" } else { "S" }) + } + _ => None, + } + } _ => None, }; let Some(code) = op else { diff --git a/src/specializer/tests.rs b/src/specializer/tests.rs index 4c44218..3a55e90 100644 --- a/src/specializer/tests.rs +++ b/src/specializer/tests.rs @@ -502,12 +502,15 @@ fn unsupported_constructs_are_named_cleanly() { // function arm until the catalogue distinguishes aggregation. ("SELECT sum(a) FROM __THIS__", "aggregate function sum"), ("SELECT a FROM __THIS__ GROUP BY a", "aggregation"), - // Wave-3 named rejects: regex family, grapheme-based reverse. - ("SELECT regexp_matches('x', 'y') FROM __THIS__", "RE2"), + // Scalar regexp serves since wave B; list-valued forms stay named. + ( + "SELECT regexp_extract_all('x', 'y') FROM __THIS__", + "list-valued", + ), ("SELECT reverse('abc') FROM __THIS__", "grapheme"), ("SELECT jaro_similarity('x', 'y') FROM __THIS__", "function"), - // Star forms now expand; COLUMNS stays wave-B (regexp). - ("SELECT COLUMNS('a') FROM __THIS__", "function COLUMNS"), + // Bare COLUMNS expands since wave B; expression forms stay named. + ("SELECT COLUMNS('a') + 1 FROM __THIS__", "COLUMNS"), ("SELECT a FROM __THIS__ ORDER BY a", "ORDER BY"), ("SELECT NULL FROM __THIS__", "NULL literal"), ("SELECT a FROM other_table", "must be the dynamic table"), @@ -1590,3 +1593,123 @@ fn column_renaming_table_alias() { other => panic!("wrong outcome: {:?}", other.err()), } } + +#[test] +fn regexp_family_end_to_end() { + // Wave-B pins: regexp_matches is a SEARCH, ~ / SIMILAR TO are FULL + // match on the raw pattern (no % translation), extract returns '' on + // no match, replace is first-match-only without 'g'. + let schema = cols(&[("s", Ty::Str, true)]); + let input = || { + batch( + 3, + vec![c_str(&[Some("hello world"), Some("abc123def"), None])], + ) + }; + let got = run_sql( + "SELECT regexp_matches(s, 'ell') AS m, s ~ 'ell' AS f, \ + s ~ 'h.*d' AS w, s SIMILAR TO 'h%o' AS pct, \ + regexp_extract(s, '[0-9]+') AS num, \ + regexp_extract(s, '(\\w+) (\\w+)', 2) AS g2, \ + regexp_replace(s, 'l', 'L') AS r1, \ + regexp_replace(s, 'l', 'L', 'g') AS rg FROM __THIS__", + &schema, + input(), + ) + .unwrap(); + assert_eq!( + got, + rows(&[ + &[ + "true", "false", "true", "false", "", "world", "heLlo world", + "heLLo worLd" + ], + &[ + "false", "false", "false", "false", "123", "", "abc123def", + "abc123def" + ], + &["NULL", "NULL", "NULL", "NULL", "NULL", "NULL", "NULL", "NULL"], + ]) + ); + // ASCII \d semantics survive the translation (the differential pin): + // Arabic-Indic digits do NOT match \d but DO match \p{Nd}. + let got = run_sql( + "SELECT regexp_matches(s, '\\d') AS a, regexp_matches(s, '\\p{Nd}') AS u \ + FROM __THIS__", + &schema, + batch(1, vec![c_str(&[Some("\u{0663}\u{0664}")])]), + ) + .unwrap(); + assert_eq!(got, rows(&[&["false", "true"]])); + // Backrefs are backslash-style; $1 is a literal. Options 'i' folds. + let got = run_sql( + "SELECT regexp_replace(s, '(h)ello', '[\\1]') AS b, \ + regexp_replace(s, '(h)ello', '[$1]') AS lit, \ + regexp_matches(s, 'HELLO', 'i') AS ci FROM __THIS__", + &schema, + batch(1, vec![c_str(&[Some("hello world")])]), + ) + .unwrap(); + assert_eq!(got, rows(&[&["[h] world", "[$1] world", "true"]])); + // Invalid-rewrite quirks: out-of-range backref = silent no-op. + let got = run_sql( + "SELECT regexp_replace(s, '(h)', '\\2') AS noop FROM __THIS__", + &schema, + batch(1, vec![c_str(&[Some("hello")])]), + ) + .unwrap(); + assert_eq!(got, rows(&[&["hello"]])); + // Bad constant pattern errors at PREPARE (pinned eagerness). + match prep("SELECT regexp_matches(s, '(') AS x FROM __THIS__", &schema) { + Err(PrepareError::Bind(m)) => assert!(m.contains("Invalid Input Error"), "{m}"), + other => panic!("wrong outcome: {:?}", other.err()), + } + // Divergence guard: \B rejects cleanly. + match prep("SELECT regexp_matches(s, 'a\\B') AS x FROM __THIS__", &schema) { + Err(PrepareError::Unsupported(m)) => assert!(m.contains("\\B"), "{m}"), + other => panic!("wrong outcome: {:?}", other.err()), + } +} + +#[test] +fn star_similar_to_and_columns() { + // Wave-B pins: positive * SIMILAR TO = unanchored search over names, + // NOT form = NOT full-match — NOT complements ('a.*' on a col set + // where a name merely CONTAINS 'a' lands in both results). + let schema = cols(&[ + ("abc", Ty::I64, false), + ("abd", Ty::I64, false), + ("xyz", Ty::Str, false), + ]); + let names = |sql: &str| { + prep(sql, &schema) + .unwrap() + .out_cols + .iter() + .map(|c| c.name.clone()) + .collect::>() + }; + assert_eq!(names("SELECT * SIMILAR TO 'c' FROM __THIS__"), ["abc"]); + assert_eq!( + names("SELECT * NOT SIMILAR TO 'c' FROM __THIS__"), + ["abc", "abd", "xyz"] // NOT full-match: 'c' full-matches no name + ); + assert_eq!( + names("SELECT * NOT SIMILAR TO 'ab.' FROM __THIS__"), + ["xyz"] + ); + assert_eq!(names("SELECT COLUMNS('ab.') FROM __THIS__"), ["abc", "abd"]); + assert_eq!( + names("SELECT COLUMNS(*) FROM __THIS__"), + ["abc", "abd", "xyz"] + ); + // Alias stamps every expansion; duplicates feed the dedup rename. + assert_eq!( + names("SELECT COLUMNS('ab.') AS x FROM __THIS__"), + ["x", "x_1"] + ); + match prep("SELECT COLUMNS('zz.*') FROM __THIS__", &schema) { + Err(PrepareError::Bind(m)) => assert!(m.contains("No matching columns"), "{m}"), + other => panic!("wrong outcome: {:?}", other.err()), + } +} diff --git a/tests/test_duckdb_waveB_regexp.py b/tests/test_duckdb_waveB_regexp.py new file mode 100644 index 0000000..09e164a --- /dev/null +++ b/tests/test_duckdb_waveB_regexp.py @@ -0,0 +1,73 @@ +"""Wave-B regexp family vs the duckdb oracle. + +Pins: docs/superpowers/specs/2026-07-27-waveB-regexp-pins.md — the rust +`regex` crate behind the bind-time RE2 translation layer; ~ / SIMILAR TO +are FULL match; extract returns '' on no-match; replace backrefs are +backslash-style with the invalid-rewrite quirks. +""" + +from __future__ import annotations + +from test_duckdb_interpreter import duck_check + +T = {"a": "int", "s": "str?"} +T_ROWS = [ + {"a": 1, "s": "hello world"}, + {"a": 2, "s": "abc123def"}, + {"a": 3, "s": None}, + {"a": 4, "s": ""}, + {"a": 5, "s": "héllo ٣٤"}, + {"a": 6, "s": "HELLO"}, +] + + +def test_regexp_match_forms(): + duck_check( + "SELECT regexp_matches(s, 'ell') AS m, regexp_full_match(s, 'ell') AS f, " + "s ~ 'h.*' AS t, s !~ 'h.*' AS nt, s SIMILAR TO 'h%o' AS pct, " + "s SIMILAR TO 'h.llo.*' AS re, s NOT SIMILAR TO 'h.*' AS ns, " + "regexp_matches(s, '') AS emp, regexp_matches(s, 'HELLO', 'i') AS ci " + "FROM __THIS__", + T, + T_ROWS, + ) + # ASCII perl classes vs Unicode property classes (the differential pin). + duck_check( + "SELECT regexp_matches(s, '\\d') AS d, regexp_matches(s, '\\p{Nd}') AS nd, " + "regexp_matches(s, '\\w+') AS w FROM __THIS__", + T, + T_ROWS, + ) + + +def test_regexp_extract(): + duck_check( + "SELECT regexp_extract(s, '[0-9]+') AS num, " + "regexp_extract(s, '(\\w+) (\\w+)', 2) AS g2, " + "regexp_extract(s, '(h)(x)?', 2) AS npart, " + "regexp_extract(s, 'zzz') AS miss, " + "regexp_extract(s, '(a)|(b)', 1) AS alt, " + "regexp_extract(s, '.', 0, 'i') AS opt FROM __THIS__", + T, + T_ROWS, + ) + + +def test_regexp_replace(): + duck_check( + "SELECT regexp_replace(s, 'l', 'L') AS one, " + "regexp_replace(s, 'l', 'L', 'g') AS all_, " + "regexp_replace(s, '(h)ello', '[\\1]') AS bref, " + "regexp_replace(s, '(h)ello', '[$1]') AS dollar_lit, " + "regexp_replace(s, '(h)', '\\2') AS oor, " + "regexp_replace(s, 'o', 'O', 'gi') AS gi FROM __THIS__", + T, + T_ROWS, + ) + + +def test_star_similar_and_columns(): + duck_check("SELECT * SIMILAR TO 'a' FROM __THIS__", T, T_ROWS) + duck_check("SELECT * NOT SIMILAR TO 'a.*' FROM __THIS__", T, T_ROWS) + duck_check("SELECT COLUMNS('s') FROM __THIS__", T, T_ROWS) + duck_check("SELECT COLUMNS(*) FROM __THIS__", T, T_ROWS) From 189f851222789ec5d9727259d476f8b56b23c522 Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Mon, 27 Jul 2026 02:36:03 +0200 Subject: [PATCH 4/5] chore(backlog): TASK-53 all ACs checked + Done Co-Authored-By: Claude Fable 5 --- ...ons-~-~-SIMILAR-TO-deferred-star-forms.md" | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git "a/backlog/tasks/task-53 - Specializer-wave-B-\342\200\224-regexp-family-regexp_-functions-~-~-SIMILAR-TO-deferred-star-forms.md" "b/backlog/tasks/task-53 - Specializer-wave-B-\342\200\224-regexp-family-regexp_-functions-~-~-SIMILAR-TO-deferred-star-forms.md" index 98ee7b2..defb20f 100644 --- "a/backlog/tasks/task-53 - Specializer-wave-B-\342\200\224-regexp-family-regexp_-functions-~-~-SIMILAR-TO-deferred-star-forms.md" +++ "b/backlog/tasks/task-53 - Specializer-wave-B-\342\200\224-regexp-family-regexp_-functions-~-~-SIMILAR-TO-deferred-star-forms.md" @@ -3,9 +3,10 @@ id: TASK-53 title: >- Specializer wave B — regexp family (regexp_* functions, ~/!~/SIMILAR TO, deferred star forms) -status: In Progress +status: Done assignee: [] created_date: '2026-07-26 23:50' +updated_date: '2026-07-27 00:35' labels: [] milestone: m-7 dependencies: @@ -31,11 +32,34 @@ Pins-first; wave-3 over-generalization precedent applies — every claim needs a ## Acceptance Criteria -- [ ] #1 Pins-first: wave-B pins spec (md + JSON) committed before implementation, incl. the duckdb-vs-rust-regex differential battery and the crate decision -- [ ] #2 regexp_matches / regexp_full_match / regexp_extract / regexp_replace serve per pins (options strings, group semantics, backrefs, NULL handling); list/table forms classify clean -- [ ] #3 Operators ~ / !~ and SIMILAR TO on values serve per pins (anchoring semantics measured, not assumed) -- [ ] #4 Deferred star forms serve: * SIMILAR TO name filter (incl. the pinned NOT asymmetry) and COLUMNS('re') expansion -- [ ] #5 Patterns whose semantics differ between RE2 and rust-regex classify clean-unsupported (guard at bind time from the pinned divergence list) — never a wrong answer -- [ ] #6 Corpus replay: three-outcome contract holds, zero FAILs, match count reported -- [ ] #7 Gate green both backends, clippy clean, serving-bench parity gate passes +- [x] #1 Pins-first: wave-B pins spec (md + JSON) committed before implementation, incl. the duckdb-vs-rust-regex differential battery and the crate decision +- [x] #2 regexp_matches / regexp_full_match / regexp_extract / regexp_replace serve per pins (options strings, group semantics, backrefs, NULL handling); list/table forms classify clean +- [x] #3 Operators ~ / !~ and SIMILAR TO on values serve per pins (anchoring semantics measured, not assumed) +- [x] #4 Deferred star forms serve: * SIMILAR TO name filter (incl. the pinned NOT asymmetry) and COLUMNS('re') expansion +- [x] #5 Patterns whose semantics differ between RE2 and rust-regex classify clean-unsupported (guard at bind time from the pinned divergence list) — never a wrong answer +- [x] #6 Corpus replay: three-outcome contract holds, zero FAILs, match count reported +- [x] #7 Gate green both backends, clippy clean, serving-bench parity gate passes + +## Implementation Plan + + +Stages (each lands with tests + corpus replay green): +1. Regex infrastructure: regex = "1" dep (RegexBuilder::octal(true), default Unicode — unicode(false) measured to BREAK (?i) parity); translation module (perl-class rewrite \d->(?-u:\d) etc incl. in-class variants, reject list: \B, (?<>), dup group names, bounds>1000, stacked quantifiers, \u, \Q\E; options parser c/i/l/s live, m/n/p no-ops, g replace-only; replacement translation \N->${N}, $->$$, with the out-of-range->identity and bad-escape global/non-global asymmetry resolved at BIND time); Program-level regex table; IR ops ReMatch/ReExtract/ReReplace + print/parse/verify + both backends. +2. Functions + operators: regexp_matches (search) / regexp_full_match / regexp_extract (''-on-no-match, flat 0..9 group check) / regexp_replace; ~ / !~ = full match (NOT Postgres search!); SIMILAR TO = raw-pattern full match (no % translation); NULL rules incl. the options-arg asymmetry. +3. Star forms: * SIMILAR TO (unanchored search) / NOT SIMILAR TO (NOT full match — independent predicates) via rewrite.rs marker codes; COLUMNS('re') interception + declared-order expansion. +4. Census + bench parity + close-out + PR. +Spec: docs/superpowers/specs/2026-07-27-waveB-regexp-pins.md (pins committed b003122 before implementation). + + +## Final Summary + + +Wave B shipped: the regexp family, corpus 484 -> 505 bit-exact of 678 (zero wrong answers, zero replay FAILs). Pins-first: 6-agent fleet (5 DuckDB areas + the RE2-vs-rust-regex DIFFERENTIAL battery) committed as docs/superpowers/specs/2026-07-27-waveB-regexp-pins.md + pins-waveB/*.json before implementation. + +The engine is the rust `regex` crate (new direct dependency) behind the measured parity recipe: RegexBuilder::octal(true) + default Unicode mode (unicode(false) was measured to BREAK (?i) folding parity — the 'obvious' fix is the wrong one), a bind-time Perl-class rewrite in retrans.rs (\d -> (?-u:\d) etc, in-class variants included) closing the whole RE2-ASCII-vs-rust-Unicode gap, a reject list for irreconcilables (\B unservable in DuckDB itself, (?), duplicate group names, bounds > 1000, stacked quantifiers a*+ — silently WRONG in rust, not just error-shaped different, \u, \Q\E), and replacement-template translation with RE2's invalid-rewrite quirks resolved at bind (out-of-range backref = identity; global bad escape = consume-with-prefix). With that applied all 98 differential battery entries were byte-identical or identically-rejected. + +Infrastructure: Program grew a prepare-time regex table (print/parse round-trips it; verify checks indices and rewrite presence); ReMatch/ReExtract/ReReplace on both backends (interp closures own their compiled Regex; cranelift helpers read a CompiledRe table through Cx). Semantics per pins: regexp_matches = unanchored SEARCH, regexp_full_match / ~ / !~ / SIMILAR TO = FULL match (~ is NOT the Postgres search — the DuckDB binder error names regexp_full_match; SIMILAR TO translates NO wildcards), regexp_extract '' on no-match with the flat 0..9 group check and NULL-group -> '', regexp_replace backslash backrefs with $-literals and the NULL-options -> NULL asymmetry, options alphabet c/i/l/s live with m/n/p as measured no-ops, constant patterns compile at prepare (pinned eagerness). Star forms: * SIMILAR TO = unanchored name search, * NOT SIMILAR TO = NOT full-match (independent predicates — pinned non-complement asymmetry), bare COLUMNS('re')/COLUMNS(*) expand in declared order with alias stamping into the wave-5 dedup. List-valued forms (extract_all, split_to_array) and COLUMNS-in-expression stay clean-unsupported. + +Gate: 154 Rust + 557 py green, interp backend identical (548 minus the backend-identity guards), clippy clean on wave files, serving-bench parity gate passed with spec in the usual 1.4-2x band. Remaining pool after the wave: aggregation 45 + table-fns 24 (out of scope), lists/structs 25 (wave C), dynamic self-joins 10, small tails. + From 2c8de65d38a07276cd729a723aba0385bc318f19 Mon Sep 17 00:00:00 2001 From: AmirHossein Roozbahani Date: Mon, 27 Jul 2026 02:51:01 +0200 Subject: [PATCH 5/5] docs(specializer): known-limitations reference + executable twin suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs/known-limitations.md is the user-facing contract: every deliberate limitation grouped by kind (the specialization bargain, row-serving scope, type-system boundaries, measured descopes, contract choices), each with its justification and the exact build-time message. tests/ test_known_limitations.py asserts all of them — lifting a limitation breaks a test and forces the doc to change in the same commit. Co-Authored-By: Claude Fable 5 --- docs/known-limitations.md | 130 +++++++++++++++++++++ tests/test_known_limitations.py | 194 ++++++++++++++++++++++++++++++++ 2 files changed, 324 insertions(+) create mode 100644 docs/known-limitations.md create mode 100644 tests/test_known_limitations.py diff --git a/docs/known-limitations.md b/docs/known-limitations.md new file mode 100644 index 0000000..bc778ba --- /dev/null +++ b/docs/known-limitations.md @@ -0,0 +1,130 @@ +# Known limitations — deliberate, named, and loud + +This is the user-facing contract of the SQL specializer (`DuckDBInferFn`): +what it refuses to serve, **why**, and what you see when you hit a limit. +Its executable twin is [`tests/test_known_limitations.py`](../tests/test_known_limitations.py) — +every limitation below is asserted there, so lifting one breaks a test and +forces this document to change with it. + +**The contract.** For any SQL you hand it, the engine does exactly one of: + +1. **Serve it bit-for-bit identical to DuckDB** (verified continuously + against DuckDB's own test corpus: 505 of 678 statements as of wave B), or +2. **Refuse loudly at BUILD time** — `DuckDBInferFn(...)` raises a + `ValueError` naming the construct. Nothing is ever silently wrong or + silently dropped at inference time. + +There is no third mode. Every limitation here is a *measured decision* +recorded in a pins spec (`docs/superpowers/specs/`), not an accident. + +--- + +## 1. The specialization bargain (inherent to the engine model) + +The specializer's speed comes from doing ALL general work at build time: +parse once, bind once, compile once, freeze the static tables into the +code. Anything that would require re-doing general work per row is +rejected, permanently by design: + +| Limitation | You'll see | Why | +|---|---|---| +| Regex patterns must be constants (`regexp_matches(s, pattern_col)` rejects) | `unsupported: non-constant regex pattern (compiled at prepare in v0)` | Regexes compile at prepare; DuckDB compiles per row. Per-row compilation is the opposite of specialization. | +| Replacement strings / regex options / extract group indexes must be constants | `non-constant regexp_replace replacement` etc. | Same. | +| Static (join) tables must be provided at build time, with unique keys and no NULL values | `duplicate map key`, `has a NULL in value column` | Joins are frozen hash maps baked into the function. Duplicate keys mean 1:N join multiplicity — a designed extension (stage B, TASK-50 notes) that is deliberately not built yet. | +| The dynamic table cannot be joined to itself | `joining the dynamic table to itself` | The batch is the probe side; using it as a build side too needs stage-B machinery. | +| Exactly one row table drives the query | `the specializer takes exactly one row table`, `must be the dynamic table` | The serving contract is rows-in → rows-out for one entity stream. | + +## 2. Out of scope for row-serving (by decision, not difficulty) + +The engine serves **row-at-a-time feature transforms**. Whole-relation +constructs are out of scope because their output shape is not +one-row-in/one-row-out: + +- Aggregation: `GROUP BY`, `HAVING`, `sum`/`count`/`avg`/... → + `aggregate function ... (no aggregation in v0)` +- `ORDER BY`, `LIMIT`/`OFFSET`, `DISTINCT` — row-independent transforms + don't reorder or deduplicate. +- CTEs (`WITH`), `UNION`/`INTERSECT`/`EXCEPT`, subqueries, multiple + statements. +- Table functions in FROM (`range(...)`) — there is no base table. +- `rowid` pseudo-column — rows have no stable identity in a stream. +- `FULL OUTER JOIN` — emits rows that no input row produced. + +## 3. Type-system boundaries + +The engine computes in exactly four types: `i64`, `f64`, UTF-8 string, +bool. Measured consequences: + +- **f32 base tables reject** (`engine is f64-only`). +- **Static-table key/value columns must fit BIGINT** — `UBIGINT`/`HUGEINT` + payloads outside i64 reject with a named message. +- **Lists and structs reject** (`row column 'x' has a non-scalar type`) — + this is the designated next capability wave ("wave C"), and it also gates + `regexp_extract_all` / `regexp_split_to_array` / the STRUCT form of + `regexp_extract` (`list-valued — non-scalar in v0`). +- **DECIMAL literals are f64** — a documented divergence: DuckDB types + `1.5` as `DECIMAL(2,1)` and does decimal arithmetic; we map to f64. + Values agree on every corpus case; exact-decimal accumulation semantics + are not reproduced. +- Narrow integer widths don't exist: bitwise ops compute in i64, which + matches DuckDB whenever either operand is BIGINT (always true for + row-model ints). Explicit narrow CASTs are rejected rather than + risking DuckDB's narrow-width overflow behavior. + +## 4. Semantics descoped after measurement + +Each of these was measured against DuckDB 1.5.5 first (pins in +`docs/superpowers/specs/`), and rejected because serving it would risk a +wrong answer or require semantics we can't reproduce exactly: + +| Construct | Why it's descoped | +|---|---| +| `reverse()` | DuckDB reverses UAX-29 grapheme clusters (incl. regional-indicator pairing), not codepoints. A codepoint reverse would be silently wrong on emoji/accents. | +| `^` operator | It IS pow in DuckDB, but sqlparser's precedence differs from DuckDB's (`2*x^y` would parse as `(2*x)^y`). Mapping it computes the wrong tree silently. Use `pow()`. | +| prefix `~`, `#`, `NOT GLOB` | Same class: precedence/parse divergences that would silently mis-associate. `xor()` covers bit-xor; `NOT (x GLOB p)` works. | +| Regex reject list: `\B`, `\Q…\E`, `(?…)`, duplicate group names, bounds > 1000, stacked quantifiers (`a*+`), `\u` escapes, negated Perl classes inside `[...]` | The RE2↔rust-regex differential battery (98 entries) proved these are the constructs where the engines disagree or DuckDB itself is broken (`\B` crashes DuckDB at runtime on non-ASCII). Everything else is byte-identical. | +| `SIMILAR TO ... ESCAPE` | Not implemented in DuckDB itself. | +| `* EXCLUDE (t.key)` on a USING join | DuckDB UNMERGES the coalesced column (it reappears at the right table's position) — measured, not modeled. Unqualified EXCLUDE works. | +| `BETWEEN`/`IN` mixing non-numeric string literals with numbers | DuckDB converts at EXECUTION time (an empty input succeeds!); a bind-time conversion was measured to be over-eager. Numeric literals convert fine. | +| `COLUMNS(...)` inside expressions, lambda/list forms | Only bare `COLUMNS('re')` / `COLUMNS(*)` as select items are served. | +| Bare `NULL` with no typing context, `CASE` where every branch is NULL, `COALESCE`/`NULLIF` of only NULLs | DuckDB's SQLNULL type has no i64/f64/str/bool home. `NULL NULL` IS served with the measured result types. | + +## 5. Deliberate contract choices (behavior differs from raw DuckDB surface) + +These are served, but with a consciously chosen surface — know them: + +- **Duplicate output column names are renamed**, using DuckDB's own + boundary-rename algorithm (`id, id, id_1` → `id, id_1, id_1_1`; + case-insensitive collision check). Raw DuckDB keeps duplicates at the + top level, but a pydantic model or dict cannot — and this rename is + bit-identical to what DuckDB itself does at every subquery/CTE/CTAS + boundary and in `.df()`. Verified against `.df()` in tests. +- **`NULL || NULL` types as VARCHAR** in the output model (value is NULL + either way; DuckDB's SQLNULL materializes as INTEGER). +- **Error TEXTS are approximate where noted.** Runtime traps + (overflow, shifts, substring range) reproduce DuckDB's message bodies + verbatim; some bind-time rejections (star-filter zero-match, regex + compile errors) use our own wording with the same error class. The + corpus only ever compares successful results, so texts never affect + parity. +- **Two known oracle divergences** (excluded from the corpus by name in + `tests/test_corpus_replay.py::_KNOWN_DIVERGENT_SOURCES`): DuckDB + behaviors that depend on column STATISTICS (e.g. ILIKE's NUL handling + selects a different kernel depending on *sibling rows*). A row-at-a-time + engine cannot reproduce statistics-dependent semantics even in + principle; the engine is NUL-transparent (the ASCII-kernel behavior). +- **`%`-by-zero NaN bit pattern is platform-libm** — pinned as + engine==oracle bit agreement per platform, not a constant. + +## 6. How to read a rejection + +Every rejection is a `ValueError` at `DuckDBInferFn(...)` construction +whose message starts with a classification: + +- `unsupported: ...` — real SQL, deliberately not served (this document). +- `parse error: ...` — the dialect surface ends here. +- `bind error: ...` — the query is wrong against YOUR schema (typo, + type mismatch), not a limitation. + +If a message you hit isn't in this document or the tests, that's a bug in +our bookkeeping — file it. diff --git a/tests/test_known_limitations.py b/tests/test_known_limitations.py new file mode 100644 index 0000000..4d1e823 --- /dev/null +++ b/tests/test_known_limitations.py @@ -0,0 +1,194 @@ +"""Executable twin of docs/known-limitations.md. + +Every DELIBERATE limitation is asserted here: the SQL that hits it and the +named build-time rejection (or, for contract choices, the chosen behavior). +If an engine change lifts one of these, the test fails — update the doc in +the same commit. Section numbers mirror the document. +""" + +from __future__ import annotations + +import pytest +from pydantic import create_model +from test_duckdb_interpreter import duck_check, static + +from sql_transform._interpreter import DuckDBInferFn + +T = create_model("T", a=(int, ...), s=(str | None, None)) + + +def build(sql, statics=None): + return DuckDBInferFn(sql, row_tables={"__THIS__": T}, static_tables=statics or {}) + + +def rejects(sql, needle, statics=None): + with pytest.raises(ValueError, match=needle): + build(sql, statics) + + +# ---- 1. The specialization bargain ---------------------------------------- + + +def test_non_constant_regex_pattern_rejects(): + # Regexes compile ONCE at prepare; per-row compilation is the opposite + # of specialization. + rejects("SELECT regexp_matches(s, s) FROM __THIS__", "non-constant regex") + rejects( + "SELECT regexp_replace(s, 'a', s) FROM __THIS__", + "non-constant regexp_replace replacement", + ) + + +def test_static_tables_are_frozen_unique_key_maps(): + dup = static({"id": "int", "v": "int"}, [{"id": 1, "v": 1}, {"id": 1, "v": 2}]) + # Duplicate keys = 1:N multiplicity — designed (stage B), not built. + rejects( + "SELECT v FROM __THIS__ JOIN d ON a = d.id", + "duplicate map key", + {"d": dup}, + ) + withnull = static({"id": "int", "v": "int?"}, [{"id": 1, "v": None}]) + rejects( + "SELECT v FROM __THIS__ JOIN d ON a = d.id", + "NULL in value column", + {"d": withnull}, + ) + + +def test_dynamic_self_join_rejects(): + rejects( + "SELECT t2.a FROM __THIS__ JOIN __THIS__ t2 ON __THIS__.a = t2.a", + "dynamic table", + ) + + +# ---- 2. Out of scope for row-serving -------------------------------------- + + +@pytest.mark.parametrize( + ("sql", "needle"), + [ + ("SELECT sum(a) FROM __THIS__", "no aggregation"), + ("SELECT a FROM __THIS__ GROUP BY a", "aggregation"), + ("SELECT a FROM __THIS__ ORDER BY a", "ORDER BY"), + ("SELECT a FROM __THIS__ LIMIT 5", "LIMIT"), + ("SELECT DISTINCT a FROM __THIS__", "DISTINCT"), + ("WITH c AS (SELECT 1) SELECT a FROM __THIS__", "common table"), + ("SELECT a FROM __THIS__ UNION SELECT a FROM __THIS__", "UNION"), + ("SELECT rowid FROM __THIS__", "rowid"), + ( + "SELECT a FROM __THIS__ FULL OUTER JOIN x ON a = x.a", + "join type", + ), + ], +) +def test_whole_relation_constructs_reject(sql, needle): + rejects(sql, needle) + + +# ---- 3. Type-system boundaries --------------------------------------------- + + +def test_non_scalar_row_columns_reject(): + L = create_model("L", xs=(list[int], ...)) + with pytest.raises(ValueError, match="non-scalar"): + DuckDBInferFn( + "SELECT xs FROM __THIS__", row_tables={"__THIS__": L}, static_tables={} + ) + + +def test_list_valued_regexp_forms_reject(): + # Gated on list types (wave C), not on regex semantics. + rejects("SELECT regexp_extract_all(s, 'a') FROM __THIS__", "list-valued") + rejects("SELECT regexp_split_to_array(s, 'a') FROM __THIS__", "regexp_split") + + +def test_ubigint_static_payloads_reject(): + import pyarrow as pa + + big = pa.table({"id": pa.array([2**64 - 1], pa.uint64()), "v": [1]}) + rejects( + "SELECT v FROM __THIS__ JOIN d ON a = d.id", + "outside BIGINT range", + {"d": big}, + ) + + +# ---- 4. Semantics descoped after measurement ------------------------------- + + +@pytest.mark.parametrize( + ("sql", "needle"), + [ + # Grapheme-cluster semantics (UAX-29) not modeled. + ("SELECT reverse(s) FROM __THIS__", "grapheme"), + # ^ IS pow, but sqlparser's precedence would compute the wrong tree. + ("SELECT a ^ 2 FROM __THIS__", "precedence"), + # Regex reject list: measured RE2 <-> rust-regex divergences. + ("SELECT regexp_matches(s, 'a\\B') FROM __THIS__", "B in a regex"), + ("SELECT regexp_matches(s, '\\Qab\\E') FROM __THIS__", "literal quoting"), + ("SELECT regexp_matches(s, 'a*+') FROM __THIS__", "stacked"), + ( + "SELECT regexp_matches(s, '(?Pa)(?Pb)') FROM __THIS__", + "duplicate regex capture group", + ), + ("SELECT regexp_matches(s, 'a{1001}') FROM __THIS__", "repetition bound"), + # Not implemented in DuckDB itself. + ("SELECT s SIMILAR TO 'a' ESCAPE 'x' FROM __THIS__", "escape"), + # Exec-time conversion order (an empty input succeeds in DuckDB). + ("SELECT a IN ('abc') FROM __THIS__", "non-numeric"), + # Only bare COLUMNS as a select item is served. + ("SELECT COLUMNS('a') + 1 FROM __THIS__", "COLUMNS"), + # SQLNULL has no home type. + ("SELECT NULL FROM __THIS__", "NULL literal"), + ], +) +def test_measured_descopes_reject(sql, needle): + rejects(sql, needle) + + +def test_using_unmerge_exclude_rejects(): + d = static({"a": "int", "v": "int"}, [{"a": 1, "v": 10}]) + # DuckDB UNMERGES the coalesced column here — measured, not modeled. + rejects( + "SELECT * EXCLUDE (d.a) FROM __THIS__ JOIN d USING (a)", + "USING-merged", + {"d": d}, + ) + + +# ---- 5. Deliberate contract choices ---------------------------------------- + + +def test_duplicate_names_use_duckdbs_boundary_rename(): + # Raw DuckDB keeps top-level duplicates; a typed model cannot. We apply + # DuckDB's OWN subquery/CTAS/.df() rename — not an invention. + fn = DuckDBInferFn( + "SELECT a, a AS a, a AS a_1 FROM __THIS__", + row_tables={"__THIS__": T}, + static_tables={}, + output="dict", + ) + got = fn.infer({"__THIS__": [T(a=7)]}) + assert list(got[0].keys()) == ["a", "a_1", "a_1_1"] + + +def test_null_op_null_serves_with_measured_types(): + # The limitation is ONLY the context-free bare NULL; typed NULL ops + # serve with the pinned result types. + duck_check( + "SELECT NULL + NULL AS s, NULL / NULL AS d FROM __THIS__", + {"a": "int", "s": "str?"}, + [{"a": 1, "s": None}], + ) + + +def test_rejections_are_build_time_and_named(): + # The load-bearing property: limits surface at CONSTRUCTION, before any + # row is ever inferred — never silently at inference time. + with pytest.raises(ValueError, match="unsupported"): + DuckDBInferFn( + "SELECT sum(a) FROM __THIS__", + row_tables={"__THIS__": T}, + static_tables={}, + )