|
| 1 | +"""Generate src/specializer/exec/casemap.rs by measuring DuckDB. |
| 2 | +
|
| 3 | +Why this file exists |
| 4 | +-------------------- |
| 5 | +DuckDB's upper()/lower() use utf8proc's SIMPLE case mapping: every codepoint |
| 6 | +maps to exactly one codepoint (UnicodeData.txt's Simple_*case_Mapping). |
| 7 | +Rust std only exposes the FULL mapping (SpecialCasing.txt), where a codepoint |
| 8 | +may expand ('ss' -> "SS"). The engine's fallback rule — take Rust's full map |
| 9 | +iff it is 1:1, else keep the codepoint — agrees with DuckDB almost everywhere, |
| 10 | +and this script measures the exact exceptions and freezes them into a table. |
| 11 | +
|
| 12 | +The exceptions come from two sources, both captured by measurement: |
| 13 | + 1. Simple-vs-full divergence: codepoints whose full map is multi-char but |
| 14 | + which have a DIFFERENT 1:1 simple map (the sharp s, Turkish dotted I, |
| 15 | + the Greek ypogegrammeni block U+1F80..U+1FFC). |
| 16 | + 2. Unicode VERSION skew: case pairs added to Unicode after the utf8proc |
| 17 | + release inside this duckdb build (e.g. U+019B/U+A7DC from Unicode 16) — |
| 18 | + Rust maps them, DuckDB does not. DuckDB is the oracle, so its identity |
| 19 | + behavior wins. |
| 20 | +
|
| 21 | +Why no dependency |
| 22 | +----------------- |
| 23 | +A unicode-data crate would give us *some* Unicode version's simple maps — |
| 24 | +which is the wrong spec twice over: it ignores class 2 entirely (the crate's |
| 25 | +tables won't match utf8proc's vintage), and it drags a dependency in for |
| 26 | +what measurement shows is ~83 codepoints. The measured table is smaller, |
| 27 | +exactly right by construction, and verified end-to-end by the full-codepoint |
| 28 | +census in tests/test_duckdb_interpreter.py (which will fail loudly if this |
| 29 | +table ever drifts from the duckdb the suite runs against). |
| 30 | +
|
| 31 | +Regenerate with `uv run python scripts/gen_casemap.py` after bumping the |
| 32 | +duckdb dependency, then run the gate: the census test is the authority. |
| 33 | +""" |
| 34 | + |
| 35 | +from __future__ import annotations |
| 36 | + |
| 37 | +import sys |
| 38 | +from pathlib import Path |
| 39 | + |
| 40 | +import duckdb |
| 41 | +import pyarrow as pa |
| 42 | + |
| 43 | +REPO = Path(__file__).resolve().parent.parent |
| 44 | +OUT = REPO / "src" / "specializer" / "exec" / "casemap.rs" |
| 45 | + |
| 46 | + |
| 47 | +def rust_rule(c: str, upper: bool) -> str: |
| 48 | + """Phase-1 approximation of the engine's dependency-free fallback (full |
| 49 | + map iff 1:1, else keep). Python's tables APPROXIMATE Rust's: all three of |
| 50 | + Python, Rust, and duckdb's utf8proc can ship different Unicode versions, |
| 51 | + so phase 2 measures the actual compiled engine and patches the residue |
| 52 | + (e.g. Unicode-16 case pairs Rust maps but Python does not).""" |
| 53 | + m = c.upper() if upper else c.lower() |
| 54 | + return m if len(m) == 1 else c |
| 55 | + |
| 56 | + |
| 57 | +def engine_residue(chunks: list[str], duck_u: list[str], duck_l: list[str]): |
| 58 | + """Phase 2: run the census through the INSTALLED engine and report every |
| 59 | + codepoint still diverging from duckdb — Rust-vs-Python table skew the |
| 60 | + phase-1 approximation cannot see. Requires the built sql_transform wheel; |
| 61 | + skipped (with a warning) when it is not importable.""" |
| 62 | + try: |
| 63 | + from pydantic import create_model |
| 64 | + |
| 65 | + from sql_transform._interpreter import DuckDBInferFn |
| 66 | + except ImportError as e: # pragma: no cover -- generator convenience |
| 67 | + print( |
| 68 | + f"WARNING: engine not importable ({e}); phase 2 skipped — " |
| 69 | + "rebuild and rerun, the census test is the authority" |
| 70 | + ) |
| 71 | + return [], [] |
| 72 | + model = create_model("Row", s=(str, ...)) |
| 73 | + fn = DuckDBInferFn( |
| 74 | + "SELECT upper(s) AS u, lower(s) AS l FROM __THIS__", |
| 75 | + row_tables={"__THIS__": model}, |
| 76 | + static_tables={}, |
| 77 | + ) |
| 78 | + ru, rl = [], [] |
| 79 | + for s, du, dl in zip(chunks, duck_u, duck_l, strict=True): |
| 80 | + r = fn.infer({"__THIS__": [model(s=s)]})[0] |
| 81 | + ru.extend((ord(c), b) for c, a, b in zip(s, r.u, du, strict=True) if a != b) |
| 82 | + rl.extend((ord(c), b) for c, a, b in zip(s, r.l, dl, strict=True) if a != b) |
| 83 | + return ru, rl |
| 84 | + |
| 85 | + |
| 86 | +def main() -> int: |
| 87 | + con = duckdb.connect() |
| 88 | + cps = [cp for cp in range(1, 0x110000) if not (0xD800 <= cp <= 0xDFFF)] |
| 89 | + con.register("cps_arrow", pa.table({"cp": cps, "s": [chr(c) for c in cps]})) |
| 90 | + con.execute("CREATE TABLE cps AS SELECT * FROM cps_arrow") |
| 91 | + rows = con.execute( |
| 92 | + "SELECT cp, s, upper(s), lower(s) FROM cps ORDER BY cp" |
| 93 | + ).fetchall() |
| 94 | + |
| 95 | + upper_ex, lower_ex = {}, {} |
| 96 | + for cp, s, u, lo_ in rows: |
| 97 | + assert len(u) == 1 and len(lo_) == 1, f"non-1:1 duckdb map at U+{cp:04X}" |
| 98 | + if rust_rule(s, True) != u: |
| 99 | + upper_ex[cp] = u |
| 100 | + if rust_rule(s, False) != lo_: |
| 101 | + lower_ex[cp] = lo_ |
| 102 | + |
| 103 | + # Carry forward every codepoint already in casemap.rs, re-measured |
| 104 | + # against duckdb. Phase 2 measures the INSTALLED engine, which already |
| 105 | + # contains the current table — so skew entries it fixed report no |
| 106 | + # residue and would be silently dropped on regeneration without this. |
| 107 | + # An entry whose value equals what the fallback gives is harmless. |
| 108 | + if OUT.exists(): |
| 109 | + import re |
| 110 | + |
| 111 | + by_cp = {cp: (u, lo_) for cp, _, u, lo_ in rows} |
| 112 | + text = OUT.read_text(encoding="utf-8") |
| 113 | + entry = r"\('\\u\{([0-9A-F]+)\}', '\\u\{[0-9A-F]+\}'\)" |
| 114 | + # Split at the const DEFINITION (the lookup fns mention the names too). |
| 115 | + upper_part, _, lower_part = text.partition("const LOWER_EXCEPTIONS") |
| 116 | + for part, ex, idx in ((upper_part, upper_ex, 0), (lower_part, lower_ex, 1)): |
| 117 | + for m in re.finditer(entry, part): |
| 118 | + cp = int(m.group(1), 16) |
| 119 | + if cp in by_cp: |
| 120 | + ex.setdefault(cp, by_cp[cp][idx]) |
| 121 | + |
| 122 | + # Phase 2: census the compiled engine in chunks, patch Rust-side skew. |
| 123 | + step = 0x8000 |
| 124 | + chunks, duck_u, duck_l = [], [], [] |
| 125 | + for lo in range(1, 0x110000, step): |
| 126 | + s = "".join( |
| 127 | + chr(c) |
| 128 | + for c in range(lo, min(lo + step, 0x110000)) |
| 129 | + if not (0xD800 <= c <= 0xDFFF) |
| 130 | + ) |
| 131 | + if s: |
| 132 | + chunks.append(s) |
| 133 | + u, lo_ = con.execute("SELECT upper(?), lower(?)", [s, s]).fetchone() |
| 134 | + duck_u.append(u) |
| 135 | + duck_l.append(lo_) |
| 136 | + ru, rl = engine_residue(chunks, duck_u, duck_l) |
| 137 | + n_skew = len([cp for cp, _ in ru if cp not in upper_ex]) + len( |
| 138 | + [cp for cp, _ in rl if cp not in lower_ex] |
| 139 | + ) |
| 140 | + upper_ex.update(ru) |
| 141 | + lower_ex.update(rl) |
| 142 | + print(f"phase 2: {n_skew} Rust-vs-Python skew entries patched in") |
| 143 | + upper_ex = sorted(upper_ex.items()) |
| 144 | + lower_ex = sorted(lower_ex.items()) |
| 145 | + |
| 146 | + def table(name: str, entries: list[tuple[int, str]]) -> str: |
| 147 | + body = "\n".join( |
| 148 | + f" ('\\u{{{cp:04X}}}', '\\u{{{ord(m):04X}}}'), // {chr(cp)!r} -> {m!r}" |
| 149 | + for cp, m in entries |
| 150 | + ) |
| 151 | + return f"pub(super) const {name}: &[(char, char)] = &[\n{body}\n];\n" |
| 152 | + |
| 153 | + duck_version = duckdb.__version__ |
| 154 | + OUT.write_text( |
| 155 | + f"""//! DuckDB's SIMPLE case mapping, as measured — see scripts/gen_casemap.py |
| 156 | +//! for the full story (simple-vs-full mapping, Unicode version skew, and why |
| 157 | +//! a dependency would be the wrong spec twice over). |
| 158 | +//! |
| 159 | +//! GENERATED by `uv run python scripts/gen_casemap.py` against duckdb-python |
| 160 | +//! {duck_version}; regenerate after a duckdb bump. The full-codepoint census in |
| 161 | +//! tests/test_duckdb_interpreter.py fails loudly if this table drifts from |
| 162 | +//! the duckdb the suite runs against — the census, not this file, is the |
| 163 | +//! authority. |
| 164 | +//! |
| 165 | +//! Entries are (codepoint, mapped) pairs, sorted by codepoint, covering only |
| 166 | +//! the codepoints where the dependency-free fallback (Rust's full map iff it |
| 167 | +//! is 1:1, else identity) disagrees with DuckDB. Everything else goes through |
| 168 | +//! the fallback in `simple_upper`/`simple_lower`. |
| 169 | +
|
| 170 | +/// Simple uppercase for one codepoint, exactly as DuckDB computes it. |
| 171 | +pub fn simple_upper(c: char) -> char {{ |
| 172 | + if let Ok(i) = UPPER_EXCEPTIONS.binary_search_by_key(&c, |e| e.0) {{ |
| 173 | + return UPPER_EXCEPTIONS[i].1; |
| 174 | + }} |
| 175 | + let mut it = c.to_uppercase(); |
| 176 | + let first = it.next().expect("case iterators are non-empty"); |
| 177 | + if it.next().is_none() {{ |
| 178 | + first |
| 179 | + }} else {{ |
| 180 | + c |
| 181 | + }} |
| 182 | +}} |
| 183 | +
|
| 184 | +/// Simple lowercase for one codepoint, exactly as DuckDB computes it. |
| 185 | +pub fn simple_lower(c: char) -> char {{ |
| 186 | + if let Ok(i) = LOWER_EXCEPTIONS.binary_search_by_key(&c, |e| e.0) {{ |
| 187 | + return LOWER_EXCEPTIONS[i].1; |
| 188 | + }} |
| 189 | + let mut it = c.to_lowercase(); |
| 190 | + let first = it.next().expect("case iterators are non-empty"); |
| 191 | + if it.next().is_none() {{ |
| 192 | + first |
| 193 | + }} else {{ |
| 194 | + c |
| 195 | + }} |
| 196 | +}} |
| 197 | +
|
| 198 | +{table("UPPER_EXCEPTIONS", upper_ex)} |
| 199 | +{table("LOWER_EXCEPTIONS", lower_ex)}""", |
| 200 | + encoding="utf-8", |
| 201 | + ) |
| 202 | + print(f"wrote {OUT}: {len(upper_ex)} upper + {len(lower_ex)} lower exceptions") |
| 203 | + return 0 |
| 204 | + |
| 205 | + |
| 206 | +if __name__ == "__main__": |
| 207 | + sys.exit(main()) |
0 commit comments