Skip to content

Commit 0782bf1

Browse files
ahrzbclaude
andcommitted
feat(specializer): static-only queries become constant emitters — AC #2 (M-lower stretch 6)
A query whose driving relation is a static table is evaluated ONCE at build time by DuckDB itself; infer() returns the fixed rows (re-validated through the output model per call) and nothing dynamic remains — no IR, no probes, no filters. The fallback self-validates: it fires on Unsupported/Parse errors and a dynamic query always references the row table, which DuckDB does not know, so evaluation fails and the original clean error surfaces unchanged. Bind errors stay hard. Bonus breadth: aggregation, ORDER BY, and DuckDB-dialect-beyond-sqlparser all work on the static-only path because DuckDB is the evaluator. Corpus: 53 match / 625 clean-unsupported / 0 FAIL. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 716b0bb commit 0782bf1

3 files changed

Lines changed: 167 additions & 23 deletions

File tree

src/duckdb/mod.rs

Lines changed: 117 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -110,26 +110,80 @@ fn materialize_map(
110110
Ok(StaticData::Map(entries))
111111
}
112112

113-
fn synthesize_output_model(py: Python<'_>, out_cols: &[Col]) -> PyResult<Py<PyAny>> {
113+
fn model_from_fields(py: Python<'_>, fields: Vec<(String, FieldType)>) -> PyResult<Py<PyAny>> {
114114
let create_model = PyModule::import(py, "pydantic")?.getattr("create_model")?;
115115
let ellipsis = PyModule::import(py, "builtins")?.getattr("Ellipsis")?;
116116
let kwargs = PyDict::new(py);
117-
for c in out_cols {
118-
let ft = FieldType {
119-
base: ty_to_base(c.ty.ty),
120-
nullable: c.ty.nullable,
121-
};
122-
kwargs.set_item(&c.name, (schema::field_type_to_python(py, ft)?, &ellipsis))?;
117+
for (name, ft) in fields {
118+
kwargs.set_item(name, (schema::field_type_to_python(py, ft)?, &ellipsis))?;
123119
}
124120
Ok(create_model.call(("OutputRow",), Some(&kwargs))?.unbind())
125121
}
126122

123+
fn synthesize_output_model(py: Python<'_>, out_cols: &[Col]) -> PyResult<Py<PyAny>> {
124+
let fields = out_cols
125+
.iter()
126+
.map(|c| {
127+
(
128+
c.name.clone(),
129+
FieldType {
130+
base: ty_to_base(c.ty.ty),
131+
nullable: c.ty.nullable,
132+
},
133+
)
134+
})
135+
.collect();
136+
model_from_fields(py, fields)
137+
}
138+
139+
/// AC #2's constant emitter: a static-tables-only query is evaluated ONCE,
140+
/// here at build time, by DuckDB itself — nothing dynamic remains and no IR
141+
/// is built at all. Statics materialize as native tables (duckdb's
142+
/// registered-arrow scan path has divergent filter semantics — see the
143+
/// builtin-pins spec). Returns the fixed row dicts plus the result schema.
144+
fn eval_static_only(
145+
py: Python<'_>,
146+
sql: &str,
147+
static_tables: &HashMap<String, Py<PyAny>>,
148+
) -> PyResult<(Vec<Py<PyAny>>, Vec<(String, FieldType)>)> {
149+
let duckdb = PyModule::import(py, "duckdb")?;
150+
let con = duckdb.call_method0("connect")?;
151+
for (name, table) in static_tables {
152+
con.call_method1("register", (format!("__arrow_{name}"), table))?;
153+
con.call_method1(
154+
"execute",
155+
(format!(
156+
"CREATE TABLE \"{name}\" AS SELECT * FROM \"__arrow_{name}\""
157+
),),
158+
)?;
159+
}
160+
let arrow = con
161+
.call_method1("execute", (sql,))?
162+
.call_method0("to_arrow_table")?;
163+
let schema_obj = arrow.getattr("schema")?.unbind();
164+
let fields = schema::arrow_schema_to_ordered_fields(py, &schema_obj)?;
165+
let mut rows = Vec::new();
166+
for r in arrow.call_method0("to_pylist")?.try_iter()? {
167+
rows.push(r?.unbind());
168+
}
169+
Ok((rows, fields))
170+
}
171+
172+
enum Engine {
173+
Compiled {
174+
fun: InterpFn,
175+
in_cols: Vec<Col>,
176+
out_cols: Vec<Col>,
177+
},
178+
/// Fixed row dicts from a static-only query, re-validated through the
179+
/// output model on every `infer` call.
180+
Constant { rows: Vec<Py<PyAny>> },
181+
}
182+
127183
#[pyclass(unsendable)]
128184
pub struct DuckDBInferFn {
129-
fun: InterpFn,
185+
engine: Engine,
130186
row_table: String,
131-
in_cols: Vec<Col>,
132-
out_cols: Vec<Col>,
133187
#[pyo3(get)]
134188
output_model: Py<PyAny>,
135189
}
@@ -199,8 +253,33 @@ impl DuckDBInferFn {
199253
});
200254
}
201255

202-
let prepared =
203-
prepare(&sql, &row_table, &in_cols, &catalog).map_err(|e| build_err(e.to_string()))?;
256+
use super::specializer::PrepareError;
257+
let prepared = match prepare(&sql, &row_table, &in_cols, &catalog) {
258+
Ok(p) => p,
259+
// Unsupported/unparseable SQL might still be a static-tables-only
260+
// query (static driving table, aggregation, ORDER BY, DuckDB
261+
// dialect beyond sqlparser): try the constant-emitter path. It
262+
// self-validates — a dynamic query references the row table,
263+
// which DuckDB does not know, so evaluation fails and the
264+
// original clean error surfaces unchanged. Bind errors stay hard.
265+
Err(e @ (PrepareError::Unsupported(_) | PrepareError::Parse(_))) => {
266+
match eval_static_only(py, &sql, &static_tables) {
267+
Ok((rows, fields)) => {
268+
let output_model = match output_model {
269+
Some(m) => m,
270+
None => model_from_fields(py, fields)?,
271+
};
272+
return Ok(DuckDBInferFn {
273+
engine: Engine::Constant { rows },
274+
row_table,
275+
output_model,
276+
});
277+
}
278+
Err(_) => return Err(build_err(e.to_string())),
279+
}
280+
}
281+
Err(e) => return Err(build_err(e.to_string())),
282+
};
204283

205284
// Program statics and StaticSpecs are both indexed by join id.
206285
let mut data = Vec::with_capacity(prepared.statics.len());
@@ -221,10 +300,12 @@ impl DuckDBInferFn {
221300
None => synthesize_output_model(py, &prepared.program.out_cols)?,
222301
};
223302
Ok(DuckDBInferFn {
224-
fun,
303+
engine: Engine::Compiled {
304+
fun,
305+
in_cols,
306+
out_cols: prepared.program.out_cols.clone(),
307+
},
225308
row_table,
226-
in_cols,
227-
out_cols: prepared.program.out_cols.clone(),
228309
output_model,
229310
})
230311
}
@@ -250,9 +331,24 @@ impl DuckDBInferFn {
250331
}
251332
let rows = merged.remove(&self.row_table).unwrap_or_default();
252333

334+
let (fun, in_cols, out_cols) = match &self.engine {
335+
Engine::Compiled {
336+
fun,
337+
in_cols,
338+
out_cols,
339+
} => (fun, in_cols, out_cols),
340+
Engine::Constant { rows: fixed } => {
341+
let model = self.output_model.bind(py);
342+
let mut out = Vec::with_capacity(fixed.len());
343+
for r in fixed {
344+
out.push(model.call_method1("model_validate", (r,))?.unbind());
345+
}
346+
return Ok(out);
347+
}
348+
};
349+
253350
let n = rows.len();
254-
let mut cols: Vec<ColData> = self
255-
.in_cols
351+
let mut cols: Vec<ColData> = in_cols
256352
.iter()
257353
.map(|c| match c.ty.ty {
258354
Ty::I1 => ColData::I1 {
@@ -275,7 +371,7 @@ impl DuckDBInferFn {
275371
.collect();
276372
for row_obj in &rows {
277373
let bound = row_obj.bind(py);
278-
for (c, col) in self.in_cols.iter().zip(&mut cols) {
374+
for (c, col) in in_cols.iter().zip(&mut cols) {
279375
let attr = bound.getattr(c.name.as_str()).map_err(|e| {
280376
pyo3::exceptions::PyValueError::new_err(format!(
281377
"Row for table '{}' is missing attribute '{}': {e}",
@@ -311,16 +407,15 @@ impl DuckDBInferFn {
311407
}
312408

313409
let batch = Batch { rows: n, cols };
314-
let mut st = self.fun.new_state();
315-
self.fun
316-
.run(&batch, &mut st)
410+
let mut st = fun.new_state();
411+
fun.run(&batch, &mut st)
317412
.map_err(|t| PyErr::from(InterpError::Eval(t.0)))?;
318413

319414
let model = self.output_model.bind(py);
320415
let mut out = Vec::with_capacity(st.emitted);
321416
for r in 0..st.emitted {
322417
let dict = PyDict::new(py);
323-
for (c, oc) in self.out_cols.iter().zip(&st.out) {
418+
for (c, oc) in out_cols.iter().zip(&st.out) {
324419
match oc {
325420
OutCol::I1(v) => {
326421
let (ok, x) = v[r];

src/specializer/ir/gen.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,7 @@ fn compute(rng: &mut Rng, b: &mut Builder, scope: &mut Scope, insts: &mut Vec<In
344344
// Positions are small fresh consts, not arbitrary scope
345345
// values: the ±2^32 range guard traps, and generated
346346
// programs must stay executable for M-interp fuzzing.
347-
let mut small = |rng: &mut Rng, b: &mut Builder, insts: &mut Vec<Inst>| {
347+
let small = |rng: &mut Rng, b: &mut Builder, insts: &mut Vec<Inst>| {
348348
let dst = b.fresh();
349349
insts.push(Inst::Const {
350350
dst,

tests/test_duckdb_interpreter.py

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,3 +362,52 @@ def test_nan_filter_differential_on_native_tables():
362362
{"x": "float?"},
363363
[{"x": float("nan")}, {"x": 1.0}, {"x": None}, {"x": float("inf")}],
364364
)
365+
366+
367+
# ------------------------------------------------- static-only queries:
368+
# AC #2 — evaluated once at build time by DuckDB, nothing dynamic remains.
369+
370+
371+
def test_static_only_query_is_a_constant_emitter():
372+
dim = static(
373+
{"id": "int", "name": "str"},
374+
[
375+
{"id": 1, "name": "one"},
376+
{"id": 2, "name": "two"},
377+
{"id": 3, "name": "three"},
378+
],
379+
)
380+
model = _row_model({"a": "int"})
381+
fn = DuckDBInferFn(
382+
"SELECT name, id * 10 AS x FROM dim WHERE id <> 2 ORDER BY id DESC",
383+
row_tables={"__THIS__": model},
384+
static_tables={"dim": dim},
385+
)
386+
# Input rows are irrelevant; the result is fixed at build time —
387+
# and constructs like ORDER BY work because DuckDB itself evaluated it.
388+
for rows_in in ([], [model(a=1)], [model(a=1), model(a=2)]):
389+
got = [r.model_dump() for r in fn.infer({"__THIS__": rows_in})]
390+
assert got == [
391+
{"name": "three", "x": 30},
392+
{"name": "one", "x": 10},
393+
]
394+
395+
396+
def test_static_only_aggregation_works_via_duckdb():
397+
dim = static({"v": "int"}, [{"v": 1}, {"v": 2}, {"v": 3}])
398+
fn = DuckDBInferFn(
399+
"SELECT sum(v) AS s FROM dim",
400+
row_tables={"__THIS__": _row_model({"a": "int"})},
401+
static_tables={"dim": dim},
402+
)
403+
assert [r.model_dump() for r in fn.infer({"__THIS__": []})] == [{"s": 6}]
404+
405+
406+
def test_unknown_driving_table_stays_clean_unsupported():
407+
# Not a static table either -> the original clean unsupported surfaces.
408+
with pytest.raises(ValueError, match="driving relation"):
409+
DuckDBInferFn(
410+
"SELECT x FROM nope",
411+
row_tables={"__THIS__": _row_model({"a": "int"})},
412+
static_tables={},
413+
)

0 commit comments

Comments
 (0)